Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 4330

Last change on this file since 4330 was 4302, checked in by Henrik Bettermann, 15 years ago

store ids when removing students and check for removed ids when csdding new students

WAeUPTable alraedy contains methods for position ind course calculation

Property svn:keywords set to Id

File size: 77.2 KB

Rev	Line
[3219]	1	# -- mode: python; mode: fold; --
[197]	2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
	3	# Author: Joachim Schmitz (js@aixtraware.de)
	4	#
	5	# This program is free software; you can redistribute it and/or modify
	6	# it under the terms of the GNU General Public License version 2 as published
	7	# by the Free Software Foundation.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program; if not, write to the Free Software
	16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
	17	# 02111-1307, USA.
	18	#
	19	# $Id: WAeUPTool.py 4302 2009-06-17 13:53:22Z henrik $
[1174]	20	"""The WAeUP Tool Box.
[197]	21	"""
	22
	23	from AccessControl import ClassSecurityInfo
[828]	24	from Acquisition import aq_inner
	25	from Acquisition import aq_parent
	26	from Globals import DTMLFile
	27	from Globals import InitializeClass
	28	from OFS.SimpleItem import SimpleItem
[2283]	29	from zExceptions import BadRequest
[197]	30
[1890]	31	from Products.CMFCore.utils import getToolByName
[1747]	32	from Products.CPSSchemas.DataStructure import DataStructure
	33	from Products.CPSSchemas.DataModel import DataModel
	34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
[828]	35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
	36	from Products.CMFCore.permissions import View
	37	from Products.ZCatalog.ZCatalog import ZCatalog
	38	from Products.CMFCore.permissions import ModifyPortalContent
[1890]	39	from Products.CMFCore.permissions import ManagePortal
[197]	40	from Products.CMFCore.utils import UniqueObject
[1194]	41	from Products.CMFCore.URLTool import URLTool
[1747]	42	from Products.CMFCore.utils import getToolByName
[1151]	43	from Students import makeCertificateCode
[1285]	44	from Globals import package_home,INSTANCE_HOME
[3172]	45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
[3988]	46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport,PaymentImport
[3172]	47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
	48	from utils import makeDigest
[2094]	49	import DateTime,time
[1620]	50	import logging
[1170]	51	import transaction
[2193]	52	import csv,re,os,sys
[3172]	53	import md5
[3292]	54	from shutil import copy2,copy
[1707]	55	from Products.AdvancedQuery import Eq, Between, Le,In
[197]	56
[2413]	57	p_home = package_home(globals())
	58	i_home = INSTANCE_HOME
	59	images_base = os.path.join(i_home,"images")
[3198]	60	EMPTY = 'XXX'
[2413]	61
[2675]	62	def getImagesDir(student_id):
	63	return os.path.join("%s" % images_base,student_id[0],student_id)
[2413]	64
[1707]	65	def getObject(object,name):
	66	if object.hasObject(name):
	67	return getattr(object,name)
	68	return None
[1720]	69
[828]	70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
[197]	71	"""WAeUP tool"""
	72
[828]	73	id = 'waeup_tool'
[197]	74	meta_type = 'WAeUP Tool'
[828]	75	_actions = ()
[197]	76	security = ClassSecurityInfo()
[828]	77	security.declareObjectProtected(View)
	78	manage_options = ( ActionProviderBase.manage_options
	79	+ SimpleItem.manage_options
	80	)
	81
[2999]	82	security.declareProtected(View,'re_split') ###(
	83	def re_split(self,split_string,string):
	84	return re.split(split_string,string)
[3014]	85	###)
[2999]	86
	87	security.declareProtected(View,'difference') ###(
	88	def difference(self,l1,l2):
	89	return set(l1).difference(set(l2))
[3014]	90	###)
[2999]	91
[1818]	92	def rwrite(self,s): ###(
[1707]	93	response = self.REQUEST.RESPONSE
	94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
	95	response.write("%s<br />\r\n" % s)
[1818]	96	###)
[1174]	97
[2695]	98	def addtodict(self,d,key,item): ###(
	99	d[key].append(item)
	100	return d[key]
	101	###)
	102
[1818]	103	def sleep(self,secs): ###(
	104	"sleep"
	105	import time
	106	time.sleep(secs)
	107	return
[3473]	108	###)
[1827]	109
[2695]	110	security.declareProtected(View,'updateRoleMappingsFor') ###(
	111	def updateRoleMappingsFor(self,wf_definition,ob):
	112	"do so for public"
	113	wf_def = getattr(self.portal_workflow,wf_definition)
	114	wf_def.updateRoleMappingsFor(ob)
	115	###)
[2699]	116
[2999]	117	security.declareProtected(View,'getStatesLgas') ###(
	118	def getStatesLgas(self):
	119	"""return lga info"""
	120	voc = getattr(self.portal_vocabularies,'local_gov_areas')
	121	states = []
	122	lgas = []
	123	d = {}
	124	wd = {}
	125	for k,v in voc.items():
	126	parts = v.split(' / ')
	127	if len(parts) == 1:
	128	state = parts[0].lower()
	129	lga = ""
	130	elif len(parts) == 2:
	131	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
	132	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
	133	else:
	134	continue
	135	if state not in states:
	136	states.append(state)
	137	if lga not in lgas:
	138	lgas.append(lga)
	139	words = re.split('[^a-zA-Z0-9/]',k)
	140	words.sort()
	141	wd[k] = words
	142	d[k] = v
	143	mapping = {}
	144	mapping['word_dict'] = wd
	145	mapping['lga_dict'] = d
	146	mapping['states'] = states
	147	mapping['lgas'] = lgas
	148	return mapping
	149	###)
	150
[3014]	151	security.declareProtected(View,'findLga') ###(
	152	def findLga(self,words,words_dict):
	153	words = re.split('[^a-zA-Z0-9/]',words)
	154	lga_words = []
	155	for word in words:
	156	if word:
	157	lga_words += word.strip().lower(),
	158	lga_words.sort()
	159	state_lga = ''
	160	while not state_lga:
	161	for k,l in words_dict.items():
	162	if lga_words == l:
	163	state_lga = k
	164	break
	165	break
	166	return state_lga
	167	###)
[2975]	168	security.declareProtected(View,'getAccessInfo') ###(
	169	def getAccessInfo(self,context):
	170	"return a dict with access_info"
	171	logger = logging.getLogger('WAeUPTool.getAccessInfo')
	172	mtool = self.portal_membership
	173	member = mtool.getAuthenticatedMember()
	174	member_id = str(member)
	175	info = {}
	176	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
	177	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
	178	is_staff = info['is_staff'] = not is_anonymous and not is_student
	179	roles = member.getRolesInContext(context)
[3452]	180	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
	181	"SectionManager" in roles or
	182	"Manager" in roles)
	183	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
[3858]	184	#is_allowed = info['is_allowed'] = not is_anonymous
[2975]	185	requested_id = context.getStudentId()
	186	student_id = None
[3858]	187	if not is_anonymous and requested_id:
	188	if (is_student and member_id == requested_id) or is_staff:
	189	student_id = requested_id
	190	else:
[3025]	191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
[2975]	192	info['student_id'] = student_id
	193	return info
	194	###)
	195
[1818]	196	security.declareProtected(ModifyPortalContent,'openLog') ###(
[1716]	197	def openLog(self,name):
	198	"""open a log file"""
	199	version = 1
	200	path = "%s/log/%s_%d.log" % (i_home,name,version)
	201	while os.path.exists(path):
	202	version += 1
	203	path = "%s/log/%s_%d.log" % (i_home,name,version)
	204	log = open(path,"w")
	205	return log
[2259]	206	###)
[1716]	207
[2259]	208	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
	209	def bypassQueueCatalog(self,enable=True):
	210	"""bypass the QueueCatalog by setting all indexes to process imediate,
	211	if enable is True (default) the old settings are restored
	212	"""
[1818]	213
[2259]	214	###)
	215
[2094]	216	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
	217	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
	218	"""measure Object access Time"""
	219	import random
	220	if hasattr(self,'portal_catalog_real'):
	221	aq_portal = self.portal_catalog_real.evalAdvancedQuery
	222	else:
	223	aq_portal = self.portal_catalog.evalAdvancedQuery
	224	nr_pts = int(nr_pts)
	225	probe = int(probe)
	226	intervall = probe/10
	227	objects = ("application","clearance","personal")
	228	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
	229	#i = random.randrange(num_objects)
	230	count = 0
	231	found = 0
	232	not_found = 0
	233	t_found = 0
	234	t_not_found = 0
	235	time_found = time_not_found = 0.0
	236	t_time_found = t_time_not_found = 0.0
	237	accessed = []
	238	t_min = 1000
	239	t_max = 0
	240	#import pdb;pdb.set_trace()
	241	students = self.portal_catalog(portal_type="Student")
	242	num_students = len(students)
	243	if method == "d":
	244	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
	245	res = aq_portal(query)
	246	brains = {}
	247	for r in res:
	248	sid = r.relative_path.split('/')[-2]
	249	if brains.has_key(sid):
	250	brains[sid][r.portal_type] = r
	251	else:
	252	brains[sid] = {r.portal_type : r}
	253	brains_list = brains.keys()
	254	num_objects = len(brains_list)
	255	else:
	256	num_objects = num_students
	257	print "="*40
	258	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
	259	probe,
	260	nr_pts,
	261	num_objects)
	262	print "nr found/not time found/not min/max"
	263	elapse = time.time()
	264	i_elapse = time.time()
	265	c_elapse = time.clock()
	266	for c in range(1,probe + 1):
	267	i = random.randrange(num_objects)
	268	if method in ('a','b','c'):
	269	student_brain = students[i]
	270	elif method == "d":
	271	#import pdb;pdb.set_trace()
	272	student_brain = brains[brains_list[i]]
	273	if method == "c":
	274	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
	275	res = aq_portal(query)
	276	this_portal_types = [r.portal_type for r in res]
	277	for i in range(nr_pts):
	278	oid = objects[i]
	279	if method == "a":
	280	try:
	281	student_path = student_brain.getPath()
	282	path = "%s/%s" % (student_path,oid)
	283	doc = self.unrestrictedTraverse(path).getContent()
	284	found += 1
	285	i_time = time.time() - i_elapse
	286	time_found += i_time
	287	except:
	288	not_found += 1
	289	i_time = time.time() - i_elapse
	290	time_not_found += i_time
	291	pass
	292	elif method == "b":
	293	try:
	294	student_object = student_brain.getObject()
	295	doc = getattr(student_object,oid).getContent()
	296	found += 1
	297	i_time = time.time() - i_elapse
	298	time_found += i_time
	299	except:
	300	i_time = time.time() - i_elapse
	301	time_not_found += i_time
	302	not_found += 1
	303	pass
	304	elif method == "c":
	305	if portal_types[i] in this_portal_types:
	306	found += 1
	307	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
	308	i_time = time.time() - i_elapse
	309	time_found += i_time
	310	else:
	311	not_found += 1
	312	i_time = time.time() - i_elapse
	313	time_not_found += i_time
	314	elif method == "d":
	315	if student_brain.has_key(portal_types[i]):
	316	found += 1
	317	doc = student_brain[portal_types[i]].getObject().getContent()
	318	i_time = time.time() - i_elapse
	319	time_found += i_time
	320	else:
	321	not_found += 1
	322	i_time = time.time() - i_elapse
	323	time_not_found += i_time
	324	i_elapse = time.time()
	325	if c and (c % intervall == 0):
	326	#i_time = time.time() - i_elapse
	327	t_per = 0.0
	328	if found:
	329	t_per = time_found/found
	330	if t_per > t_max:
	331	t_max = t_per
	332	if t_per > 0.0 and t_per < t_min:
[2119]	333	t_min = t_per
[2094]	334	itf = 0.0
	335	if found:
	336	itf = time_found/found
	337	itnf = 0.0
	338	if not_found :
	339	itnf = time_not_found / not_found
[2119]	340	interval_time = time_found + time_not_found
	341	s = "%(c)d: %(found)d/%(not_found)d " % vars()
[2094]	342	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	343	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	344	print s
	345	t_found += found
	346	t_not_found += not_found
	347	t_time_found += time_found
	348	t_time_not_found += time_not_found
	349	time_found = time_not_found = 0.0
	350	found = not_found = 0
	351	# t_found += found
	352	# t_not_found += not_found
	353	elapse = time.time() - elapse
	354	itf = 0.0
	355	if t_found:
	356	itf = t_time_found/t_found
	357	itnf = 0.0
	358	if t_not_found:
	359	itnf = t_time_not_found / t_not_found
	360	#c_elapse = time.clock() - c_elapse
[2119]	361	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
[2094]	362	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	363	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	364	print "-"*40
	365	print s
	366	rel_found = float(t_found)/probe
	367	rel_not_found = float(t_not_found)/probe
	368	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
	369	print estimated_total_time
	370	###)
	371
[1818]	372	security.declareProtected(ModifyPortalContent,'writeLog') ###(
[1716]	373	def writeLog(self,logfile,s):
	374	"""write to the log file"""
	375	logfile.write(s)
[3473]	376	###)
[1716]	377
[1151]	378	def generateStudentId(self,letter): ###(
	379	import random
[3812]	380	logger = logging.getLogger('WAeUPTool.generateStudentId')
[1151]	381	r = random
[1194]	382	if letter == '?':
[2695]	383	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
[1151]	384	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1720]	385	students = self.portal_url.getPortalObject().campus.students
[4302]	386	while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid) or self.removed_student_ids(id = sid):
	387	if self.waeup_tool.picturePathExists(sid):
	388	logger.info('picture path %s exists or in ' % sid)
[1720]	389	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1151]	390	return sid
[1460]	391	###)
[1415]	392
	393	def generatePassword(self,s=None): ###(
	394	import random
	395	r = random
	396	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
	397	if s is None:
	398	s = 'abcdefghklmnpqrstuvwxy23456789'
	399	pw = ''
	400	while len(pw) < 6:
	401	pw += r.choice(s)
	402	return pw
[1151]	403	###)
[828]	404
[2645]	405	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
	406	def dumpSchoolfeePayments(self):
	407	"dump paid schoolfees"
[2666]	408	mtool = self.portal_membership
	409	member = mtool.getAuthenticatedMember()
[2645]	410	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
	411	aq_student = self.students_catalog.evalAdvancedQuery
	412	query = In('review_state',('schoolfee_paid',
	413	'courses_registered',
	414	'courses_validated',
	415	))
	416	res = aq_student(query)
[2666]	417	#import pdb;pdb.set_trace()
[2645]	418	l = []
	419	logger.info("start for %d" % len(res))
	420	count = 1
	421	log_after = 100
	422	for student in res:
[2656]	423	if not count % log_after:
[2645]	424	logger.info("processed %d total %d" % (log_after,count))
	425	count += 1
	426	fee_dict =self.getSchoolFee(student)
	427	fulltime = student.mode.endswith('_ft')
	428	d = {}
	429	d['student_id'] = student.id
[2666]	430	d['name'] = student.name
	431	d['amount'] = fee_dict.get(new_returning)
[2645]	432	l += d,
[2666]	433	csv_name = self.dumpListToCSV(l,'payments')
	434	logger.info('%s dumped payments to %s' % (member,export_file))
[2645]	435	###)
	436
[2666]	437	security.declarePublic('dumpListToCSV') ###(
	438	def dumpListToCSV(self,l,filename,fields=None):
	439	"""dump a list of dicts to a CSV file"""
	440	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	441	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
	442	if fields is None:
	443	fields = l[0].keys()
	444	headline = ','.join(fields)
	445	out = open(export_file,"wb")
	446	out.write(headline +'\n')
	447	out.close()
	448	out = open(export_file,"a")
	449	csv_writer = csv.DictWriter(out,fields,)
	450	csv_writer.writerows(l)
	451	return export_file
	452	###)
	453
[2643]	454	security.declareProtected(ManagePortal, 'listMembers') ###(
	455	def listMembers(self):
	456	"list all members"
	457	mtool = self.portal_membership
	458	member = mtool.getAuthenticatedMember()
	459	logger = logging.getLogger('WAeUPTool.listMembers')
	460	if str(member) not in ('admin','joachim'):
	461	logger.info('%s tried to list members' % (member))
	462	return None
	463	members = self.portal_directories.members
	464	all = members.listEntryIdsAndTitles()
	465	l = []
	466	for user_id,name in all:
	467	d = {}
	468	d['user_id'] = user_id
	469	d['name'] = name
	470	d['pw'] = getattr(getattr(members,user_id),'password')
[2645]	471	d['email'] = getattr(getattr(members,user_id),'email')
	472	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
	473	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
[2643]	474	l += d,
	475	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	476	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
[3272]	477	logger.info('%s dumped member list to %s' % (member,export_file))
[2643]	478	fields = l[0].keys()
	479	headline = ','.join(fields)
	480	out = open(export_file,"wb")
	481	out.write(headline +'\n')
	482	out.close()
	483	out = open(export_file,"a")
	484	csv_writer = csv.DictWriter(out,fields,)
	485	csv_writer.writerows(l)
	486	###)
[3300]	487
[3272]	488	security.declareProtected(ManagePortal, 'listStudents') ###(
	489	def listStudents(self):
	490	"list all students"
	491	mtool = self.portal_membership
	492	member = mtool.getAuthenticatedMember()
	493	logger = logging.getLogger('WAeUPTool.listStudents')
	494	if str(member) not in ('admin','joachim'):
	495	logger.info('%s tried to list students' % (member))
	496	return None
	497	students = self.portal_directories.students
	498	all = students.listEntryIdsAndTitles()
	499	l = []
	500	for user_id,name in all:
	501	d = {}
	502	d['user_id'] = user_id
	503	d['name'] = name
	504	d['pw'] = getattr(getattr(students,user_id),'password')
	505	d['email'] = getattr(getattr(students,user_id),'email')
	506	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
	507	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
	508	l += d,
	509	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	510	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
	511	logger.info('%s dumped student list to %s' % (member,export_file))
	512	fields = l[0].keys()
	513	headline = ','.join(fields)
	514	out = open(export_file,"wb")
	515	out.write(headline +'\n')
	516	out.close()
	517	out = open(export_file,"a")
	518	csv_writer = csv.DictWriter(out,fields,)
	519	csv_writer.writerows(l)
[3300]	520	###)
[2656]	521
[1890]	522	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
	523	def removeDeletedDocIds(self, max=1000):
	524	"""
	525	remove deleted docids from repository commit after max
	526	"""
	527	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
	528	repository = getToolByName(self, 'portal_repository')
	529	pxtool = getToolByName(self, 'portal_proxies')
[2632]	530	logger.info('start')
[1890]	531	pxtool_infos = pxtool.getRevisionsUsed()
[2632]	532	logger.info('found %d used revisions ' % (len(pxtool_infos)))
[1895]	533
[1890]	534	nb_revs = 0
	535	docids_d = {} # all docids
	536	unused_docids_d = {} # all docids that are unused
	537	ids_unused_revs_docids = [] # ids for revs of unused docids
[2632]	538	unused_ids = [] # ids for unused revs
[1890]	539	total = 0
[1895]	540	idlist = repository.objectIds()
[2632]	541	to_delete = 0
	542	found = False
[1895]	543	for id in idlist:
[1890]	544	docid, rev = repository._splitId(id)
	545	if docid is None:
	546	logger.info("invalid doc_id %s" % docid)
	547	continue
	548	nb_revs += 1
	549	if not pxtool_infos.has_key(docid):
[2632]	550	found = True
	551	to_delete += 1
	552	unused_ids.append(id)
[1890]	553	elif not pxtool_infos[docid].has_key(rev):
[2632]	554	found = True
	555	to_delete += 1
	556	unused_ids.append(id)
	557	if found and not to_delete % max:
	558	found = False
[1890]	559	#import pdb;pdb.set_trace()
[2632]	560	repository.manage_delObjects(unused_ids)
[1890]	561	transaction.commit()
[2632]	562	logger.info('removed %d total %d unused docids ' % (max,to_delete))
	563	else:
	564	if unused_ids:
	565	repository.manage_delObjects(unused_ids)
	566	transaction.commit()
	567	logger.info('finished removing %d unused docids ' % (to_delete))
[3473]	568	###)
[1890]	569
[3858]	570	security.declarePublic('getCredential') ###(
[1250]	571	def getCredential(self,student_id):
	572	student_entry = getattr(self.portal_directories.students,student_id,None)
[3858]	573	#import pdb;pdb.set_trace()
	574	#if not self.isStaff():
	575	# mtool = self.portal_membership
	576	# member = mtool.getAuthenticatedMember()
	577	# logger = logging.getLogger('WAeUPTool.getCredential')
	578	# logger.info('%s tried to access password of %s' % (member,student_id))
	579	# return None
[1250]	580	if student_entry is None:
[4022]	581	return "not set"
[1263]	582	return getattr(student_entry,"password","not set")
[1261]	583	###)
[1250]	584
[2747]	585	security.declarePublic('checkPassword') ###(
[1467]	586	def checkPassword(self,student_id,password):
[1460]	587	student_entry = getattr(self.portal_directories.students,student_id,None)
	588	if student_entry is None:
	589	return False
	590	return getattr(student_entry,"password","not set") == password
[2747]	591	###)
[2712]	592
[2747]	593	security.declarePublic('checkGenericPassword') ###(
[2710]	594	def checkGenericPassword(self,member_id):
	595	member_entry = getattr(self.portal_directories.members,member_id,None)
	596	if member_entry is None:
	597	return False
[2712]	598	ltool = getToolByName(self, 'portal_layouts')
[2710]	599	unsecure_words = ltool._getOb('members')['w__password'].check_words
[2712]	600	password = getattr(member_entry,"password","not set")
	601	is_unsecure = password in unsecure_words
	602	if is_unsecure:
[2713]	603	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
[2712]	604	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
	605	return is_unsecure
[2747]	606	###)
[1460]	607
[4027]	608	security.declarePublic('editPassword') ###(
[1467]	609	def editPassword(self,student_id,password):
	610	"edit a student password"
	611	student_entry = getattr(self.portal_directories.students,student_id,None)
	612	if student_entry is None:
[1571]	613	return
[1467]	614	setattr(student_entry,'password',password)
	615	###)
	616
[2555]	617	security.declareProtected(ModifyPortalContent,'doCommit') ###(
[1401]	618	def doCommit(self,logger=None):
	619	"commit some transactions"
	620	transaction.commit()
	621	###)
	622
[1285]	623	security.declarePublic('loadStudentFoto') ###(
[1813]	624	def loadStudentFoto(self,student,filename,folder):
[1285]	625	"return a student passport picture"
[2413]	626	#import pdb;pdb.set_trace()
	627	picture ="%s/import/%s/%s" % (i_home,folder,filename)
	628	student_id = student.getId()
[2675]	629	images_dir = getImagesDir(student_id)
[2413]	630	if not os.path.exists(images_dir):
	631	os.mkdir(images_dir)
	632	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	633	for extension in ('.jpg','.JPG'):
	634	fullname = "%(picture)s%(extension)s" % vars()
	635	if os.path.exists(fullname):
	636	copy2(fullname,image_name)
	637	return "successfully copied passport picture"
[2467]	638	return "passport picture not found: %s.jpg or .JPG" % picture
[2413]	639	###)
	640
[1285]	641
[1170]	642	security.declareProtected(ModifyPortalContent,'createOne') ###(
[1194]	643	def createOne(self,students_folder,student_brain,letter,commit=False):
	644	sid = self.waeup_tool.generateStudentId(letter)
[1170]	645	students_folder.invokeFactory('Student', sid)
	646	student = getattr(students_folder,sid)
	647	self.portal_workflow.doActionFor(student,'return')
	648	student.manage_setLocalRoles(sid, ['Owner',])
	649	matric_no = student_brain.matric_no
	650	jamb_reg_no = student_brain.Entryregno
	651	self.students_catalog.addRecord(id = sid,
	652	matric_no = matric_no,
	653	jamb_reg_no = jamb_reg_no,
	654	sex = student_brain.Sex == "F",
	655	name = "%s %s %s" % (student_brain.Firstname,
	656	student_brain.Middlename,
	657	student_brain.Lastname)
	658	)
	659	if commit:
	660	transaction.commit()
	661	return sid,jamb_reg_no
	662	###)
	663
[3840]	664	# Functions without docstring cannot be called from outside (via the URL),
	665	# thus we can declare addStudent public.
[3838]	666	security.declarePublic('addStudent') ###(
[1752]	667	def addStudent(self,dict):
[1415]	668	students_folder = self.portal_url.getPortalObject().campus.students
	669	sid = self.waeup_tool.generateStudentId('?')
	670	students_folder.invokeFactory('Student', sid)
	671	student_obj = getattr(students_folder,sid)
[3481]	672	f2t = StudentImport.field2types_student
[1772]	673	#from pdb import set_trace; set_trace()
	674	d = {}
[2297]	675	#d['jamb_sex'] = 'M'
	676	#if dict.get('sex'):
	677	# d['jamb_sex'] = 'F'
[1904]	678
[1898]	679	entry_session = dict.get('entry_session')
[2454]	680	if entry_session == self.getSessionId()[0]:
[1898]	681	wfaction = 'admit'
[1991]	682	wft = 'wf_transition_admit'
[1900]	683	password = None
[1904]	684	else:
[1900]	685	wfaction = 'return'
[1991]	686	wft = 'wf_transition_return'
[1900]	687	password = self.generatePassword()
[1904]	688	self.makeStudentMember(sid,password)
	689
[1728]	690	for pt in f2t.keys():
	691	student_obj.invokeFactory(pt,f2t[pt]['id'])
	692	sub_obj = getattr(student_obj,f2t[pt]['id'])
	693	sub_doc = sub_obj.getContent()
[1898]	694	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
[3481]	695	#d['Title'] = f2t[pt]['title']
[1728]	696	for field in f2t[pt]['fields']:
[1749]	697	d[field] = dict.get(field,'')
[1728]	698	sub_doc.edit(mapping = d)
[1898]	699	new_state = f2t[pt][wft]
[1772]	700	if new_state != "remain":
	701	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
[1415]	702	self.portal_workflow.doActionFor(student_obj,wfaction)
	703	student_obj.manage_setLocalRoles(sid, ['Owner',])
	704	return sid,password
	705	###)
	706
[1151]	707	security.declarePublic('getCertificateBrain') ###(
	708	def getCertificateBrain(self,cert_id):
	709	"do it"
[1849]	710	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	711	{'portal_type':"Certificate",
	712	'id': cert_id})
	713	if res:
	714	return res[0]
	715	return None
	716	###)
[1160]	717
[1756]	718	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
	719	def get_csv_filenames(self):
	720	"do it"
[1759]	721	files = [file for file in os.listdir("%s/import/" % (i_home))
[3143]	722	if file.endswith('.csv') and (file.find('imported') == -1 and
	723	file.find('pending') == -1)]
[1756]	724	return files
	725	###)
	726
[1151]	727	security.declarePublic('findStudentByMatricelNo') ###(
	728	def findStudentByMatricelNo(self,matric_no):
	729	"do it"
[1849]	730	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	731	{'portal_type':"StudentClearance",
	732	'SearchableText': matric_no})
	733	if res:
	734	return res[0]
	735	return None
	736	###)
	737
[3817]	738
	739	security.declarePublic('getOfficerName') ###(
	740	def getOfficerName(self,mid):
	741	"""get the real name of a member"""
	742	membership = self.portal_membership
	743	member_record = membership.getMemberById(mid)
	744	return member_record.getProperty('fullname',None)
	745	###)
	746
	747
[2557]	748	security.declarePublic('makeStudentMember') ###(
[1151]	749	def makeStudentMember(self,sid,password='uNsEt'):
	750	"""make the student a member"""
	751	membership = self.portal_membership
	752	membership.addMember(sid,
	753	password ,
	754	roles=('Member',
	755	'Student',
	756	),
	757	domains='',
	758	properties = {'memberareaCreationFlag': False,
	759	'homeless': True},)
	760	member = membership.getMemberById(sid)
	761	self.portal_registration.afterAdd(member, sid, password, None)
[1261]	762	#self.manage_setLocalRoles(sid, ['Owner',])
[1151]	763	###)
[1160]	764
[2560]	765	security.declareProtected(View,'makeStudentData') ###(
[1158]	766	def makeStudentData(self,student_id,email=None,phone_nr=None):
[1151]	767	"create Datastructure for a returning Student"
[1406]	768	#import pdb;pdb.set_trace()
[1571]	769	logger = logging.getLogger('WAeUPTool.makeStudentData')
[1151]	770	students_folder = self.portal_url.getPortalObject().campus.students
[1794]	771	#res = self.students_catalog(id=student_id)
	772	#if res:
	773	# st = res[0]
	774	#res = self.returning_import(matric_no = st.matric_no)
[1798]	775	res = self.returning_import(id = student_id)
[1151]	776	if res:
[1160]	777	student = res[0]
[1794]	778	else:
[1816]	779	logger.info('Id %s not found in returning_import' % student_id)
[1794]	780	return
[1580]	781	logger.info('%s creates data structure' % student_id)
[1794]	782	s_results = self.results_import(matric_no = student.matric_no)
[1816]	783	if s_results:
	784	lnr = self.getLevelFromResultsCosCode(s_results)
	785	level = "%d00" % lnr
	786	verdict,eligible = self.getVerdict(s_results[0].Verdict)
[2454]	787	#if eligible:
	788	# level = "%d00" % (lnr + 1)
[1816]	789	else:
	790	logger.info('matric_no %s not found in results_import' % student.matric_no)
[2656]	791	level = ''
	792	verdict = ''
[1171]	793	#student should not be allowed to perform this transition
[1174]	794	#wftool = self.portal_workflow
	795	#wftool.doActionFor(student,'return')
[1151]	796	certcode_org = student.Coursemajorcode
	797	certcode = makeCertificateCode(certcode_org)
	798	certificate_brain = self.getCertificateBrain(certcode)
	799	if not certificate_brain:
	800	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
	801	logger.info(em)
	802	matric_no = student.matric_no
	803	sid = student_id
	804	student_obj = getattr(students_folder,sid)
[3114]	805	if not getattr(student_obj,'application'):
	806	student_obj.invokeFactory('StudentApplication','application')
[1151]	807	application = student_obj.application
[1169]	808	self.portal_workflow.doActionFor(application,'open',dest_container=application)
[1151]	809	da = {'Title': 'Application Data'}
	810	student_obj.invokeFactory('StudentPersonal','personal')
	811	da['jamb_reg_no'] = student.Entryregno
[1462]	812	em = self.getEntryMode(student.Entryregno)
[1401]	813	da['entry_mode'] = em
[1151]	814	personal = student_obj.personal
	815	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
	816	dp = {'Title': 'Personal Data'}
	817	student_obj.invokeFactory('StudentClearance','clearance')
	818	clearance = student_obj.clearance
[1169]	819	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[1151]	820	dc = {'Title': 'Clearance/Eligibility Record'}
	821	dc['matric_no'] = matric_no
	822	state = student.State
	823	lga = student.LGA
	824	if state and lga:
	825	lga = state + ' / ' + lga
	826	else:
	827	lga = "None"
[1174]	828	da['jamb_lga'] = dc['lga'] = lga
[1173]	829	da['app_email'] = dp['email'] = email
	830	da['app_mobile'] = dp['phone'] = phone_nr
[1411]	831	dp['firstname'] = student.Firstname
	832	dp['middlename'] = student.Middlename
	833	dp['lastname'] = student.Lastname
	834	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
[1174]	835	da['jamb_sex'] = student.Sex
[1151]	836	dp['sex'] = student.Sex == 'F'
	837	dp['perm_address'] = student.Permanent_Address
	838	application.getContent().edit(mapping=da)
[1169]	839	self.portal_workflow.doActionFor(application,'close',dest_container=application)
[1151]	840	personal.getContent().edit(mapping=dp)
	841	clearance.getContent().edit(mapping=dc)
[1169]	842	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[1151]	843	#
	844	# Study Course
	845	#
	846	student_obj.invokeFactory('StudentStudyCourse','study_course')
	847	studycourse = student_obj.study_course
	848	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
	849	dsc = {}
	850	dsc['study_course'] = certcode
[1401]	851	dsc['current_level'] = level
	852	dsc['current_verdict'] = verdict
[3780]	853	dsc['current_mode'] = em #no longer used
[2466]	854	dsc['current_session'] = '05'
[1151]	855	studycourse.getContent().edit(mapping=dsc)
	856	#
	857	# Level
	858	#
[3473]	859	# l = getattr(studycourse,level,None)
	860	# if l is None:
	861	# studycourse.invokeFactory('StudentStudyLevel', level)
	862	# l = getattr(studycourse, level)
	863	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
	864	# l.getContent().edit(mapping={'Title': "Level %s" % level})
	865	###)
[1160]	866
[3349]	867	def init_timing(self): ###(
	868	if self.with_timing:
	869	if not hasattr(self,'_v_step_times'):
	870	self._v_timer_count = 0
	871	self._v_total = 0
	872	self._v_step_times = {}
	873	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	874	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
	875	self.timer_step = 0
	876	self.total_time = 0
	877	self.elapse = time.time()
	878	self.i_elapse = time.time()
	879	self.c_elapse = time.clock()
[3473]	880	###)
[3349]	881
	882	def do_timing(self): ###(
	883	if self.with_timing:
	884	try:
	885	raise 'dummy'
	886	except:
	887	frame = sys.exc_traceback.tb_frame.f_back
	888	locals = frame.f_locals
	889	globals = frame.f_globals
	890	functionname = frame.f_code.co_name
	891	filename = os.path.basename(frame.f_code.co_filename)
	892	lineno = frame.f_lineno
	893	mod_line = "%(functionname)s:%(lineno)s" % vars()
	894	i_time = time.time() - self.i_elapse
	895	td = {}
	896	if self._v_step_times.has_key(mod_line):
	897	a_time = self._v_step_times[mod_line]['a_time'] + i_time
	898	td['a_time'] = a_time
	899	else:
	900	td['a_time'] = i_time
	901	td['i_time'] = i_time
	902	self._v_step_times[mod_line] = td
	903	self.i_time = i_time
	904	self.total_time += i_time
	905	self.timer_step +=1
	906	self.i_elapse = time.time()
[3473]	907	###)
[3349]	908
	909	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
	910	def print_timing(self):
	911	if self.with_timing:
	912	l = []
	913	timer_count = self._v_timer_count + 1
	914	mod_lines = self._v_step_times.keys()
	915	mod_lines.sort(cmp,reverse=0)
	916	for mod_line in mod_lines:
	917	td = self._v_step_times[mod_line]
	918	i_time = td['i_time']
	919	a_time = td['a_time']/(self._v_timer_count + 1)
	920	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
	921	total_time = self.total_time
	922	total_avarage = self._v_total / timer_count
	923	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
	924	print "\r\n".join(l)
	925	out = open(self._v_timer_file,'a')
	926	out.write("\r\n".join(l))
	927	out.close()
[3473]	928	###)
[3349]	929
	930	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
	931	def get_timing_data(self):
	932	if self.with_timing:
	933	timer_count = self._v_timer_count + 1
	934	results = {}
	935	for k,d in self._v_step_times.items():
	936	dd = {}
	937	dd['a_time'] = d['a_time']/timer_count
	938	dd['i_time'] = d['i_time']
	939	dd['count'] = timer_count
	940	results[k] = dd
	941	dd = {}
	942	dd['a_time'] = self._v_total / timer_count
	943	dd['i_time'] = self.total_time
	944	dd['count'] = timer_count
	945	results["total"] = dd
	946	return results
	947	###)
	948
[2556]	949	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
[3349]	950	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
[2514]	951	"create Datastructure for an admitted Student"
	952	#import pdb;pdb.set_trace()
[3349]	953	logger = logging.getLogger('WAeUPTool.admitOneStudent')
	954	self.with_timing = with_timing
[3681]	955
[3705]	956	if brain.screening_type in ('cest','sandwich',):
	957	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
	958	else:
	959	reg_no = brain.reg_no
	960
[3681]	961	#ignore argument entry_session
	962	if not brain.entry_session:
	963	logger.info('no entry_session for %s provided' % (reg_no))
	964	return
[3961]	965	else:
	966	es = brain.entry_session
	967	if len(es) == 1:
	968	es = '0%c' % es
[3681]	969
[2727]	970	if not hasattr(self,"_v_certificates"):
[3339]	971	self._v_certificates = self.getCertificatesDict()
[2514]	972	students_folder = self.portal_url.getPortalObject().campus.students
[3705]	973
[3264]	974	res = self.students_catalog(jamb_reg_no = reg_no)
[3198]	975	if res:
[3800]	976	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
[3198]	977	return
[2514]	978	if brain.status != "admitted":
[3264]	979	logger.info('status of %s is %s' % (reg_no,brain.status))
[2540]	980	return
[2514]	981	pin_parts = brain.pin.split('-')
[2755]	982	if pin_parts and len(pin_parts) != 3:
[3264]	983	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
[2540]	984	return
[3764]	985	if not brain.course_admitted:
	986	logger.info('no course_admitted provided for %s' % (reg_no))
	987	return
[2727]	988	if brain.course_admitted not in self._v_certificates:
[3264]	989	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
[2727]	990	return
[2959]	991	if brain.sex not in (True,False):
[3264]	992	logger.info('sex of %s not available' % (reg_no))
[3002]	993	return
[3349]	994	self.init_timing()
[2540]	995	student_id = self.generateStudentId('?')
[2514]	996	students_folder.invokeFactory('Student', student_id)
	997	student_object = getattr(students_folder,student_id)
[3349]	998	self.do_timing()
[2576]	999	if pin_password:
	1000	password = pin_parts[2]
	1001	self.makeStudentMember(student_id,password = password)
[2540]	1002	student_object.manage_setLocalRoles(student_id, ['Owner',])
[3349]	1003	self.do_timing()
[3264]	1004	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
[2514]	1005	#
	1006	# application
	1007	#
[2540]	1008	student_object.invokeFactory('StudentApplication','application')
	1009	application = student_object.application
[2575]	1010	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
[3350]	1011	#self.do_timing()
[2514]	1012	da = {'Title': 'Application Data'}
[3264]	1013	da['jamb_reg_no'] = reg_no
[2576]	1014
[2540]	1015	sex = 'M'
	1016	if brain.sex:
	1017	sex = 'F'
	1018	da['jamb_sex'] = sex
[2670]	1019	da['jamb_age'] = brain.jamb_age
[3341]	1020	da['app_reg_pin'] = brain.pin
[2670]	1021	da['jamb_lga'] = brain.jamb_lga
	1022	da['jamb_state'] = brain.jamb_state
[2575]	1023	da['jamb_score'] = brain.aggregate
[2514]	1024	da['app_email'] = brain.email
[2540]	1025	da['app_mobile'] = brain.phone
[3339]	1026
[3775]	1027	# entry_mode cannot be retrieved from the certtificate
	1028	# because ug_ft has two different entry modes
	1029
[3780]	1030	if brain.entry_mode: # does not happen because import_application
	1031	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
[3775]	1032	elif brain.screening_type == 'pume':
[3776]	1033	da['entry_mode'] = 'ume_ft'
[3775]	1034	elif brain.screening_type == 'pde':
[3776]	1035	da['entry_mode'] = 'de_ft'
[3775]	1036	else:
	1037	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
	1038
	1039	#elif brain.screening_type == 'pce':
	1040	# da['entry_mode'] = 'pce'
	1041	#elif brain.screening_type == 'prence':
	1042	# da['entry_mode'] = 'prence'
	1043	#else:
	1044	# da['entry_mode'] = ''
[3776]	1045
[3339]	1046
[3681]	1047	#da['entry_session'] = entry_session
[3961]	1048	da['entry_session'] = es
[2575]	1049	da['jamb_lastname'] = brain.lastname
	1050	da['jamb_middlename'] = brain.middlenames # different field names!
[2670]	1051	da['jamb_firstname'] = brain.firstname
[2575]	1052	da['screening_application_date'] = brain.application_date
	1053	da['date_of_birth'] = brain.date_of_birth
	1054	da['jamb_first_cos'] = brain.course1
	1055	da['jamb_second_cos'] = brain.course2
	1056	da['course3'] = brain.course3
	1057	da['screening_type'] = brain.screening_type
	1058	da['screening_score'] = brain.screening_score
	1059	da['screening_date'] = brain.screening_date
	1060	da['hq_type'] = brain.hq_type
	1061	da['hq_grade'] = brain.hq_grade
	1062	da['aos'] = brain.aos
[2576]	1063
[2514]	1064	application.getContent().edit(mapping=da)
[3349]	1065	self.do_timing()
[2575]	1066	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
[2514]	1067	#
	1068	# personal
	1069	#
[2540]	1070	student_object.invokeFactory('StudentPersonal','personal')
	1071	personal = student_object.personal
[2575]	1072	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
[3350]	1073	#self.do_timing()
[2514]	1074	dp = {'Title': 'Personal Data'}
	1075	dp['sex'] = brain.sex
	1076	dp['email'] = brain.email
[2540]	1077	dp['phone'] = brain.phone
[2514]	1078	dp['lastname'] = brain.lastname
[2575]	1079	dp['middlename'] = brain.middlenames # different field names!
[2576]	1080	dp['firstname'] = brain.firstname
[2514]	1081	personal.getContent().edit(mapping=dp)
[3349]	1082	self.do_timing()
[2514]	1083	#
	1084	# clearance
	1085	#
[2540]	1086	student_object.invokeFactory('StudentClearance','clearance')
	1087	clearance = student_object.clearance
[2575]	1088	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[2514]	1089	dc = {'Title': 'Clearance/Eligibility Record'}
	1090	dc['lga'] = brain.lga
[2670]	1091	dc['birthday'] = brain.date_of_birth
[2514]	1092	clearance.getContent().edit(mapping=dc)
[3349]	1093	self.do_timing()
[2575]	1094	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[2514]	1095	#
	1096	# study Course
	1097	#
[2540]	1098	student_object.invokeFactory('StudentStudyCourse','study_course')
	1099	studycourse = student_object.study_course
[2575]	1100	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
[3350]	1101	#self.do_timing()
[2514]	1102	dsc = {}
	1103	dsc['study_course'] = brain.course_admitted
	1104	dsc['current_verdict'] = ''
[3780]	1105	dsc['current_mode'] = da['entry_mode'] # no longer used
[2575]	1106	if da['entry_mode'].startswith('de'):
	1107	dsc['current_level'] = '200'
[3040]	1108	elif da['entry_mode'].startswith('pre'):
	1109	dsc['current_level'] = '000'
[2575]	1110	else:
	1111	dsc['current_level'] = '100'
[3961]	1112	dsc['current_session'] = es
[2514]	1113	studycourse.getContent().edit(mapping=dsc)
[3349]	1114	self.do_timing()
[2514]	1115	#
	1116	# payments folder
[2540]	1117	student_object.invokeFactory('PaymentsFolder','payments')
	1118	payments = getattr(student_object,'payments')
[3350]	1119	#self.do_timing()
[2514]	1120	dpay = {}
	1121	dpay['Title'] = 'Payments'
	1122	payments.getContent().edit(mapping=dpay)
[2540]	1123	self.portal_workflow.doActionFor(payments,'open')
[3349]	1124	self.do_timing()
[2580]	1125	#
	1126	# passport foto
	1127	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
	1128	brain.screening_type,
	1129	brain.reg_no)
[2675]	1130	images_dir = getImagesDir(student_id)
	1131	#images_dir = os.path.join("%s" % images_base,student_id)
	1132	letter_dir,student_dir = os.path.split(images_dir)
	1133	if not os.path.exists(letter_dir):
	1134	os.mkdir(letter_dir)
[2580]	1135	if not os.path.exists(images_dir):
	1136	os.mkdir(images_dir)
	1137	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	1138	if os.path.exists(app_picture):
	1139	copy2(app_picture,image_name)
	1140	else:
	1141	logger.info('passport of %s/%s not found: %s' % (student_id,
	1142	brain.reg_no,
	1143	app_picture))
[2656]	1144
[3349]	1145	self.do_timing()
	1146	self.print_timing()
	1147	if with_timing:
	1148	self.timer_step = 0
	1149	self._v_timer_count += 1
	1150	self._v_total += self.total_time
[2514]	1151	return student_id
	1152	###)
	1153
[2556]	1154	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
[1194]	1155	def makeStudentLevel(self,student_id):
	1156	"create the StudyLevel for a returning Student"
	1157	#import pdb;pdb.set_trace()
[1571]	1158	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
[1194]	1159	students_folder = self.portal_url.getPortalObject().campus.students
	1160	res = self.students_catalog(id=student_id)
	1161	if res:
	1162	st = res[0]
	1163	course = st.course
	1164	matric_no = st.matric_no
	1165	level = st.level
	1166	res = self.results_import(matric_no = matric_no)
	1167	if res:
	1168	results = res
[1571]	1169	logger.info('%s creating Level %s' % (student_id,level))
[1194]	1170	#
	1171	# Level
	1172	#
	1173	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1174	studycourse = getattr(student_obj,"study_course",None)
	1175	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
	1176	l = getattr(studycourse,level,None)
	1177	if l is None:
	1178	studycourse.invokeFactory('StudentStudyLevel', level)
	1179	l = getattr(studycourse, level)
	1180	self.portal_workflow.doActionFor(l,'open',dest_container=l)
	1181	l.getContent().edit(mapping={'Title': "Level %s" % level})
	1182	###)
[3984]	1183
	1184
	1185	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
	1186	def exportAllStudyLevels(self,student_id):
	1187	"export the StudyLevels for a student"
	1188	#import pdb;pdb.set_trace()
	1189	logger = logging.getLogger('WAeUPTool.exportAllStudyLevels')
	1190	students_folder = self.portal_url.getPortalObject().campus.students
	1191	res = self.students_catalog(id=student_id)
	1192	if res:
	1193	st = res[0]
	1194	course = st.course
	1195	matric_no = st.matric_no
	1196	level = st.level
	1197	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1198	studycourse = getattr(student_obj,"study_course",None)
	1199	levels = studycourse.objectIds()
	1200
	1201	stool = getToolByName(self, 'portal_schemas')
	1202	schema = stool._getOb('student_study_level')
	1203	fields = ['student_id','level']
	1204	fields.extend(schema.keys())
	1205	format = '"%(' + ')s","%('.join(fields) + ')s"'
	1206	export_file = "%s/export/study_levels_removed.csv" % (i_home)
	1207
[3989]	1208	if not os.path.exists(export_file):
	1209	file_handler = open(export_file,"a")
	1210	headline = ','.join(fields)
	1211	file_handler.write(headline +'\n')
	1212	else:
	1213	file_handler = open(export_file,"a")
	1214
[3984]	1215	for level in levels:
	1216	level_object = getattr(studycourse,level)
	1217	level_content = level_object.getContent()
	1218	d = {'student_id':student_id,'level':level}
	1219	for field in schema.keys():
	1220	d[field] = getattr(level_content,field,'')
	1221	line = format % d
	1222	file_handler.write(line +'\n')
	1223
	1224	###)
[1194]	1225
[2000]	1226	security.declarePublic('getHallInfo') ###(
	1227	def getHallInfo(self,bed):
	1228	"""return Hall Info"""
[828]	1229	info = {}
[3406]	1230	bedsplit = bed.split('_')
	1231	if len(bedsplit) == 4:
	1232	hall,block,room,letter = bed.split('_')
	1233	else:
	1234	info['maintenance_code'] = 'None'
	1235	return info
[1846]	1236	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
[828]	1237	if res and len(res) == 1:
	1238	hall_brain = res[0]
	1239	hall_doc = hall_brain.getObject().getContent()
	1240	else:
	1241	return info
	1242	info['hall_title'] = hall_brain.Title
	1243	info['maintenance_code'] = hall_doc.maintenance_code
[1846]	1244	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
[828]	1245	batch_doc = None
	1246	for brain in res:
	1247	if brain.id.startswith(info['maintenance_code']):
	1248	batch_doc = brain.getObject().getContent()
	1249	break
	1250	if batch_doc is None:
[2913]	1251	info['maintenance_fee'] = ''
[828]	1252	else:
	1253	info['maintenance_fee'] = batch_doc.cost
	1254	return info
[1151]	1255	###)
[828]	1256
[2396]	1257	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
	1258	def removePictureFolder(self,student_id):
	1259	"""remove picture_folder by renaming it"""
[3807]	1260	picture_path = getImagesDir(student_id)
	1261	dest_path = os.path.join("%s" % images_base,'removed',student_id)
	1262	dest_path = dest_path + "_removed"
	1263	if os.path.exists(dest_path) or not os.path.exists(picture_path):
[2396]	1264	return False
[3807]	1265	os.rename(picture_path,dest_path)
[2396]	1266	return True
	1267	###)
	1268
	1269	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
	1270	def restorePictureFolder(self,student_id):
	1271	"""restore picture_folder by renaming it"""
[3807]	1272	picture_path = getImagesDir(student_id)
	1273	orig_path = os.path.join("%s" % images_base,'removed',student_id)
	1274	orig_path = orig_path + "_removed"
	1275	if os.path.exists(picture_path) or not os.path.exists(orig_path):
	1276	return False
[2396]	1277	os.rename(picture_path + "_removed",picture_path)
	1278	return True
	1279	###)
	1280
[2363]	1281	security.declarePublic('picturesExist') ###(
[2420]	1282	def picturesExist(self, ids,student_id=None):
[2363]	1283	"""check if pictures exist in the filesystem"""
[2420]	1284	if student_id is None:
	1285	student_id = self.getStudentId()
	1286	if student_id is None:
	1287	return False
[2675]	1288	picture_path = getImagesDir(student_id)
	1289	#picture_path = os.path.join(images_base,student_id)
[2363]	1290	if not os.path.exists(picture_path):
	1291	return False
	1292	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1293	return set(ids).issubset(set(pictures))
	1294	###)
[3805]	1295
[3812]	1296	security.declarePublic('picturePathExists') ###(
	1297	def picturePathExists(self, student_id=None):
	1298	"""check if picture path exists in the filesystem"""
	1299	if student_id is None:
	1300	student_id = self.getStudentId()
	1301	if student_id is None:
	1302	return False
	1303	picture_path = getImagesDir(student_id)
	1304	if os.path.exists(picture_path):
	1305	return True
	1306	return False
	1307	###)
[3805]	1308
[3812]	1309
[3807]	1310	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
	1311	def removeUnusedImageFolders(self):
[3806]	1312	"""check if an unused image folders exists in the filesystem"""
[3807]	1313	mtool = self.portal_membership
	1314	member = mtool.getAuthenticatedMember()
	1315	member_id = str(member)
	1316	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
[3806]	1317	abc = os.listdir(images_base)
[3805]	1318	ifolders = []
[3806]	1319	for i in abc:
[3805]	1320	picture_path = os.path.join(images_base,i)
	1321	ifolders.extend(os.listdir(picture_path))
	1322	unused_ids = []
	1323	for id in ifolders:
	1324	res = self.students_catalog(id=id)
	1325	if not res:
	1326	unused_ids.append(id)
[3807]	1327	#import pdb;pdb.set_trace()
	1328	if not id.endswith('removed'):
	1329	removed = self.waeup_tool.removePictureFolder(id)
	1330	if removed:
	1331	logger.info('%s: image folder %s successfully removed' % (member_id,id))
	1332	else:
	1333	logger.info('%s: image folder %s could not be removed' % (member_id,id))
	1334	return
[3805]	1335
	1336	###)
	1337
[2364]	1338	security.declarePublic('picturesList') ###(
	1339	def picturesList(self):
	1340	"""check if pictures exist in the filesystem"""
	1341	path = 'images'
	1342	student_id = self.getStudentId()
[2677]	1343	#picture_path = os.path.join(i_home,path,student_id)
	1344	picture_path = getImagesDir(student_id)
[2364]	1345	if not os.path.exists(picture_path):
	1346	return []
	1347	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1348	###)
	1349
[2114]	1350	security.declarePublic('showFsPicture') ###(
	1351	def showFsPicture(self,path):
	1352	"""return a picture from the filesystem"""
[2675]	1353	#picture_path = os.path.join(i_home,path)
	1354	picture_path = os.path.join(images_base,path)
[2335]	1355	response = self.REQUEST.RESPONSE
[2342]	1356	#import pdb;pdb.set_trace()
[2354]	1357	registry = getToolByName(self, 'mimetypes_registry')
	1358	mimetype = str(registry.lookupExtension(path.lower()) or
	1359	registry.lookupExtension('file.bin'))
[2114]	1360	if os.path.exists(picture_path):
[2354]	1361	response.setHeader('Content-type',mimetype)
[2114]	1362	return open(picture_path).read()
[2335]	1363	picture_path = os.path.join(i_home,'import',path)
	1364	if os.path.exists(picture_path):
	1365	return open(picture_path).read()
[2114]	1366	###)
	1367
[1151]	1368	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
	1369	def deleteAllCourses(self,department="All"):
	1370	''' delete the courses'''
	1371	pm = self.portal_membership
	1372	member = pm.getAuthenticatedMember()
[1160]	1373
[1151]	1374	if str(member) not in ("henrik","joachim"):
	1375	return "not possible"
	1376	if department == "All":
	1377	res = self.portal_catalog({'meta_type': 'Department'})
	1378	if len(res) < 1:
	1379	return "No Departments found"
[1160]	1380
[1151]	1381	deleted = []
	1382	for dep in res:
	1383	cf = dep.getObject().courses
	1384	if cf:
	1385	cf.manage_delObjects(ids=cf.objectIds())
	1386	deleted.append("deleted Courses in %s" % dep.getId)
	1387	return "\r".join(deleted)
[1160]	1388	###)
[1151]	1389
[1572]	1390	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
	1391	def getLogfileLines(self,filename="event.log",numlines=20):
	1392	"""Get last NUMLINES lines of logfile FILENAME.
[1160]	1393
[1572]	1394	Return last lines' of a file in the instances logfile directory as
	1395	a list. The number of returned lines equals `numlines' or less. If
	1396	less than `numlines' lines are available, the whole file ist
	1397	returned. If the file can not be opened or some other error
	1398	occurs, empty list is returend.
	1399	"""
	1400	result = []
	1401	lines_hit = 0
	1402
	1403	# We only handle files in instances' log directory...
	1404	logpath = os.path.join(i_home, "log")
	1405	filename = str(os.path.abspath( os.path.join( logpath, filename )))
	1406	if not filename.startswith( logpath ):
	1407	# Attempt to access file outside log-dir...
	1408	return []
	1409
	1410	try:
	1411	fd = file( filename, "rb" )
	1412	except IOError:
	1413	return []
	1414	if not fd:
	1415	return []
	1416
	1417	if os.linesep == None:
	1418	linesep = '\n'
	1419	else:
	1420	linesep = os.linesep
	1421
	1422	# Try to find 'numlines' times a lineseparator, searching from end
	1423	# and moving to the beginning of file...
	1424	fd.seek( 0, 2) # Move to end of file...
	1425	while lines_hit < numlines:
	1426	if fd.read(1) == linesep[-1]: # This moves filedescriptor
	1427	# one step forward...
	1428	lines_hit += 1
	1429	try:
	1430	fd.seek( -2, 1) # Go two bytes back from current pos...
	1431	except IOError:
	1432	# We cannot go back two bytes. Maybe the file is too small...
	1433	break
	1434	fd.seek(2,1)
	1435
	1436	# Read all lines from current position...
	1437	result = fd.readlines()
	1438	# Remove line endings...
	1439	result = [x.strip() for x in result]
	1440	fd.close()
	1441	return result
	1442	###)
	1443
[1700]	1444	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
[1665]	1445	def getCallbacksFromLog(self,filename):
	1446	"""fix Online Payment Transactions from Z2.log entries"""
	1447	import transaction
	1448	import random
	1449	from cgi import parse_qs
	1450	from urlparse import urlparse
	1451	#from pdb import set_trace
	1452	wftool = self.portal_workflow
	1453	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1454	students_folder = self.portal_url.getPortalObject().campus.students
	1455	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
	1456	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
	1457	s += r'(?P<intersw>.)" "(?P<agent>.)"'
	1458	data = re.compile(s)
	1459	start = True
	1460	tr_count = 1
	1461	total = 0
	1462	#name = 'pume_results'
	1463	#name = 'epaymentsuccessful_z2log2'
	1464	name = filename
	1465	no_import = []
	1466	imported = []
	1467	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
	1468	try:
	1469	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
	1470	except:
	1471	logger.error('Error reading %s' % name)
	1472	return
	1473	tas = []
	1474	for line in transactions:
	1475	dict = {}
	1476	items = data.search(line)
	1477	dict['idict'] = idict = items.groupdict()
	1478	#print idict
	1479	#from pdb import set_trace;set_trace()
	1480	urlparsed = urlparse(idict['get'][4:])
	1481	#print urlparsed
	1482	path = urlparsed[2].split('/')
	1483	dict['student_id'] = student_id = path[8]
	1484	dict['payment_id'] = payment_id = path[10]
	1485	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
	1486	tas.append(dict)
	1487	tr_count += 1
	1488	return tas
	1489	###)
	1490
[1620]	1491	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
	1492	def importOnlinePaymentTransactions(self):
	1493	"""load Online Payment Transactions from CSV values"""
	1494	import transaction
	1495	import random
	1496	#from pdb import set_trace
	1497	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
[1625]	1498	opt = self.online_payments_import
[1620]	1499	students_folder = self.portal_url.getPortalObject().campus.students
	1500	start = True
	1501	tr_count = 1
	1502	total = 0
	1503	#name = 'pume_results'
	1504	name = 'OnlineTransactions'
	1505	no_import = []
	1506	imported = []
	1507	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
	1508	try:
	1509	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1510	except:
	1511	logger.error('Error reading %s.csv' % name)
	1512	return
	1513	for pay_transaction in transactions:
	1514	if start:
	1515	start = False
	1516	logger.info('Start loading from %s.csv' % name)
	1517	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
	1518	no_import.append('%s,"Error"' % s)
	1519	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
	1520	format_error = format + ',"%(Error)s"'
	1521	data = {}
[1644]	1522
[1781]	1523	# format of the first file sent by Tayo
[1643]	1524	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
	1525	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1526	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1527	#data['response_code'] = response_code = pay_transaction['Resp Code']
	1528	#data['amount'] = amount = pay_transaction['Amount']
[1644]	1529
[1781]	1530	# format of the second file sent by Tayo
[1798]	1531	#data['datetime'] = date = 0
	1532	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1533	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1534	#data['response_code'] = response_code = '00'
	1535	#data['amount'] = amount = pay_transaction['Amount']
[1813]	1536
[1798]	1537	# format of the third file sent by Kehinde
[1643]	1538	data['datetime'] = date = 0
[1798]	1539	data['student_id'] = student_id = pay_transaction['customer_id']
	1540	data['order_id'] = order_id = pay_transaction['merchant_reference']
[1644]	1541	data['response_code'] = response_code = '00'
[1813]	1542	data['amount'] = amount = pay_transaction['Amount']
[1644]	1543
[1620]	1544	dup = False
	1545	if response_code == "12":
	1546	continue
	1547	try:
	1548	opt.addRecord(**data)
	1549	except ValueError:
	1550	dup = True
	1551	#from pdb import set_trace;set_trace()
	1552	if dup:
	1553	if response_code == "00":
[1798]	1554	try:
	1555	opt.modifyRecord(**data)
	1556	except:
	1557	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
	1558	continue
[1620]	1559	else:
[1674]	1560	pay_transaction['Error'] = "Duplicate order_id"
[1620]	1561	no_import.append( format_error % pay_transaction)
[1674]	1562	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
[1620]	1563	continue
	1564	tr_count += 1
	1565	if tr_count > 1000:
	1566	if len(no_import) > 0:
	1567	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1568	'\n'.join(no_import) + '\n')
	1569	no_import = []
[1645]	1570	em = '%d transactions committed\n' % (tr_count)
[1620]	1571	transaction.commit()
	1572	regs = []
	1573	logger.info(em)
	1574	total += tr_count
	1575	tr_count = 0
	1576	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1577	'\n'.join(no_import))
	1578	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1579	###)
[2237]	1580
[3172]	1581	security.declareProtected(ModifyPortalContent,"importData")###(
	1582	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
	1583	"""load data from CSV values"""
	1584	import transaction
	1585	import random
	1586	students_folder = self.portal_url.getPortalObject().campus.students
[3292]	1587	uploads_folder = self.portal_url.getPortalObject().campus.uploads
[3172]	1588	pending_only = False
[3177]	1589	pend_str = '--'
[3172]	1590	elapse = time.time()
	1591	#
	1592	# preparations
	1593	#
[3177]	1594	if filename == pend_str:
[3172]	1595	pending_only = True
	1596	importer_name = ''.join([part.capitalize() for part in name.split('_')])
[3191]	1597	importer = eval("%sImport" % importer_name)(self)
[3172]	1598	logger = importer.logger
	1599	if importer.init_errors:
[3191]	1600	logger.info(importer.init_errors)
	1601	return importer.init_errors
[3172]	1602	member = importer.member
[3320]	1603	#current = importer.current
[3172]	1604	import_date = importer.import_date
	1605	#
	1606	# not_imported
	1607	#
	1608	info = importer.info
	1609	data_keys = importer.data_keys
[3181]	1610	csv_keys = importer.csv_keys
[3172]	1611	#csv_keys.extend(info.keys())
	1612	headline_mapping = dict((k,k) for k in csv_keys)
	1613	#
	1614	# pending
	1615	#
	1616	pending_path = importer.pending_path
	1617	pending_tmp = importer.pending_tmp
	1618	pending_backup = importer.pending_backup
	1619	pending_fn = importer.pending_fn
	1620	imported_path = importer.imported_path
	1621	imported_fn = importer.imported_fn
[3180]	1622	commit_after = importer.commit_after
[3172]	1623	pending = []
	1624	pending_digests = []
	1625	#total_added_to_pending = 0
	1626	if not pending_only:
	1627	pending,pending_digests = importer.makeIdLists()
	1628	pending_at_start = len(pending)
	1629	datafile = open(pending_tmp,"w")
	1630	pending_csv_writer = csv.DictWriter(datafile,
	1631	csv_keys,
	1632	extrasaction='ignore')
	1633	pending_csv_writer.writerow(headline_mapping)
	1634	datafile.close()
	1635	#
	1636	# imported
	1637	#
	1638	if not os.path.exists(imported_path):
	1639	datafile = open(imported_path,"w")
	1640	imported_csv_writer = csv.DictWriter(datafile,
	1641	csv_keys,
	1642	extrasaction='ignore')
	1643	imported_csv_writer.writerow(headline_mapping)
	1644	datafile.close()
	1645	start = True
	1646	tr_count = 0
	1647	total = 0
	1648	total_added_to_imported = 0
	1649	total_pending = 0
[3292]	1650	import_source_done = ""
[3172]	1651	if pending_only:
	1652	import_source_path = pending_path
[2193]	1653	else:
[3172]	1654	import_source_path = "%s/import/%s.csv" % (i_home,filename)
[3292]	1655	import_source_done = "%s/import/%s.done" % (i_home,filename)
[3172]	1656	if not os.path.exists(import_source_path):
[3195]	1657	fn = os.path.split(import_source_path)[1]
[3268]	1658	em = 'no import file %(fn)s' % vars()
[3172]	1659	return em
	1660	import_source_fn = os.path.split(import_source_path)[1]
[3267]	1661	if not pending_only:
	1662	info['imported_from'] = import_source_fn
[3277]	1663	headline = csv.reader(open(import_source_path,"rb")).next()
	1664	if "import_mode" not in headline:
[3191]	1665	msg = 'import_mode must be in heading'
	1666	return msg
[3277]	1667	invalid_keys = importer.checkHeadline(headline)
	1668	if invalid_keys:
	1669	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
[3201]	1670
[3300]	1671	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
	1672	or k.strip() in info.keys())]
[3277]	1673	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
	1674	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
	1675	# if diff2schema and diff2schema != set(['id',]):
	1676	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
	1677	# return msg
[3172]	1678	#
	1679	# start importing
	1680	#
[2193]	1681	try:
[3172]	1682	reader = csv.DictReader(open(import_source_path,"rb"))
	1683	except:
	1684	msg = 'Error reading %s.csv' % filename
	1685	logger.error(msg)
	1686	return msg
	1687	items = [item for item in reader]
	1688	total_to_import = len(items)
[3181]	1689	tti_float = float(total_to_import)
[3172]	1690	if pending_only:
	1691	pending_at_start = total_to_import
	1692	count = 0
	1693	imported = []
[3391]	1694	old_commit_count = 0
[3204]	1695	error_count = imported_count = 0
[3172]	1696	already_in = 0
	1697	for record in items:
	1698	item = {}
[3198]	1699	empty_value_keys = []
[3172]	1700	for k,v in record.items():
[3346]	1701	if k is None:
	1702	continue
[3172]	1703	if v:
[3198]	1704	if v == EMPTY:
	1705	empty_value_keys += k,
	1706	v = ''
[3172]	1707	item[k.strip()] = v.strip()
	1708	count += 1
	1709	if start:
	1710	start = False
	1711	adapters = [MappingStorageAdapter(importer.schema, item)]
[3191]	1712	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
[3172]	1713	dm = DataModel(item, adapters,context=self)
	1714	ds = DataStructure(data=item,datamodel=dm)
	1715	error_string = ""
	1716	total += 1
[3271]	1717	import_mode = item.get('import_mode','')
[3191]	1718	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
	1719	if import_method is None:
[3271]	1720	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
[3251]	1721	elif (import_mode in importer.required_modes and
[3248]	1722	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
	1723	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
	1724	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
[3191]	1725	else:
	1726	for k in import_keys:
	1727	if k not in item.keys() or k not in importer.validators.keys():
	1728	continue
	1729	if not importer.validators[k](ds,mode=import_mode):
[3201]	1730	error_string += ' ++ '
	1731	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
[3172]	1732	ds.getErrorMapping(k)))
	1733	if error_string:
	1734	error = error_string
	1735	id = ''
	1736	mapping = item
[2699]	1737	else:
[3172]	1738	temp_item = item.copy()
	1739	temp_item.update(dm)
[3185]	1740	results = import_method(temp_item)
[3172]	1741	id = results[0]
	1742	error = results[1]
	1743	mapping = results[2]
[3267]	1744	#
	1745	# restore some values
	1746	#
[3198]	1747	for k in empty_value_keys:
	1748	mapping[k] = EMPTY
[3186]	1749	if mapping.has_key('sex'):
	1750	#import pdb;pdb.set_trace()
[3267]	1751	if mapping['sex'] == True:
[3186]	1752	mapping['sex'] = 'F'
[3267]	1753	elif mapping['sex'] == False:
[3186]	1754	mapping['sex'] = 'M'
[3267]	1755	if pending_only:
	1756	info['imported_from'] = item['imported_from']
[3181]	1757	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
[3172]	1758	info['error'] = error
[3326]	1759	info['import_record_no'] = count + 1
[3172]	1760	mapping.update(info)
[3178]	1761	log_list = []
[3172]	1762	if error:
[3204]	1763	error_count += 1
[3172]	1764	digest = makeDigest(mapping,data_keys)
	1765	if digest not in pending_digests:
	1766	pending_digests += digest,
	1767	pending.append(mapping)
	1768	if not pending_only:
[3251]	1769	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
[3172]	1770	else:
	1771	already_in += 1
	1772	pass
	1773	else:
	1774	imported_count += 1
	1775	imported += mapping,
[3251]	1776	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
[3178]	1777	if log_list:
	1778	time_till_now = time.time() - elapse
[3204]	1779	percent_finished = (error_count + imported_count)/tti_float*100
[3181]	1780	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
[3178]	1781	logger.info(' '.join(log_list))
[3172]	1782	finished = count > total_to_import - 1
[3277]	1783	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
	1784	if must_commit:
	1785	old_commit_count = imported_count
[3181]	1786
[3172]	1787	if must_commit or finished:
	1788	if len(imported):
	1789	transaction.commit()
	1790	datafile = open(imported_path,"a")
	1791	writer = csv.DictWriter(datafile,
	1792	csv_keys,
	1793	extrasaction='ignore')
	1794	writer.writerows(imported)
	1795	datafile.close()
	1796	total_added_to_imported += len(imported)
	1797	imported = []
	1798	if len(pending) > 0:
	1799	datafile = open(pending_tmp,"a")
	1800	writer = csv.DictWriter(datafile,
	1801	csv_keys,
	1802	extrasaction='ignore')
	1803	writer.writerows(pending)
	1804	datafile.close()
	1805	total_pending += len(pending)
	1806	#total_added_to_pending += len(pending)
	1807	pending = []
	1808	if not finished:
[3335]	1809	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
[3172]	1810	logger.info(msg)
	1811	elapse = time.time() - elapse
[3271]	1812	if os.path.exists(pending_path):
	1813	copy2(pending_path,pending_backup)
[3172]	1814	copy2(pending_tmp,pending_path)
[3191]	1815	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
[3181]	1816	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
[3172]	1817	if pending_only:
	1818	removed_pending = pending_at_start - total_pending
[3184]	1819	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
[2699]	1820	else:
[3181]	1821	added_pending = total_pending - pending_at_start
	1822	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
	1823	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
[3172]	1824	logger.info(msg)
[3292]	1825	if import_source_done:
	1826	copy(import_source_path,import_source_done)
	1827	os.remove(import_source_path)
	1828	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
	1829	if upload is not None:
	1830	upload_doc = upload.getContent()
	1831	mapping = {}
[3308]	1832	#mapping['import_date'] = DateTime.DateTime()
	1833	mapping['import_date'] = import_date
[3327]	1834	mapping['imported_by'] = importer.imported_by
[3292]	1835	mapping['import_message'] = msg
	1836	upload_doc.edit(mapping = mapping)
[3178]	1837	os.remove(pending_tmp)
[3172]	1838	return msg
[2514]	1839	###)
	1840
[2292]	1841	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
	1842	def moveImagesToFS(self,student_id="O738726"):
	1843	"move the images to the filesystem"
[2675]	1844	images_dir = getImagesDir(student_id)
	1845	#images_dir = os.path.join("%s" % images_base,student_id)
[2292]	1846	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1847	stool = getToolByName(self, 'portal_schemas')
	1848	schemas = ['student_application',
	1849	'student_clearance',
	1850	]
	1851	created = False
	1852	for schema_id in schemas:
	1853	schema = stool._getOb(schema_id)
	1854	object = getattr(student_folder,schema_id[len('student_'):],None)
	1855	if object is None:
	1856	continue
	1857	doc = object.getContent()
	1858	for key in schema.keys():
	1859	if schema[key].meta_type != "CPS Image Field":
	1860	continue
	1861	#import pdb;pdb.set_trace()
[2351]	1862	image = getattr(doc,key,None)
	1863	if not image or not hasattr(image,"data"):
[2292]	1864	continue
	1865	if not created:
	1866	if not os.path.exists(images_dir):
	1867	os.mkdir(images_dir)
	1868	created = True
	1869	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
	1870	open(filename,"wb").write(str(image.data))
[2500]	1871	###)
[2292]	1872
[2300]	1873	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
	1874	def movePassportToFS(self,student_id="O738726"):
	1875	"move the passports to the filesystem"
	1876	images_dir = os.path.join("%s" % i_home,'passports')
	1877	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1878	stool = getToolByName(self, 'portal_schemas')
	1879	schemas = ['student_application',
	1880	#'student_clearance',
	1881	]
	1882	created = False
	1883	for schema_id in schemas:
	1884	schema = stool._getOb(schema_id)
	1885	object = getattr(student_folder,schema_id[len('student_'):],None)
	1886	if object is None:
	1887	continue
	1888	doc = object.getContent()
	1889	for key in schema.keys():
	1890	if schema[key].meta_type != "CPS Image Field":
	1891	continue
	1892	#import pdb;pdb.set_trace()
	1893	image = getattr(doc,key)
	1894	if not hasattr(image,"data"):
	1895	continue
	1896	if not created:
	1897	if not os.path.exists(images_dir):
	1898	os.mkdir(images_dir)
	1899	created = True
	1900	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
	1901	open(filename,"wb").write(str(image.data))
[2500]	1902	###)
[3824]	1903
	1904
	1905	security.declarePublic('getConfigParams')
	1906	def getConfigParams(self,conf_id="configuration"):
	1907	conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
	1908	conf_obj = conf.getContent()
	1909	stool = getToolByName(self, 'portal_schemas')
	1910	schema = stool._getOb('configuration')
	1911	d = {}
	1912	for key in schema.keys():
	1913	d[key] = getattr(conf_obj,key,None)
	1914	return d
[2300]	1915
[3824]	1916
[828]	1917	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: