Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3141

Last change on this file since 3141 was 3141, checked in by Henrik Bettermann, 17 years ago
logging improved
Property svn:keywords set to `Id`
File size: 104.3 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3141 2008-02-11 06:25:00Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	import DateTime,time
46	import logging
47	import transaction
48	import csv,re,os,sys
49	from shutil import copy2
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	p_home = package_home(globals())
53	i_home = INSTANCE_HOME
54	images_base = os.path.join(i_home,"images")
55
56	def getImagesDir(student_id):
57	return os.path.join("%s" % images_base,student_id[0],student_id)
58
59	def getObject(object,name):
60	if object.hasObject(name):
61	return getattr(object,name)
62	return None
63
64
65	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66	"""WAeUP tool"""
67
68	id = 'waeup_tool'
69	meta_type = 'WAeUP Tool'
70	_actions = ()
71	security = ClassSecurityInfo()
72	security.declareObjectProtected(View)
73	manage_options = ( ActionProviderBase.manage_options
74	+ SimpleItem.manage_options
75	)
76
77	security.declareProtected(View,'re_split') ###(
78	def re_split(self,split_string,string):
79	return re.split(split_string,string)
80	###)
81
82	security.declareProtected(View,'difference') ###(
83	def difference(self,l1,l2):
84	return set(l1).difference(set(l2))
85	###)
86
87	def rwrite(self,s): ###(
88	response = self.REQUEST.RESPONSE
89	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
90	response.write("%s<br />\r\n" % s)
91	###)
92
93	def addtodict(self,d,key,item): ###(
94	d[key].append(item)
95	return d[key]
96	###)
97
98	def sleep(self,secs): ###(
99	"sleep"
100	import time
101	time.sleep(secs)
102	return
103
104	###)
105
106	security.declareProtected(View,'updateRoleMappingsFor') ###(
107	def updateRoleMappingsFor(self,wf_definition,ob):
108	"do so for public"
109	wf_def = getattr(self.portal_workflow,wf_definition)
110	wf_def.updateRoleMappingsFor(ob)
111	###)
112
113	security.declareProtected(View,'getStatesLgas') ###(
114	def getStatesLgas(self):
115	"""return lga info"""
116	voc = getattr(self.portal_vocabularies,'local_gov_areas')
117	states = []
118	lgas = []
119	d = {}
120	wd = {}
121	for k,v in voc.items():
122	parts = v.split(' / ')
123	if len(parts) == 1:
124	state = parts[0].lower()
125	lga = ""
126	elif len(parts) == 2:
127	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
128	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
129	else:
130	continue
131	if state not in states:
132	states.append(state)
133	if lga not in lgas:
134	lgas.append(lga)
135	words = re.split('[^a-zA-Z0-9/]',k)
136	words.sort()
137	wd[k] = words
138	d[k] = v
139	mapping = {}
140	mapping['word_dict'] = wd
141	mapping['lga_dict'] = d
142	mapping['states'] = states
143	mapping['lgas'] = lgas
144	return mapping
145	###)
146
147	security.declareProtected(View,'findLga') ###(
148	def findLga(self,words,words_dict):
149	words = re.split('[^a-zA-Z0-9/]',words)
150	lga_words = []
151	for word in words:
152	if word:
153	lga_words += word.strip().lower(),
154	lga_words.sort()
155	state_lga = ''
156	while not state_lga:
157	for k,l in words_dict.items():
158	if lga_words == l:
159	state_lga = k
160	break
161	break
162	return state_lga
163	###)
164	security.declareProtected(View,'getAccessInfo') ###(
165	def getAccessInfo(self,context):
166	"return a dict with access_info"
167	logger = logging.getLogger('WAeUPTool.getAccessInfo')
168	mtool = self.portal_membership
169	member = mtool.getAuthenticatedMember()
170	member_id = str(member)
171	info = {}
172	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
173	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
174	is_staff = info['is_staff'] = not is_anonymous and not is_student
175	roles = member.getRolesInContext(context)
176	is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
177	"SectionManager" in roles or
178	"Manager" in roles)
179	is_allowed = info['is_allowed'] = not is_anonymous
180	requested_id = context.getStudentId()
181	student_id = None
182	if is_allowed:
183	if not is_student and requested_id:
184	student_id = requested_id
185	elif is_anonymous or member_id != requested_id:
186	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
187	else:
188	student_id = member_id
189	info['student_id'] = student_id
190	return info
191	###)
192
193	security.declareProtected(ModifyPortalContent,'openLog') ###(
194	def openLog(self,name):
195	"""open a log file"""
196	version = 1
197	path = "%s/log/%s_%d.log" % (i_home,name,version)
198	while os.path.exists(path):
199	version += 1
200	path = "%s/log/%s_%d.log" % (i_home,name,version)
201	log = open(path,"w")
202	return log
203	###)
204
205	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
206	def bypassQueueCatalog(self,enable=True):
207	"""bypass the QueueCatalog by setting all indexes to process imediate,
208	if enable is True (default) the old settings are restored
209	"""
210
211	###)
212
213	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
214	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
215	"""measure Object access Time"""
216	import random
217	if hasattr(self,'portal_catalog_real'):
218	aq_portal = self.portal_catalog_real.evalAdvancedQuery
219	else:
220	aq_portal = self.portal_catalog.evalAdvancedQuery
221	nr_pts = int(nr_pts)
222	probe = int(probe)
223	intervall = probe/10
224	objects = ("application","clearance","personal")
225	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
226	#i = random.randrange(num_objects)
227	count = 0
228	found = 0
229	not_found = 0
230	t_found = 0
231	t_not_found = 0
232	time_found = time_not_found = 0.0
233	t_time_found = t_time_not_found = 0.0
234	accessed = []
235	t_min = 1000
236	t_max = 0
237	#import pdb;pdb.set_trace()
238	students = self.portal_catalog(portal_type="Student")
239	num_students = len(students)
240	if method == "d":
241	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
242	res = aq_portal(query)
243	brains = {}
244	for r in res:
245	sid = r.relative_path.split('/')[-2]
246	if brains.has_key(sid):
247	brains[sid][r.portal_type] = r
248	else:
249	brains[sid] = {r.portal_type : r}
250	brains_list = brains.keys()
251	num_objects = len(brains_list)
252	else:
253	num_objects = num_students
254	print "="*40
255	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
256	probe,
257	nr_pts,
258	num_objects)
259	print "nr found/not time found/not min/max"
260	elapse = time.time()
261	i_elapse = time.time()
262	c_elapse = time.clock()
263	for c in range(1,probe + 1):
264	i = random.randrange(num_objects)
265	if method in ('a','b','c'):
266	student_brain = students[i]
267	elif method == "d":
268	#import pdb;pdb.set_trace()
269	student_brain = brains[brains_list[i]]
270	if method == "c":
271	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
272	res = aq_portal(query)
273	this_portal_types = [r.portal_type for r in res]
274	for i in range(nr_pts):
275	oid = objects[i]
276	if method == "a":
277	try:
278	student_path = student_brain.getPath()
279	path = "%s/%s" % (student_path,oid)
280	doc = self.unrestrictedTraverse(path).getContent()
281	found += 1
282	i_time = time.time() - i_elapse
283	time_found += i_time
284	except:
285	not_found += 1
286	i_time = time.time() - i_elapse
287	time_not_found += i_time
288	pass
289	elif method == "b":
290	try:
291	student_object = student_brain.getObject()
292	doc = getattr(student_object,oid).getContent()
293	found += 1
294	i_time = time.time() - i_elapse
295	time_found += i_time
296	except:
297	i_time = time.time() - i_elapse
298	time_not_found += i_time
299	not_found += 1
300	pass
301	elif method == "c":
302	if portal_types[i] in this_portal_types:
303	found += 1
304	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
305	i_time = time.time() - i_elapse
306	time_found += i_time
307	else:
308	not_found += 1
309	i_time = time.time() - i_elapse
310	time_not_found += i_time
311	elif method == "d":
312	if student_brain.has_key(portal_types[i]):
313	found += 1
314	doc = student_brain[portal_types[i]].getObject().getContent()
315	i_time = time.time() - i_elapse
316	time_found += i_time
317	else:
318	not_found += 1
319	i_time = time.time() - i_elapse
320	time_not_found += i_time
321	i_elapse = time.time()
322	if c and (c % intervall == 0):
323	#i_time = time.time() - i_elapse
324	t_per = 0.0
325	if found:
326	t_per = time_found/found
327	if t_per > t_max:
328	t_max = t_per
329	if t_per > 0.0 and t_per < t_min:
330	t_min = t_per
331	itf = 0.0
332	if found:
333	itf = time_found/found
334	itnf = 0.0
335	if not_found :
336	itnf = time_not_found / not_found
337	interval_time = time_found + time_not_found
338	s = "%(c)d: %(found)d/%(not_found)d " % vars()
339	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
340	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
341	print s
342	t_found += found
343	t_not_found += not_found
344	t_time_found += time_found
345	t_time_not_found += time_not_found
346	time_found = time_not_found = 0.0
347	found = not_found = 0
348	# t_found += found
349	# t_not_found += not_found
350	elapse = time.time() - elapse
351	itf = 0.0
352	if t_found:
353	itf = t_time_found/t_found
354	itnf = 0.0
355	if t_not_found:
356	itnf = t_time_not_found / t_not_found
357	#c_elapse = time.clock() - c_elapse
358	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
359	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
360	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
361	print "-"*40
362	print s
363	rel_found = float(t_found)/probe
364	rel_not_found = float(t_not_found)/probe
365	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
366	print estimated_total_time
367	###)
368
369	security.declareProtected(ModifyPortalContent,'writeLog') ###(
370	def writeLog(self,logfile,s):
371	"""write to the log file"""
372	logfile.write(s)
373
374	###)
375
376	def generateStudentId(self,letter): ###(
377	import random
378	r = random
379	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
380	if letter == '?':
381	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
382	sid = "%c%d" % (letter,r.randint(99999,1000000))
383	students = self.portal_url.getPortalObject().campus.students
384	## while hasattr(students, sid):
385	## sid = "%c%d" % (letter,r.randint(99999,1000000))
386	while self.students_catalog(id = sid):
387	sid = "%c%d" % (letter,r.randint(99999,1000000))
388	return sid
389	###)
390
391	def generatePassword(self,s=None): ###(
392	import random
393	r = random
394	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
395	if s is None:
396	s = 'abcdefghklmnpqrstuvwxy23456789'
397	pw = ''
398	while len(pw) < 6:
399	pw += r.choice(s)
400	return pw
401	###)
402
403	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
404	def dumpSchoolfeePayments(self):
405	"dump paid schoolfees"
406	mtool = self.portal_membership
407	member = mtool.getAuthenticatedMember()
408	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
409	aq_student = self.students_catalog.evalAdvancedQuery
410	query = In('review_state',('schoolfee_paid',
411	'courses_registered',
412	'courses_validated',
413	))
414	res = aq_student(query)
415	#import pdb;pdb.set_trace()
416	l = []
417	logger.info("start for %d" % len(res))
418	count = 1
419	log_after = 100
420	for student in res:
421	if not count % log_after:
422	logger.info("processed %d total %d" % (log_after,count))
423	count += 1
424	fee_dict =self.getSchoolFee(student)
425	fulltime = student.mode.endswith('_ft')
426	d = {}
427	d['student_id'] = student.id
428	d['name'] = student.name
429	d['amount'] = fee_dict.get(new_returning)
430	l += d,
431	csv_name = self.dumpListToCSV(l,'payments')
432	logger.info('%s dumped payments to %s' % (member,export_file))
433	###)
434
435	security.declarePublic('dumpListToCSV') ###(
436	def dumpListToCSV(self,l,filename,fields=None):
437	"""dump a list of dicts to a CSV file"""
438	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
439	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
440	if fields is None:
441	fields = l[0].keys()
442	headline = ','.join(fields)
443	out = open(export_file,"wb")
444	out.write(headline +'\n')
445	out.close()
446	out = open(export_file,"a")
447	csv_writer = csv.DictWriter(out,fields,)
448	csv_writer.writerows(l)
449	return export_file
450	###)
451
452	security.declareProtected(ManagePortal, 'listMembers') ###(
453	def listMembers(self):
454	"list all members"
455	mtool = self.portal_membership
456	member = mtool.getAuthenticatedMember()
457	logger = logging.getLogger('WAeUPTool.listMembers')
458	if str(member) not in ('admin','joachim'):
459	logger.info('%s tried to list members' % (member))
460	return None
461	members = self.portal_directories.members
462	all = members.listEntryIdsAndTitles()
463	l = []
464	for user_id,name in all:
465	d = {}
466	d['user_id'] = user_id
467	d['name'] = name
468	d['pw'] = getattr(getattr(members,user_id),'password')
469	d['email'] = getattr(getattr(members,user_id),'email')
470	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
471	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
472	l += d,
473	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
474	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
475	logger.info('%s dumped memberlist to %s' % (member,export_file))
476	fields = l[0].keys()
477	headline = ','.join(fields)
478	out = open(export_file,"wb")
479	out.write(headline +'\n')
480	out.close()
481	out = open(export_file,"a")
482	csv_writer = csv.DictWriter(out,fields,)
483	csv_writer.writerows(l)
484	###)
485
486	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
487	def removeDeletedDocIds(self, max=1000):
488	"""
489	remove deleted docids from repository commit after max
490	"""
491	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
492	repository = getToolByName(self, 'portal_repository')
493	pxtool = getToolByName(self, 'portal_proxies')
494	logger.info('start')
495	pxtool_infos = pxtool.getRevisionsUsed()
496	logger.info('found %d used revisions ' % (len(pxtool_infos)))
497
498	nb_revs = 0
499	docids_d = {} # all docids
500	unused_docids_d = {} # all docids that are unused
501	ids_unused_revs_docids = [] # ids for revs of unused docids
502	unused_ids = [] # ids for unused revs
503	total = 0
504	idlist = repository.objectIds()
505	to_delete = 0
506	found = False
507	for id in idlist:
508	docid, rev = repository._splitId(id)
509	if docid is None:
510	logger.info("invalid doc_id %s" % docid)
511	continue
512	nb_revs += 1
513	if not pxtool_infos.has_key(docid):
514	found = True
515	to_delete += 1
516	unused_ids.append(id)
517	elif not pxtool_infos[docid].has_key(rev):
518	found = True
519	to_delete += 1
520	unused_ids.append(id)
521	if found and not to_delete % max:
522	found = False
523	#import pdb;pdb.set_trace()
524	repository.manage_delObjects(unused_ids)
525	transaction.commit()
526	logger.info('removed %d total %d unused docids ' % (max,to_delete))
527	else:
528	if unused_ids:
529	repository.manage_delObjects(unused_ids)
530	transaction.commit()
531	logger.info('finished removing %d unused docids ' % (to_delete))
532
533
534	###)
535
536	security.declareProtected(View,'getCredential') ###(
537	def getCredential(self,student_id):
538	student_entry = getattr(self.portal_directories.students,student_id,None)
539	if not self.isStaff():
540	mtool = self.portal_membership
541	member = mtool.getAuthenticatedMember()
542	logger = logging.getLogger('WAeUPTool.getCredential')
543	logger.info('%s tried to access password of %s' % (member,student_id))
544	return None
545	if student_entry is None:
546	return None
547	return getattr(student_entry,"password","not set")
548	###)
549
550	security.declarePublic('checkPassword') ###(
551	def checkPassword(self,student_id,password):
552	student_entry = getattr(self.portal_directories.students,student_id,None)
553	if student_entry is None:
554	return False
555	return getattr(student_entry,"password","not set") == password
556	###)
557
558	security.declarePublic('checkGenericPassword') ###(
559	def checkGenericPassword(self,member_id):
560	member_entry = getattr(self.portal_directories.members,member_id,None)
561	if member_entry is None:
562	return False
563	ltool = getToolByName(self, 'portal_layouts')
564	unsecure_words = ltool._getOb('members')['w__password'].check_words
565	password = getattr(member_entry,"password","not set")
566	is_unsecure = password in unsecure_words
567	if is_unsecure:
568	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
569	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
570	return is_unsecure
571	###)
572
573	security.declareProtected(ModifyPortalContent,'editPassword') ###(
574	def editPassword(self,student_id,password):
575	"edit a student password"
576	student_entry = getattr(self.portal_directories.students,student_id,None)
577	if student_entry is None:
578	return
579	setattr(student_entry,'password',password)
580	###)
581
582	security.declareProtected(ModifyPortalContent,'doCommit') ###(
583	def doCommit(self,logger=None):
584	"commit some transactions"
585	transaction.commit()
586	###)
587
588	security.declarePublic('loadStudentFoto') ###(
589	def loadStudentFoto(self,student,filename,folder):
590	"return a student passport picture"
591	#import pdb;pdb.set_trace()
592	picture ="%s/import/%s/%s" % (i_home,folder,filename)
593	student_id = student.getId()
594	images_dir = getImagesDir(student_id)
595	if not os.path.exists(images_dir):
596	os.mkdir(images_dir)
597	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
598	for extension in ('.jpg','.JPG'):
599	fullname = "%(picture)s%(extension)s" % vars()
600	if os.path.exists(fullname):
601	copy2(fullname,image_name)
602	return "successfully copied passport picture"
603	return "passport picture not found: %s.jpg or .JPG" % picture
604	###)
605
606	def old____loadStudentFoto(self,student,filename,folder): ###(
607	"return a student passport picture"
608	app = student.application
609	app_doc = app.getContent()
610	#clear = student.clearance
611	#clear_doc = clear.getContent()
612	#matric_no = clear_doc.matric_no.upper()
613	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
614	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
615	#import pdb;pdb.set_trace()
616	if os.path.exists(picture1):
617	file = open(picture1)
618	elif os.path.exists(picture2):
619	file = open(picture2)
620	else:
621	return "passport picture not found %s" % picture1
622	reopened = False
623	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
624	self.portal_workflow.doActionFor(app,'open')
625	reopened = True
626	outfile = file.read()
627	app_doc.manage_addFile('passport',
628	file=outfile,
629	title="%s.jpg" % filename)
630	if reopened:
631	self.portal_workflow.doActionFor(app,'close')
632	return "successfully loaded passport picture"
633	###)
634
635	security.declareProtected(ModifyPortalContent,'createOne') ###(
636	def createOne(self,students_folder,student_brain,letter,commit=False):
637	sid = self.waeup_tool.generateStudentId(letter)
638	students_folder.invokeFactory('Student', sid)
639	student = getattr(students_folder,sid)
640	self.portal_workflow.doActionFor(student,'return')
641	student.manage_setLocalRoles(sid, ['Owner',])
642	matric_no = student_brain.matric_no
643	jamb_reg_no = student_brain.Entryregno
644	self.students_catalog.addRecord(id = sid,
645	matric_no = matric_no,
646	jamb_reg_no = jamb_reg_no,
647	sex = student_brain.Sex == "F",
648	name = "%s %s %s" % (student_brain.Firstname,
649	student_brain.Middlename,
650	student_brain.Lastname)
651	)
652	if commit:
653	transaction.commit()
654	return sid,jamb_reg_no
655	###)
656
657	security.declareProtected(ModifyPortalContent,'addStudent') ###(
658	def addStudent(self,dict):
659	students_folder = self.portal_url.getPortalObject().campus.students
660	sid = self.waeup_tool.generateStudentId('?')
661	students_folder.invokeFactory('Student', sid)
662	student_obj = getattr(students_folder,sid)
663	f2t = self.field2types_student
664	#from pdb import set_trace; set_trace()
665	d = {}
666	#d['jamb_sex'] = 'M'
667	#if dict.get('sex'):
668	# d['jamb_sex'] = 'F'
669
670	entry_session = dict.get('entry_session')
671	if entry_session == self.getSessionId()[0]:
672	wfaction = 'admit'
673	wft = 'wf_transition_admit'
674	password = None
675	else:
676	wfaction = 'return'
677	wft = 'wf_transition_return'
678	password = self.generatePassword()
679	self.makeStudentMember(sid,password)
680
681	for pt in f2t.keys():
682	student_obj.invokeFactory(pt,f2t[pt]['id'])
683	sub_obj = getattr(student_obj,f2t[pt]['id'])
684	sub_doc = sub_obj.getContent()
685	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
686	d['Title'] = f2t[pt]['title']
687	for field in f2t[pt]['fields']:
688	d[field] = dict.get(field,'')
689	sub_doc.edit(mapping = d)
690	new_state = f2t[pt][wft]
691	if new_state != "remain":
692	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
693	self.portal_workflow.doActionFor(student_obj,wfaction)
694	student_obj.manage_setLocalRoles(sid, ['Owner',])
695	return sid,password
696	###)
697
698	security.declarePublic('getCertificateBrain') ###(
699	def getCertificateBrain(self,cert_id):
700	"do it"
701	res = ZCatalog.searchResults(self.portal_catalog_real,
702	{'portal_type':"Certificate",
703	'id': cert_id})
704	if res:
705	return res[0]
706	return None
707	###)
708
709	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
710	def get_csv_filenames(self):
711	"do it"
712	files = [file for file in os.listdir("%s/import/" % (i_home))
713	if file.endswith('.csv') and file.find('imported') == -1]
714	return files
715	###)
716
717	security.declarePublic('findStudentByMatricelNo') ###(
718	def findStudentByMatricelNo(self,matric_no):
719	"do it"
720	res = ZCatalog.searchResults(self.portal_catalog_real,
721	{'portal_type':"StudentClearance",
722	'SearchableText': matric_no})
723	if res:
724	return res[0]
725	return None
726	###)
727
728	security.declarePublic('makeStudentMember') ###(
729	def makeStudentMember(self,sid,password='uNsEt'):
730	"""make the student a member"""
731	membership = self.portal_membership
732	membership.addMember(sid,
733	password ,
734	roles=('Member',
735	'Student',
736	),
737	domains='',
738	properties = {'memberareaCreationFlag': False,
739	'homeless': True},)
740	member = membership.getMemberById(sid)
741	self.portal_registration.afterAdd(member, sid, password, None)
742	#self.manage_setLocalRoles(sid, ['Owner',])
743	###)
744
745	security.declareProtected(View,'makeStudentData') ###(
746	def makeStudentData(self,student_id,email=None,phone_nr=None):
747	"create Datastructure for a returning Student"
748	#import pdb;pdb.set_trace()
749	logger = logging.getLogger('WAeUPTool.makeStudentData')
750	students_folder = self.portal_url.getPortalObject().campus.students
751	#res = self.students_catalog(id=student_id)
752	#if res:
753	# st = res[0]
754	#res = self.returning_import(matric_no = st.matric_no)
755	res = self.returning_import(id = student_id)
756	if res:
757	student = res[0]
758	else:
759	logger.info('Id %s not found in returning_import' % student_id)
760	return
761	logger.info('%s creates data structure' % student_id)
762	s_results = self.results_import(matric_no = student.matric_no)
763	if s_results:
764	lnr = self.getLevelFromResultsCosCode(s_results)
765	level = "%d00" % lnr
766	verdict,eligible = self.getVerdict(s_results[0].Verdict)
767	#if eligible:
768	# level = "%d00" % (lnr + 1)
769	else:
770	logger.info('matric_no %s not found in results_import' % student.matric_no)
771	level = ''
772	verdict = ''
773	#student should not be allowed to perform this transition
774	#wftool = self.portal_workflow
775	#wftool.doActionFor(student,'return')
776	certcode_org = student.Coursemajorcode
777	certcode = makeCertificateCode(certcode_org)
778	certificate_brain = self.getCertificateBrain(certcode)
779	if not certificate_brain:
780	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
781	logger.info(em)
782	matric_no = student.matric_no
783	sid = student_id
784	student_obj = getattr(students_folder,sid)
785	if not getattr(student_obj,'application'):
786	student_obj.invokeFactory('StudentApplication','application')
787	application = student_obj.application
788	self.portal_workflow.doActionFor(application,'open',dest_container=application)
789	da = {'Title': 'Application Data'}
790	student_obj.invokeFactory('StudentPersonal','personal')
791	da['jamb_reg_no'] = student.Entryregno
792	em = self.getEntryMode(student.Entryregno)
793	da['entry_mode'] = em
794	personal = student_obj.personal
795	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
796	dp = {'Title': 'Personal Data'}
797	student_obj.invokeFactory('StudentClearance','clearance')
798	clearance = student_obj.clearance
799	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
800	dc = {'Title': 'Clearance/Eligibility Record'}
801	dc['matric_no'] = matric_no
802	state = student.State
803	lga = student.LGA
804	if state and lga:
805	lga = state + ' / ' + lga
806	else:
807	lga = "None"
808	da['jamb_lga'] = dc['lga'] = lga
809	da['app_email'] = dp['email'] = email
810	da['app_mobile'] = dp['phone'] = phone_nr
811	dp['firstname'] = student.Firstname
812	dp['middlename'] = student.Middlename
813	dp['lastname'] = student.Lastname
814	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
815	da['jamb_sex'] = student.Sex
816	dp['sex'] = student.Sex == 'F'
817	dp['perm_address'] = student.Permanent_Address
818	application.getContent().edit(mapping=da)
819	self.portal_workflow.doActionFor(application,'close',dest_container=application)
820	personal.getContent().edit(mapping=dp)
821	clearance.getContent().edit(mapping=dc)
822	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
823	#
824	# Study Course
825	#
826	student_obj.invokeFactory('StudentStudyCourse','study_course')
827	studycourse = student_obj.study_course
828	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
829	dsc = {}
830	dsc['study_course'] = certcode
831	dsc['current_level'] = level
832	dsc['current_verdict'] = verdict
833	dsc['current_mode'] = em
834	dsc['current_session'] = '05'
835	studycourse.getContent().edit(mapping=dsc)
836	#
837	# Level
838	#
839	## l = getattr(studycourse,level,None)
840	## if l is None:
841	## studycourse.invokeFactory('StudentStudyLevel', level)
842	## l = getattr(studycourse, level)
843	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
844	## l.getContent().edit(mapping={'Title': "Level %s" % level})
845	###)
846
847	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
848	def admitOneStudent(self,brain,entry_session,pin_password):
849	"create Datastructure for an admitted Student"
850	#import pdb;pdb.set_trace()
851	if not hasattr(self,"_v_certificates"):
852	self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
853	students_folder = self.portal_url.getPortalObject().campus.students
854	logger = logging.getLogger('WAeUPTool.admitOneStudent')
855	if brain.status != "admitted":
856	logger.info('status of %s is %s' % (brain.reg_no,brain.status))
857	return
858	pin_parts = brain.pin.split('-')
859	if pin_parts and len(pin_parts) != 3:
860	logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
861	return
862	if brain.course_admitted not in self._v_certificates:
863	logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
864	return
865	if brain.sex not in (True,False):
866	logger.info('sex of %s not available' % (brain.reg_no))
867	return
868	student_id = self.generateStudentId('?')
869	students_folder.invokeFactory('Student', student_id)
870	student_object = getattr(students_folder,student_id)
871	if pin_password:
872	password = pin_parts[2]
873	self.makeStudentMember(student_id,password = password)
874	student_object.manage_setLocalRoles(student_id, ['Owner',])
875	#logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
876	#
877	# application
878	#
879	student_object.invokeFactory('StudentApplication','application')
880	application = student_object.application
881	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
882	da = {'Title': 'Application Data'}
883	da['jamb_reg_no'] = brain.reg_no
884
885	sex = 'M'
886	if brain.sex:
887	sex = 'F'
888	da['jamb_sex'] = sex
889	da['jamb_age'] = brain.jamb_age
890	#da['app_ac_pin'] = brain.pin
891	da['jamb_lga'] = brain.jamb_lga
892	da['jamb_state'] = brain.jamb_state
893	da['jamb_score'] = brain.aggregate
894	da['app_email'] = brain.email
895	da['app_mobile'] = brain.phone
896	if brain.entry_mode:
897	da['entry_mode'] = brain.entry_mode
898	elif brain.screening_type == 'pume':
899	da['entry_mode'] = 'ume_ft'
900	elif brain.screening_type == 'pde':
901	da['entry_mode'] = 'de_ft'
902	elif brain.screening_type == 'pce':
903	da['entry_mode'] = 'pce'
904	elif brain.screening_type == 'prence':
905	da['entry_mode'] = 'prence'
906	else:
907	da['entry_mode'] = ''
908	da['entry_session'] = entry_session
909	da['jamb_lastname'] = brain.lastname
910	da['jamb_middlename'] = brain.middlenames # different field names!
911	da['jamb_firstname'] = brain.firstname
912	da['screening_application_date'] = brain.application_date
913	da['date_of_birth'] = brain.date_of_birth
914	da['jamb_first_cos'] = brain.course1
915	da['jamb_second_cos'] = brain.course2
916	da['course3'] = brain.course3
917	da['screening_type'] = brain.screening_type
918	da['screening_score'] = brain.screening_score
919	da['screening_date'] = brain.screening_date
920	da['hq_type'] = brain.hq_type
921	da['hq_grade'] = brain.hq_grade
922	da['aos'] = brain.aos
923
924	application.getContent().edit(mapping=da)
925	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
926	#
927	# personal
928	#
929	student_object.invokeFactory('StudentPersonal','personal')
930	personal = student_object.personal
931	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
932	dp = {'Title': 'Personal Data'}
933	dp['sex'] = brain.sex
934	dp['email'] = brain.email
935	dp['phone'] = brain.phone
936	dp['lastname'] = brain.lastname
937	dp['middlename'] = brain.middlenames # different field names!
938	dp['firstname'] = brain.firstname
939	personal.getContent().edit(mapping=dp)
940	#
941	# clearance
942	#
943	student_object.invokeFactory('StudentClearance','clearance')
944	clearance = student_object.clearance
945	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
946	dc = {'Title': 'Clearance/Eligibility Record'}
947	dc['lga'] = brain.lga
948	dc['birthday'] = brain.date_of_birth
949	clearance.getContent().edit(mapping=dc)
950	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
951	#
952	# study Course
953	#
954	student_object.invokeFactory('StudentStudyCourse','study_course')
955	studycourse = student_object.study_course
956	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
957	dsc = {}
958	dsc['study_course'] = brain.course_admitted
959	dsc['current_verdict'] = ''
960	dsc['current_mode'] = da['entry_mode']
961	if da['entry_mode'].startswith('de'):
962	dsc['current_level'] = '200'
963	elif da['entry_mode'].startswith('pre'):
964	dsc['current_level'] = '000'
965	else:
966	dsc['current_level'] = '100'
967	dsc['current_session'] = entry_session
968	studycourse.getContent().edit(mapping=dsc)
969	#
970	# payments folder
971	student_object.invokeFactory('PaymentsFolder','payments')
972	payments = getattr(student_object,'payments')
973	dpay = {}
974	dpay['Title'] = 'Payments'
975	payments.getContent().edit(mapping=dpay)
976	self.portal_workflow.doActionFor(payments,'open')
977	#
978	# passport foto
979	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
980	brain.screening_type,
981	brain.reg_no)
982	images_dir = getImagesDir(student_id)
983	#images_dir = os.path.join("%s" % images_base,student_id)
984	letter_dir,student_dir = os.path.split(images_dir)
985	if not os.path.exists(letter_dir):
986	os.mkdir(letter_dir)
987	if not os.path.exists(images_dir):
988	os.mkdir(images_dir)
989	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
990	if os.path.exists(app_picture):
991	copy2(app_picture,image_name)
992	else:
993	logger.info('passport of %s/%s not found: %s' % (student_id,
994	brain.reg_no,
995	app_picture))
996
997	return student_id
998	###)
999
1000	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1001	def makeStudentLevel(self,student_id):
1002	"create the StudyLevel for a returning Student"
1003	#import pdb;pdb.set_trace()
1004	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1005	students_folder = self.portal_url.getPortalObject().campus.students
1006	res = self.students_catalog(id=student_id)
1007	if res:
1008	st = res[0]
1009	course = st.course
1010	matric_no = st.matric_no
1011	level = st.level
1012	res = self.results_import(matric_no = matric_no)
1013	if res:
1014	results = res
1015	logger.info('%s creating Level %s' % (student_id,level))
1016	#
1017	# Level
1018	#
1019	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1020	studycourse = getattr(student_obj,"study_course",None)
1021	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1022	l = getattr(studycourse,level,None)
1023	if l is None:
1024	studycourse.invokeFactory('StudentStudyLevel', level)
1025	l = getattr(studycourse, level)
1026	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1027	l.getContent().edit(mapping={'Title': "Level %s" % level})
1028	###)
1029
1030	security.declarePublic('getHallInfo') ###(
1031	def getHallInfo(self,bed):
1032	"""return Hall Info"""
1033	info = {}
1034	hall,block,room,letter = bed.split('_')
1035	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1036	if res and len(res) == 1:
1037	hall_brain = res[0]
1038	hall_doc = hall_brain.getObject().getContent()
1039	else:
1040	return info
1041	info['hall_title'] = hall_brain.Title
1042	info['maintenance_code'] = hall_doc.maintenance_code
1043	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1044	batch_doc = None
1045	for brain in res:
1046	if brain.id.startswith(info['maintenance_code']):
1047	batch_doc = brain.getObject().getContent()
1048	break
1049	if batch_doc is None:
1050	info['maintenance_fee'] = ''
1051	else:
1052	info['maintenance_fee'] = batch_doc.cost
1053	return info
1054	###)
1055
1056	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1057	def removePictureFolder(self,student_id):
1058	"""remove picture_folder by renaming it"""
1059	path = 'images'
1060	picture_path = os.path.join(i_home,path,student_id)
1061	if not os.path.exists(picture_path):
1062	return False
1063	os.rename(picture_path,picture_path + "_removed")
1064	return True
1065	###)
1066
1067	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1068	def restorePictureFolder(self,student_id):
1069	"""restore picture_folder by renaming it"""
1070	path = 'images'
1071	picture_path = os.path.join(i_home,path,student_id)
1072	if not os.path.exists(picture_path + "_removed"):
1073	return False
1074	os.rename(picture_path + "_removed",picture_path)
1075	return True
1076	###)
1077
1078	security.declarePublic('picturesExist') ###(
1079	def picturesExist(self, ids,student_id=None):
1080	"""check if pictures exist in the filesystem"""
1081	if student_id is None:
1082	student_id = self.getStudentId()
1083	if student_id is None:
1084	return False
1085	picture_path = getImagesDir(student_id)
1086	#picture_path = os.path.join(images_base,student_id)
1087	if not os.path.exists(picture_path):
1088	return False
1089	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1090	return set(ids).issubset(set(pictures))
1091	###)
1092
1093	security.declarePublic('picturesList') ###(
1094	def picturesList(self):
1095	"""check if pictures exist in the filesystem"""
1096	path = 'images'
1097	student_id = self.getStudentId()
1098	#picture_path = os.path.join(i_home,path,student_id)
1099	picture_path = getImagesDir(student_id)
1100	if not os.path.exists(picture_path):
1101	return []
1102	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1103	###)
1104
1105	security.declarePublic('showFsPicture') ###(
1106	def showFsPicture(self,path):
1107	"""return a picture from the filesystem"""
1108	#picture_path = os.path.join(i_home,path)
1109	picture_path = os.path.join(images_base,path)
1110	response = self.REQUEST.RESPONSE
1111	#import pdb;pdb.set_trace()
1112	registry = getToolByName(self, 'mimetypes_registry')
1113	mimetype = str(registry.lookupExtension(path.lower()) or
1114	registry.lookupExtension('file.bin'))
1115	if os.path.exists(picture_path):
1116	response.setHeader('Content-type',mimetype)
1117	return open(picture_path).read()
1118	picture_path = os.path.join(i_home,'import',path)
1119	if os.path.exists(picture_path):
1120	return open(picture_path).read()
1121	###)
1122
1123	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1124	def deleteAllCourses(self,department="All"):
1125	''' delete the courses'''
1126	pm = self.portal_membership
1127	member = pm.getAuthenticatedMember()
1128
1129	if str(member) not in ("henrik","joachim"):
1130	return "not possible"
1131	if department == "All":
1132	res = self.portal_catalog({'meta_type': 'Department'})
1133	if len(res) < 1:
1134	return "No Departments found"
1135
1136	deleted = []
1137	for dep in res:
1138	cf = dep.getObject().courses
1139	if cf:
1140	cf.manage_delObjects(ids=cf.objectIds())
1141	deleted.append("deleted Courses in %s" % dep.getId)
1142	return "\r".join(deleted)
1143	###)
1144
1145	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1146	def getLogfileLines(self,filename="event.log",numlines=20):
1147	"""Get last NUMLINES lines of logfile FILENAME.
1148
1149	Return last lines' of a file in the instances logfile directory as
1150	a list. The number of returned lines equals `numlines' or less. If
1151	less than `numlines' lines are available, the whole file ist
1152	returned. If the file can not be opened or some other error
1153	occurs, empty list is returend.
1154	"""
1155	result = []
1156	lines_hit = 0
1157
1158	# We only handle files in instances' log directory...
1159	logpath = os.path.join(i_home, "log")
1160	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1161	if not filename.startswith( logpath ):
1162	# Attempt to access file outside log-dir...
1163	return []
1164
1165	try:
1166	fd = file( filename, "rb" )
1167	except IOError:
1168	return []
1169	if not fd:
1170	return []
1171
1172	if os.linesep == None:
1173	linesep = '\n'
1174	else:
1175	linesep = os.linesep
1176
1177	# Try to find 'numlines' times a lineseparator, searching from end
1178	# and moving to the beginning of file...
1179	fd.seek( 0, 2) # Move to end of file...
1180	while lines_hit < numlines:
1181	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1182	# one step forward...
1183	lines_hit += 1
1184	try:
1185	fd.seek( -2, 1) # Go two bytes back from current pos...
1186	except IOError:
1187	# We cannot go back two bytes. Maybe the file is too small...
1188	break
1189	fd.seek(2,1)
1190
1191	# Read all lines from current position...
1192	result = fd.readlines()
1193	# Remove line endings...
1194	result = [x.strip() for x in result]
1195	fd.close()
1196	return result
1197	###)
1198
1199	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1200	def getCallbacksFromLog(self,filename):
1201	"""fix Online Payment Transactions from Z2.log entries"""
1202	import transaction
1203	import random
1204	from cgi import parse_qs
1205	from urlparse import urlparse
1206	#from pdb import set_trace
1207	wftool = self.portal_workflow
1208	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1209	students_folder = self.portal_url.getPortalObject().campus.students
1210	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1211	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1212	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1213	data = re.compile(s)
1214	start = True
1215	tr_count = 1
1216	total = 0
1217	#name = 'pume_results'
1218	#name = 'epaymentsuccessful_z2log2'
1219	name = filename
1220	no_import = []
1221	imported = []
1222	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1223	try:
1224	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1225	except:
1226	logger.error('Error reading %s' % name)
1227	return
1228	tas = []
1229	for line in transactions:
1230	dict = {}
1231	items = data.search(line)
1232	dict['idict'] = idict = items.groupdict()
1233	#print idict
1234	#from pdb import set_trace;set_trace()
1235	urlparsed = urlparse(idict['get'][4:])
1236	#print urlparsed
1237	path = urlparsed[2].split('/')
1238	dict['student_id'] = student_id = path[8]
1239	dict['payment_id'] = payment_id = path[10]
1240	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1241	tas.append(dict)
1242	tr_count += 1
1243	return tas
1244	###)
1245
1246	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1247	def importOnlinePaymentTransactions(self):
1248	"""load Online Payment Transactions from CSV values"""
1249	import transaction
1250	import random
1251	#from pdb import set_trace
1252	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1253	opt = self.online_payments_import
1254	students_folder = self.portal_url.getPortalObject().campus.students
1255	start = True
1256	tr_count = 1
1257	total = 0
1258	#name = 'pume_results'
1259	name = 'OnlineTransactions'
1260	no_import = []
1261	imported = []
1262	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1263	try:
1264	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1265	except:
1266	logger.error('Error reading %s.csv' % name)
1267	return
1268	for pay_transaction in transactions:
1269	if start:
1270	start = False
1271	logger.info('Start loading from %s.csv' % name)
1272	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1273	no_import.append('%s,"Error"' % s)
1274	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1275	format_error = format + ',"%(Error)s"'
1276	data = {}
1277
1278	# format of the first file sent by Tayo
1279	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1280	#data['student_id'] = student_id = pay_transaction['Payer ID']
1281	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1282	#data['response_code'] = response_code = pay_transaction['Resp Code']
1283	#data['amount'] = amount = pay_transaction['Amount']
1284
1285	# format of the second file sent by Tayo
1286	#data['datetime'] = date = 0
1287	#data['student_id'] = student_id = pay_transaction['Payer ID']
1288	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1289	#data['response_code'] = response_code = '00'
1290	#data['amount'] = amount = pay_transaction['Amount']
1291
1292	# format of the third file sent by Kehinde
1293	data['datetime'] = date = 0
1294	data['student_id'] = student_id = pay_transaction['customer_id']
1295	data['order_id'] = order_id = pay_transaction['merchant_reference']
1296	data['response_code'] = response_code = '00'
1297	data['amount'] = amount = pay_transaction['Amount']
1298
1299	dup = False
1300	if response_code == "12":
1301	continue
1302	try:
1303	opt.addRecord(**data)
1304	except ValueError:
1305	dup = True
1306	#from pdb import set_trace;set_trace()
1307	if dup:
1308	if response_code == "00":
1309	try:
1310	opt.modifyRecord(**data)
1311	except:
1312	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1313	continue
1314	else:
1315	pay_transaction['Error'] = "Duplicate order_id"
1316	no_import.append( format_error % pay_transaction)
1317	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1318	continue
1319	tr_count += 1
1320	if tr_count > 1000:
1321	if len(no_import) > 0:
1322	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1323	'\n'.join(no_import) + '\n')
1324	no_import = []
1325	em = '%d transactions committed\n' % (tr_count)
1326	transaction.commit()
1327	regs = []
1328	logger.info(em)
1329	total += tr_count
1330	tr_count = 0
1331	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1332	'\n'.join(no_import))
1333	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1334	###)
1335
1336	security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1337	def mass_create_faculty(self,mapping):
1338	"create a faculty"
1339	logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1340	academics_folder = self.portal_url.getPortalObject().campus.academics
1341	fid = mapping['code']
1342	if getattr(academics_folder,fid,None) is not None:
1343	return '', "Faculty with ID: %s exists" % fid
1344	logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1345	try:
1346	academics_folder.invokeFactory('Faculty', fid)
1347	except BadRequest,E:
1348	return '', "%s" % E
1349	f = getattr(academics_folder,fid,None)
1350	f.getContent().edit(mapping=mapping)
1351	return fid,''
1352	###)
1353
1354	security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1355	def mass_edit_faculty(self,mapping):
1356	"edit a faculty"
1357	logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1358	academics_folder = self.portal_url.getPortalObject().campus.academics
1359	fid = mapping['code']
1360	f = getattr(academics_folder,fid,None)
1361	if f is None:
1362	return '', "Faculty with ID: %s does not exist" % fid
1363	logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1364	f.getContent().edit(mapping=mapping)
1365	return fid,''
1366	###)
1367
1368	security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1369	def mass_create_department(self,mapping):
1370	"create a department in the correct faculty"
1371	logger = logging.getLogger('WAeUPTool.mass_create_department')
1372	fid = mapping['faculty_code']
1373	if getattr(self,'_v_faculties',None) is None:
1374	res = self.portal_catalog(portal_type = "Faculty")
1375	self._v_faculties = {}
1376	for f in res:
1377	self._v_faculties[f.getId] = f.getObject()
1378	f = self._v_faculties.get(fid,None)
1379	if f is None:
1380	return '', "No Faculty with ID: %s" % fid
1381	else:
1382	did = mapping.get('code')
1383	d = getattr(f,did,None)
1384	if d is None or d.portal_type == "Faculty":
1385	logger.info('Creating Department %(code)s, %(title)s' % mapping)
1386	try:
1387	f.invokeFactory('Department', did)
1388	except BadRequest,E:
1389	return '', "%s" % E
1390	d = getattr(f,did)
1391	d.invokeFactory('CoursesFolder','courses')
1392	courses = getattr(d,'courses')
1393	dict = {'Title': 'Courses'}
1394	courses.getContent().edit(mapping=dict)
1395	d.invokeFactory('CertificatesFolder','certificates')
1396	certificates = getattr(d,'certificates')
1397	dict = {'Title': 'Certificates'}
1398	certificates.getContent().edit(mapping=dict)
1399	d.getContent().edit(mapping=mapping)
1400	return did,''
1401	###)
1402
1403	security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1404	def mass_edit_department(self,mapping):
1405	"create a department in the correct faculty"
1406	logger = logging.getLogger('WAeUPTool.mass_create_department')
1407	academics_folder = self.portal_url.getPortalObject().campus.academics
1408	fid = mapping['faculty_code']
1409	did = mapping.get('code')
1410	try:
1411	d = getattr(getattr(academics_folder,fid),did,None)
1412	except KeyError:
1413	return '', "Department %s or Faculty %s wrong" % (did,fid)
1414	else:
1415	if d is None or d.portal_type == "Faculty":
1416	logger.info('Editing Department %(code)s, %(title)s' % mapping)
1417	d.getContent().edit(mapping=mapping)
1418	return did,''
1419	###)
1420
1421	security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1422	def mass_create_course(self,mapping):
1423	#import pdb;pdb.set_trace()
1424	if getattr(self,'_v_course_list',None) is None:
1425	self._v_course_list = []
1426	if getattr(self,'_v_departments',None) is None:
1427	res = self.portal_catalog(portal_type = "Department")
1428	self._v_department_courses = {}
1429	for d in res:
1430	self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1431	did = mapping['department_code']
1432	d = self._v_department_courses.get(did,None)
1433	if d is None:
1434	return '', "No Department with ID: %s" % did
1435	course_id = mapping.get('code')
1436	if course_id in self._v_course_list:
1437	return '', "Duplicate Course ID: %s" % did
1438	c = getattr(d,course_id,None)
1439	if c is not None:
1440	return '', "Duplicate Course ID: %s" % did
1441	try:
1442	d.invokeFactory('Course', course_id)
1443	except BadRequest,E:
1444	return '', "%s" % E
1445	self._v_course_list.append(course_id)
1446	c = getattr(d,course_id)
1447	c.getContent().edit(mapping=mapping)
1448	return course_id,''
1449	###)
1450
1451	security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1452	def mass_edit_course(self,mapping):
1453	#import pdb;pdb.set_trace()
1454	course_id = mapping.get('code')
1455	res = self.portal_catalog(id=course_id)
1456	if not res:
1457	return '', "No Course with ID: %s" % course_id
1458	c = res[0].getObject()
1459	c.getContent().edit(mapping=mapping)
1460	return course_id,''
1461	###)
1462
1463	security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1464	def mass_create_certificate(self,mapping):
1465	if getattr(self,'_v_certificate_list',None) is None:
1466	self._v_certificate_list = []
1467	if getattr(self,'_v_department_certificates',None) is None:
1468	res = self.portal_catalog(portal_type = "Department")
1469	self._v_department_certificates = {}
1470	for d in res:
1471	self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1472	did = mapping['department_code']
1473	d = self._v_department_certificates.get(did,None)
1474	if d is None:
1475	return '', "No Department with ID: %s" % did
1476	certificate_id = mapping.get('code')
1477	if certificate_id in self._v_certificate_list:
1478	return '', "Duplicate Certificate ID: %s" % did
1479	c = getattr(d,certificate_id,None)
1480	if c is not None:
1481	return '', "Duplicate Certificate ID: %s" % did
1482	try:
1483	d.invokeFactory('Certificate', certificate_id)
1484	except BadRequest,E:
1485	return '', "%s" % E
1486	self._v_certificate_list.append(certificate_id)
1487	c = getattr(d,certificate_id)
1488	c.getContent().edit(mapping=mapping)
1489	return certificate_id,''
1490	###)
1491
1492	security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1493	def mass_edit_certificate(self,mapping):
1494	#import pdb;pdb.set_trace()
1495	certificate_id = mapping.get('code')
1496	res = self.portal_catalog(id=certificate_id)
1497	if not res:
1498	return '', "No Certificate with ID: %s" % certificate_id
1499	c = res[0].getObject()
1500	c.getContent().edit(mapping=mapping)
1501	return certificate_id,''
1502	###)
1503
1504	security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1505	def mass_create_application(self,mapping):
1506	#import pdb;pdb.set_trace()
1507	reg_no = mapping.get('reg_no')
1508	try:
1509	self.applicants_catalog.addRecord(**mapping)
1510	except ValueError:
1511	return '', "applicant record with reg_no %s already exists" % reg_no
1512	return reg_no,''
1513	###)
1514
1515	security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1516	def mass_edit_application(self,mapping):
1517	#import pdb;pdb.set_trace()
1518	reg_no = mapping.get('reg_no')
1519	status = mapping.get('status')
1520	res = self.applicants_catalog(reg_no = reg_no)
1521	if len(res):
1522	if res[0].status == 'created' and status != 'created':
1523	return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1524	elif status == 'created' and res[0].status != 'created':
1525	return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1526	else:
1527	self.applicants_catalog.modifyRecord(**mapping)
1528	return reg_no,''
1529	else:
1530	return '', "applicant record with reg_no %s does not exist" % reg_no
1531
1532	###)
1533
1534	security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1535	def mass_create_course_result(self,mapping):
1536	#import pdb;pdb.set_trace()
1537	students_folder = self.portal_url.getPortalObject().campus.students
1538	if getattr(self,'_v_courses',None) is None:
1539	res = self.courses_catalog()
1540	self._v_courses = {}
1541	for brain in res:
1542	self._v_courses[brain.code] = brain
1543	if getattr(self,'_v_level_created',None) is None:
1544	self._v_level_created = []
1545	course_id = mapping.get('code')
1546	if course_id not in self._v_courses.keys():
1547	return '', "No course with ID: %s" % course_id
1548	id_key = ''
1549	for id_key in ('student_id','matric_no'):
1550	id_field = mapping.get(id_key,None)
1551	if id_field is not None:
1552	student_id = id_field
1553	break
1554	query = Eq(id_key,id_field)
1555	res = self.students_catalog.evalAdvancedQuery(query)
1556	if not res:
1557	return '', "no student with %(id_key)s %(id_field)s" % vars()
1558	student_rec = res[0]
1559	if id_field != "student_id":
1560	mapping['student_id'] = student_rec.id
1561	student_id = student_rec.id
1562	level_id = mapping['level_id']
1563	code = mapping['code']
1564	if student_id not in self._v_level_created:
1565	try:
1566	context = getattr(getattr(students_folder,
1567	"%(student_id)s" % vars()),
1568	'study_course')
1569	except:
1570	return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1571	if level_id not in context.objectIds():
1572	context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1573	level = getattr(context,"%(level_id)s" % vars())
1574	self.portal_workflow.doActionFor(level,'open')
1575	# the session string must not be copied into the level object
1576	current_verdict = getattr(student_rec,'current_verdict','')
1577	current_session = getattr(student_rec,'current_session','')
1578	if current_verdict and student_rec.current_level == level_id:
1579	level.getContent().edit(mapping={'verdict': "%s" %
1580	current_verdict,
1581	'session': "%s" %
1582	current_session,
1583	})
1584	self.portal_workflow.doActionFor(level,'close')
1585	self._v_level_created += student_id,
1586	mapping['key'] = key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1587	for k in ('semester','credits',):
1588	mapping[k] = getattr(self._v_courses[course_id],k)
1589	try:
1590	self.course_results.addRecord(**mapping)
1591	except ValueError:
1592	return '', "course result already exists: %s" % key
1593	return key,''
1594	###)
1595
1596	security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1597	def mass_edit_course_result(self,mapping):
1598	while True:
1599	result_id = mapping.get('id',None)
1600	if result_id is not None:
1601	mapping['key'] = key = result_id
1602	break
1603	for id_key in ('student_id','matric_no'):
1604	id_field = mapping.get(id_key,None)
1605	if id_field is not None:
1606	student_id = id_field
1607	break
1608	query = Eq(id_key,id_field)
1609	res = self.students_catalog.evalAdvancedQuery(query)
1610	if not res:
1611	return '', "No student with %(id_key)s %(id_field)s" % vars()
1612	if id_field != "student_id":
1613	mapping['student_id'] = res[0].id
1614	if not res:
1615	return '', "No student with %(id_key)s %(id_field)s" % vars()
1616	student_rec = res[0]
1617	student_id = student_rec.id
1618	level_id = mapping['level_id']
1619	code = mapping['code']
1620	mapping['key'] = key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1621	break
1622	try:
1623	self.course_results.modifyRecord(**mapping)
1624	except KeyError:
1625	return key, "No course result to edit: %s" % key
1626	return key,''
1627	###)
1628
1629	security.declareProtected(ModifyPortalContent,'mass_delete_course_result') ###(
1630	def mass_delete_course_result(self,mapping):
1631	id_key = ''
1632	for id_key in ('student_id','matric_no'):
1633	id_field = mapping.get(id_key,None)
1634	if id_field is not None:
1635	student_id = id_field
1636	break
1637	query = Eq(id_key,id_field)
1638	res = self.students_catalog.evalAdvancedQuery(query)
1639	if not res:
1640	return '', "No student with %(id_field)s: %(id_key)s" % vars()
1641	if id_field != "student_id":
1642	mapping['student_id'] = res[0].id
1643	if not res:
1644	return '', "no student with %(id_key)s %(id_field)s" % vars()
1645	student_rec = res[0]
1646	student_id = student_rec.id
1647	level_id = mapping['level_id']
1648	code = mapping['code']
1649	key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1650	# for k in ('semester',):
1651	# mapping[k] = getattr(self._v_courses[course_id],k)
1652	if self.course_results.getRecordByKey(key) is None:
1653	return '', "no course-result with %(key)s" % vars()
1654	self.course_results.deleteRecord(key)
1655	return key,''
1656	###)
1657
1658	security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1659	def mass_create_certificate_course(self,mapping):
1660	if getattr(self,'_v_courses',None) is None:
1661	res = self.courses_catalog()
1662	self._v_courses= [course.code for course in res]
1663	if getattr(self,'_v_ceritficates',None) is None:
1664	res = self.portal_catalog(portal_type = "Certificate")
1665	self._v_certificates = {}
1666	for cert in res:
1667	self._v_certificates[cert.getId] = cert.getObject()
1668	certificate_course_id = mapping.get('code')
1669	if certificate_course_id not in self._v_courses:
1670	return '', "No Course with ID: %s" % certificate_course_id
1671	cert_id = mapping['certificate_code']
1672	cert = self._v_certificates.get(cert_id,None)
1673	if cert is None:
1674	return '', "No Certificate with ID: %s" % cert_id
1675	level_id = mapping.get('level')
1676	level = getattr(cert,level_id,None)
1677	if level is None:
1678	cert.invokeFactory('StudyLevel', level_id)
1679	level = getattr(cert,level_id,None)
1680	elif hasattr(level,certificate_course_id):
1681	return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1682	(certificate_course_id,cert_id,level_id)
1683	level.invokeFactory('CertificateCourse', certificate_course_id)
1684	c = getattr(level,certificate_course_id)
1685	c.getContent().edit(mapping=mapping)
1686	return certificate_course_id,''
1687	###)
1688
1689	field2types_student = { ###(
1690	'StudentApplication':
1691	{'id': 'application',
1692	'title': 'Application Data',
1693	'wf_transition_return': 'close',
1694	'wf_transition_admit': 'remain',
1695	'fields':
1696	('jamb_reg_no',
1697	'entry_mode',
1698	'entry_session',
1699	'jamb_score',
1700	'app_email',
1701	'jamb_age',
1702	'jamb_state',
1703	'jamb_lga',
1704	'jamb_sex',
1705	)
1706	},
1707	#'StudentPume':
1708	# {'id': 'pume',
1709	# 'title': 'Pume Data',
1710	# 'wf_transition_return': 'close',
1711	# 'wf_transition_admit': 'close',
1712	# 'fields':
1713	# ('pume_score',
1714	# )
1715	# },
1716	'StudentClearance':
1717	{'id': 'clearance',
1718	'title': 'Clearance/Eligibility Record',
1719	'wf_transition_return': 'close',
1720	'wf_transition_admit': 'remain',
1721	'fields':
1722	('matric_no',
1723	'nationality',
1724	'lga',
1725	'birthday',
1726	)
1727	},
1728	'StudentPersonal':
1729	{'id': 'personal',
1730	'title': 'Personal Data',
1731	'wf_transition_return': 'open',
1732	'wf_transition_admit': 'remain',
1733	'fields':
1734	('firstname',
1735	'middlename',
1736	'lastname',
1737	'sex',
1738	'email',
1739	'phone',
1740	'perm_address',
1741	)
1742	},
1743	'StudentStudyCourse':
1744	{'id': 'study_course',
1745	'title': 'Study Course',
1746	'wf_transition_return': 'open',
1747	'wf_transition_admit': 'remain',
1748	'fields':
1749	('study_course',
1750	'current_level',
1751	'current_session',
1752	'current_mode',
1753	'current_verdict',
1754	'previous_verdict',
1755	)
1756	},
1757	# 'StudentStudyLevel':
1758	# {'id': 'current_level',
1759	# 'title': '',
1760	# 'wf_transition_return': 'open',
1761	# 'wf_transition_admit': 'remain',
1762	# 'fields':
1763	# ('verdict',
1764	# 'session',
1765	# )
1766	# },
1767	'PaymentsFolder':
1768	{'id': 'payments',
1769	'title': 'Payments',
1770	'wf_transition_return': 'open',
1771	'wf_transition_admit': 'open',
1772	'fields':
1773	()
1774	},
1775	}
1776	###)
1777
1778	security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1779	def mass_create_student(self,mapping):
1780	"create student records due import"
1781	logger = logging.getLogger('WAeUPTool.mass_create_student')
1782	students_folder = self.portal_url.getPortalObject().campus.students
1783	jamb_reg_no = mapping.get('jamb_reg_no',None)
1784	if jamb_reg_no:
1785	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1786	if res:
1787	return '',"jamb_reg_no exists"
1788	matric_no = mapping.get('matric_no',None)
1789	if matric_no:
1790	res = self.students_catalog(matric_no = matric_no)
1791	if res:
1792	return '',"matric_no exists"
1793	sid = self.waeup_tool.generateStudentId('?')
1794	students_folder.invokeFactory('Student', sid)
1795	student_obj = getattr(students_folder,sid)
1796	f2t = self.field2types_student
1797	d = {}
1798	transition = mapping.get('reg_transition','admit')
1799	if transition not in ('admit','return'):
1800	return '',"no valid transition provided"
1801	for pt in f2t.keys():
1802	student_obj.invokeFactory(pt,f2t[pt]['id'])
1803	sub_obj = getattr(student_obj,f2t[pt]['id'])
1804	sub_doc = sub_obj.getContent()
1805	d['Title'] = f2t[pt]['title']
1806	for field in f2t[pt]['fields']:
1807	d[field] = mapping.get(field,'')
1808
1809	if pt == "StudentApplication":
1810	#d['jamb_sex'] = 'M'
1811	#if mapping.get('sex'):
1812	# d['jamb_sex'] = 'F'
1813	d['jamb_firstname'] = mapping.get('firstname',None)
1814	d['jamb_middlename'] = mapping.get('middlename',None)
1815	d['jamb_lastname'] = mapping.get('lastname',None)
1816
1817	# if pt == "StudyCourse":
1818	# for von,zu in (('entry_mode','current_mode'),
1819	# ('entry_session','current_session')):
1820	# if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1821	# d[zu] = mapping[von]
1822	sub_doc.edit(mapping = d)
1823
1824	#import pdb;pdb.set_trace()
1825	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1826	if new_state != "remain":
1827	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1828	self.portal_workflow.doActionFor(student_obj,transition)
1829	student_obj.manage_setLocalRoles(sid, ['Owner',])
1830	return sid,''
1831	###)
1832
1833	security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1834	def mass_edit_student(self,mapping):
1835	wftool = self.portal_workflow
1836	"edit student records due import"
1837	logger = logging.getLogger('WAeUPTool.mass_edit_student')
1838	students_folder = self.portal_url.getPortalObject().campus.students
1839	sid = mapping.get('id',None)
1840	jamb_reg_no = mapping.get('jamb_reg_no',None)
1841	matric_no = mapping.get('matric_no',None)
1842	editable_keys = mapping.keys()
1843	if sid:
1844	res = self.students_catalog(id = sid)
1845	if not res:
1846	return '',"no student with id %s" % sid
1847	if matric_no and res[0].matric_no and\
1848	matric_no != res[0].matric_no:
1849	logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1850	if jamb_reg_no and res[0].jamb_reg_no and\
1851	jamb_reg_no != res[0].jamb_reg_no:
1852	logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1853	elif jamb_reg_no:
1854	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1855	if not res:
1856	return '',"no student with jamb_reg_no %s" % jamb_reg_no
1857	editable_keys.remove('jamb_reg_no')
1858	elif matric_no:
1859	res = self.students_catalog(matric_no = matric_no)
1860	if not res:
1861	return '',"no student with matric_no %s" % matric_no
1862	editable_keys.remove('matric_no')
1863
1864	## included only to change wf state from admitted to returning
1865	#if res[0].review_state not in ('admitted','objection_raised'):
1866	# return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1867	## end inclusion
1868
1869	sid = res[0].id
1870	student_obj = getattr(students_folder,sid)
1871	f2t = self.field2types_student
1872	d = {}
1873	#import pdb;pdb.set_trace()
1874	any_change = False
1875	#special treatment for StudentStudyLevel
1876	d['verdict'] = mapping.get('current_verdict','')
1877	d['session'] = mapping.get('current_session','')
1878	current_level = mapping.get('current_level','')
1879	while d['session'] and d['verdict'] and current_level:
1880	sub_obj = getattr(student_obj,'study_course',None)
1881	if sub_obj is None:
1882	break
1883	level_obj = getattr(sub_obj,current_level,None)
1884	if level_obj is None:
1885	break
1886	any_change = True
1887	level_obj.getContent().edit(mapping = d)
1888	try:
1889	wftool.doActionFor(level_obj,'close')
1890	except:
1891	pass
1892	break
1893	for pt in f2t.keys():
1894	#if pt == "StudentApplication":
1895	# d['jamb_sex'] = 'M'
1896	# if mapping.get('sex'):
1897	# d['jamb_sex'] = 'F'
1898	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1899	if intersect and pt not in ('StudentStudyLevel',):
1900	object_id = f2t[pt]['id']
1901	sub_obj = getattr(student_obj,object_id,None)
1902	if sub_obj is None:
1903	try:
1904	student_obj.invokeFactory(pt,object_id)
1905	except:
1906	continue
1907	sub_obj = getattr(student_obj,object_id)
1908	if f2t[pt]['title'] != '':
1909	d['Title'] = f2t[pt]['title']
1910	sub_doc = sub_obj.getContent()
1911	for field in intersect:
1912	changed = False
1913	if getattr(sub_doc,field,None) != mapping.get(field,''):
1914	any_change = True
1915	changed = True
1916	d[field] = mapping.get(field,'')
1917	if changed:
1918	sub_doc.edit(mapping = d)
1919
1920
1921	## included only to change wf state from admitted to returning
1922	# if res[0].review_state in ('admitted','objection_raised'):
1923	# new_state = f2t[pt]['wf_transition_return']
1924	# sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1925	# if sub_obj and new_state != "remain":
1926	# try:
1927	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1928	# except:
1929	# #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1930	# pass
1931	#if res[0].review_state in ('admitted','objection_raised'):
1932	# wfaction = 'return'
1933	# try:
1934	# self.portal_workflow.doActionFor(student_obj,wfaction)
1935	# logger.info('%s, wf state changed' % sid)
1936	# any_change = True
1937	# except:
1938	# logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1939	# pass
1940	## end inclusion
1941
1942
1943	if any_change:
1944	return sid,''
1945	else:
1946	return sid,'not modified'
1947	###)
1948
1949	security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
1950	def mass_edit_verdict(self,mapping):
1951	wftool = self.portal_workflow
1952	"edit student verdicts"
1953	logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
1954	students_folder = self.portal_url.getPortalObject().campus.students
1955	student_id = mapping.get('id',None)
1956	matric_no = mapping.get('matric_no',None)
1957	editable_keys = mapping.keys()
1958	if student_id:
1959	student_record = self.students_catalog.getRecordByKey(student_id)
1960	if student_record is None:
1961	return '',"no student with id %s" % student_id
1962	if matric_no and student_record.matric_no and\
1963	matric_no != student_record.matric_no:
1964	return '', 'student %s: matric_no %s does not match %s' % (student_record.id,
1965	student_record.matric_no,
1966	matric_no)
1967	elif matric_no:
1968	res = self.students_catalog(matric_no = matric_no)
1969	if not res:
1970	return '',"no student with matric_no %s" % matric_no
1971	student_record = res[0]
1972	editable_keys.remove('matric_no')
1973	else:
1974	return '',"no id or matric_no specified"
1975	student_id = student_record.id
1976	d = {}
1977	#import pdb;pdb.set_trace()
1978	any_change = False
1979	#special treatment for StudentStudyLevel
1980	current_session = d['session'] = mapping.get('current_session','')
1981	if current_session and student_record.session != current_session:
1982	return student_id,'student %s: imported session %s does not match current_session %s' % (student_id,
1983	current_session,
1984	student_record.session)
1985	current_level = mapping.get('current_level','')
1986	if not current_level.isdigit():
1987	return '','student %s: imported level is empty' % (student_id,)
1988	if current_level and student_record.level != current_level:
1989	return student_id,'student %s: imported level %s does not match current_level %s' % (student_id,
1990	current_level,
1991	student_record.level)
1992	student_review_state = student_record.review_state
1993	if student_review_state == 'deactivated':
1994	return '',"student %s in review_state %s" % (student_id, student_review_state)
1995	if student_review_state not in ('courses_validated','returning'):
1996	return student_id,"student %s in wrong review_state %s" % (student_id, student_review_state)
1997	student_obj = getattr(students_folder,student_id)
1998	f2t = self.field2types_student
1999	study_course_obj = getattr(student_obj,'study_course',None)
2000	if study_course_obj is None:
2001	return '','student %s: no study_course object' % student_id
2002	level_obj = getattr(study_course_obj,current_level,None)
2003	if level_obj is None:
2004	return '','student %s: no study_level object for level %s' % (student_id,
2005	current_level)
2006	verdict = d['verdict'] = d['current_verdict'] = mapping.get('current_verdict','')
2007
2008	#if verdict == student_record.verdict:
2009	# return '','student %s: verdict already set to %s' % (student_id,
2010	# verdict)
2011
2012	level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
2013	if level_review_state != "closed":
2014	wftool.doActionFor(level_obj,'close')
2015	# return '','student %s: level %s is not closed' % (student_id,
2016	# current_level)
2017
2018	study_course_obj.getContent().edit(mapping = d)
2019	level_obj.getContent().edit(mapping = d)
2020	if student_review_state != "returning":
2021	wftool.doActionFor(student_obj,'return')
2022	# try:
2023	# wftool.doActionFor(level_obj,'close')
2024	# except:
2025	# pass
2026	return student_id,''
2027	###)
2028
2029	security.declareProtected(ModifyPortalContent,"importData")###(
2030	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
2031	"""load data from CSV values"""
2032	import transaction
2033	import random
2034	member = self.portal_membership.getAuthenticatedMember()
2035	logger = logging.getLogger('WAeUPTool.importData')
2036	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2037	students_folder = self.portal_url.getPortalObject().campus.students
2038	start = True
2039	tr_count = 0
2040	total_imported = 0
2041	total_not_imported = 0
2042	total = 0
2043	pending_only = False
2044	pend_str = '_pending'
2045	if name.endswith(pend_str):
2046	pending_only = True
2047	name = name[:-len(pend_str)]
2048	iname = "import_%s" % name
2049	if name in ('application','course_result',):
2050	commit_after = 2000
2051	else:
2052	commit_after = 100
2053	stool = getToolByName(self, 'portal_schemas')
2054	ltool = getToolByName(self, 'portal_layouts')
2055	schema = stool._getOb(iname)
2056	if schema is None:
2057	em = 'No such schema %s' % iname
2058	logger.error('No such schema %s' % iname)
2059	return em
2060	layout = ltool._getOb(iname)
2061	if layout is None:
2062	em = 'No such layout %s' % iname
2063	logger.error(em)
2064	return em
2065	validators = {}
2066	for widget in layout.keys():
2067	validators[widget] = layout[widget].validate
2068	mode = "create"
2069	if edit:
2070	if filename.endswith('_toDelete'):
2071	mode = "delete"
2072	else:
2073	mode = "edit"
2074	importer_name = "mass_%(mode)s_%(name)s" % vars()
2075	importer = getattr(self, '%s' % importer_name,None)
2076	if importer is None:
2077	em = 'No importer function %s' % importer_name
2078	logger.error(em)
2079	return em
2080	pending_fn = "%s/import/%ss_pending.csv" % (i_home,name)
2081	if pending_only:
2082	import_source_fn = pending_fn
2083	imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
2084	not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current)
2085	if not os.path.exists(pending_fn):
2086	em = 'No %s' % os.path.split(pending_fn)
2087	return em
2088	attrs = csv.reader(open(pending_fn,"rb")).next()
2089	else:
2090	import_source_fn = "%s/import/%s.csv" % (i_home,filename)
2091	imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
2092	not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
2093	attrs = csv.reader(open(import_source_fn,"rb")).next()
2094	import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
2095	or k.strip() == 'Error')]
2096	diff2schema = set(import_keys).difference(set(schema.keys()))
2097	diff2layout = set(import_keys).difference(set(layout.keys()))
2098	if diff2schema and diff2schema != set(['id',]):
2099	#import pdb;pdb.set_trace()
2100	msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2101	return msg
2102	if mode == "create":
2103	required_keys = [layout.getIdUnprefixed(id)
2104	for id,widget in layout.objectItems()
2105	if widget.is_required]
2106	if not set(required_keys).issubset(set(import_keys)):
2107	diff2import = set(required_keys).difference(set(import_keys))
2108	msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2109	return msg
2110	#
2111	# not_imported
2112	#
2113	not_imported_keys = import_keys + ["Error",]
2114	not_imported_file = open(not_imported_fn,"a")
2115	not_imported_csv_writer = csv.DictWriter(not_imported_file,
2116	not_imported_keys,
2117	extrasaction='ignore')
2118	mapping = dict((k,k) for k in not_imported_keys)
2119	not_imported_csv_writer.writerow(mapping)
2120	not_imported = []
2121	#
2122	# imported
2123	#
2124	imported_keys = import_keys[:]
2125	if 'id' not in imported_keys:
2126	imported_keys.insert(0,"id")
2127	imported_file = open(imported_fn,"a")
2128	imported_csv_writer = csv.DictWriter(imported_file,
2129	imported_keys,
2130	extrasaction='ignore')
2131	mapping = dict((k,k) for k in imported_keys)
2132	imported_csv_writer.writerow(mapping)
2133	imported = []
2134	#
2135	# pending
2136	#
2137	pending_records = {}
2138	imports_pending = 0
2139	total_pending = 0
2140	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2141	format_error = format + ',"%(Error)s"'
2142	#format = '"%(id)s",'+ format
2143	imported = []
2144	if name in ('verdict','course_result',):
2145	#global pending_records
2146	pending_keys = imported_keys[:]
2147	pending_keys += "Error",
2148	pending_records = {}
2149	if os.path.exists(pending_fn):
2150	pending_imports_file = open(pending_fn)
2151	pending_imports = csv.DictReader(pending_imports_file,
2152	fieldnames=pending_keys)
2153	if pending_imports:
2154	pending_records = dict((row['id'],row) for row in pending_imports
2155	if row['id'] != 'id')
2156	pending_imports_file.close()
2157	new_imports_pending = 0
2158	pending_imported = []
2159	for item in pending_records.values():
2160	id,error = importer(item)
2161	if error:
2162	continue
2163	else:
2164	if item.has_key('Error'):
2165	del item['Error']
2166	imported.append(item)
2167	msg = format % item
2168	logger.info("imported from %(pending_fn)s %(msg)s" % vars())
2169	pending_imported += id,
2170	continue
2171	total_imported = len(pending_imported)
2172	logger.info("imported %d pending imports" % len(pending_imported))
2173	for id in pending_imported:
2174	del pending_records[id]
2175	pending_imports_file = open(pending_fn,'w')
2176	pending_csv_writer = csv.DictWriter(pending_imports_file,
2177	pending_keys,
2178	extrasaction='ignore')
2179	mapping = dict((k,k) for k in pending_keys)
2180	pending_csv_writer.writerow(mapping)
2181	#
2182	if pending_only:
2183	items = []
2184	else:
2185	try:
2186	items = csv.DictReader(open(import_source_fn,"rb"))
2187	except:
2188	msg = 'Error reading %s.csv' % filename
2189	logger.error(msg)
2190	return msg
2191	not_imported = []
2192	for item in items:
2193	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2194	if start:
2195	start = False
2196	adapters = [MappingStorageAdapter(schema, item)]
2197	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2198	dm = DataModel(item, adapters,context=self)
2199	ds = DataStructure(data=item,datamodel=dm)
2200	error_string = ""
2201	total += 1
2202	error_count = 0
2203	for k in import_keys:
2204	if not validators[k](ds,mode=mode):
2205	if error_count:
2206	error_string += ' ++ '
2207	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
2208	ds.getErrorMapping(k)))
2209	error_count += 1
2210	if error_string:
2211	item['Error'] = error_string
2212	not_imported.append(item)
2213	total_not_imported += 1
2214	continue
2215	temp_item = item.copy()
2216	temp_item.update(dm)
2217	id,error = importer(temp_item)
2218	if error:
2219	item['Error'] = error
2220	if id:
2221	if id not in pending_records.keys():
2222	item['id'] = id
2223	pending_records[id] = item
2224	msg = format_error % item
2225	logger.info("%(id)s added to pending %(msg)s" % vars())
2226	new_imports_pending += 1
2227	else:
2228	logger.info("%(id)s already in pending %(msg)s" % vars())
2229	continue
2230	else:
2231	not_imported.append(item)
2232	total_not_imported += 1
2233	continue
2234	item = temp_item
2235	item['id'] = id
2236	imported.append(item)
2237	tr_count += 1
2238	total_imported += 1
2239	msg = format % item
2240	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
2241	if total and not total % commit_after:
2242	transaction.commit()
2243	if len(imported) > 0:
2244	imported_csv_writer.writerows(imported)
2245	imported = []
2246	if len(not_imported) > 0:
2247	not_imported_csv_writer.writerows(not_imported)
2248	not_imported = []
2249	if len(pending_records) > 0:
2250	pending_csv_writer.writerows(pending_records.values())
2251	total_pending += len(pending_records)
2252	pending_records = {}
2253	msg = '%d transactions committed\n' % (tr_count)
2254	regs = []
2255	logger.info(msg)
2256	tr_count = 0
2257	if len(imported) > 0:
2258	imported_csv_writer.writerows(imported)
2259	if len(not_imported) > 0:
2260	not_imported_csv_writer.writerows(not_imported)
2261	if len(pending_records) > 0:
2262	pending_csv_writer.writerows(pending_records.values())
2263	total_pending += len(pending_records)
2264	import_source_fn = os.path.split(import_source_fn)[1]
2265	pending_fn = os.path.split(pending_fn)[1]
2266	msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
2267	msg += "%(total_imported)d imported, %(total_not_imported)d not imported " % vars()
2268	if total_pending:
2269	msg += "%(new_imports_pending)d pending added, %(total_pending)d total pending in %(pending_fn)s " % vars()
2270	msg += "(of total %(total)d)" % vars()
2271	logger.info(msg)
2272	return msg
2273	###)
2274
2275	def old_importData(self,filename,name,edit=False,bypass_queue_catalog=False):###(
2276	"""load data from CSV values"""
2277	import transaction
2278	import random
2279
2280	pm = self.portal_membership
2281	member = pm.getAuthenticatedMember()
2282
2283	logger = logging.getLogger('WAeUPTool.importData')
2284	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2285	students_folder = self.portal_url.getPortalObject().campus.students
2286	start = True
2287	tr_count = 0
2288	total_imported = 0
2289	total_not_imported = 0
2290	total = 0
2291	iname = "import_%s" % name
2292	if iname in ('import_application','import_course_result',):
2293	commit_after = 2000
2294	else:
2295	commit_after = 100
2296	stool = getToolByName(self, 'portal_schemas')
2297	ltool = getToolByName(self, 'portal_layouts')
2298	schema = stool._getOb(iname)
2299	if schema is None:
2300	em = 'No such schema %s' % iname
2301	logger.error('No such schema %s' % iname)
2302	return em
2303	layout = ltool._getOb(iname)
2304	if layout is None:
2305	em = 'No such layout %s' % iname
2306	logger.error(em)
2307	return em
2308	validators = {}
2309	for widget in layout.keys():
2310	validators[widget] = layout[widget].validate
2311	mode = "create"
2312	if edit:
2313	if filename.endswith('_toDelete'):
2314	mode = "delete"
2315	else:
2316	mode = "edit"
2317	importer_name = "mass_%(mode)s_%(name)s" % vars()
2318	importer = getattr(self, '%s' % importer_name,None)
2319	if importer is None:
2320	em = 'No importer function %s' % importer_name
2321	logger.error(em)
2322	return em
2323	not_imported = []
2324	imported = []
2325	try:
2326	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
2327	except:
2328	em = 'Error reading %s.csv' % filename
2329	logger.error(em)
2330	return em
2331	for item in items:
2332	#import pdb;pdb.set_trace()
2333	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2334	if start:
2335	start = False
2336	adapters = [MappingStorageAdapter(schema, item)]
2337	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2338	attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
2339	import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
2340	diff2schema = set(import_keys).difference(set(schema.keys()))
2341	diff2layout = set(import_keys).difference(set(layout.keys()))
2342	if diff2schema:
2343	em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2344	return em
2345	if mode == "create":
2346	required_keys = [layout.getIdUnprefixed(id)
2347	for id,widget in layout.objectItems()
2348	if widget.is_required]
2349	if not set(required_keys).issubset(set(import_keys)):
2350	diff2import = set(required_keys).difference(set(import_keys))
2351	em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2352	return em
2353	s = ','.join(['"%s"' % fn for fn in import_keys])
2354	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
2355	s = '"id",' + s
2356	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
2357	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2358	format_error = format + ',"%(Error)s"'
2359	format = '"%(id)s",'+ format
2360
2361	dm = DataModel(item, adapters,context=self)
2362	ds = DataStructure(data=item,datamodel=dm)
2363	error_string = ""
2364	total += 1
2365	error_count = 0
2366	for k in import_keys:
2367	if not validators[k](ds,mode=mode):
2368	if error_count:
2369	error_string += ' ++ '
2370	error_string += "%s: %s" % (k,
2371	self.translation_service(ds.getError(k),
2372	ds.getErrorMapping(k)))
2373	error_count += 1
2374	if not error_string:
2375	temp_item = item.copy()
2376	temp_item.update(dm)
2377	temp_item['id'],error = importer(temp_item)
2378	if error:
2379	error_string += error
2380	else:
2381	item = temp_item
2382	if error_string:
2383	item['Error'] = error_string
2384	not_imported.append(format_error % item)
2385	total_not_imported += 1
2386	else:
2387	em = format % item
2388	imported.append(em)
2389	tr_count += 1
2390	total_imported += 1
2391	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(em)s" % vars())
2392
2393	if total and not total % commit_after:
2394	transaction.commit()
2395	if len(not_imported) > 0:
2396	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2397	'\n'.join(not_imported) + '\n')
2398	not_imported = []
2399	if len(imported) > 0:
2400	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2401	'\n'.join(imported) + '\n')
2402	imported = []
2403	em = '%d transactions committed\n' % (tr_count)
2404	regs = []
2405	logger.info(em)
2406	tr_count = 0
2407	#if total > 100:
2408	# return
2409	if len(imported) > 0:
2410	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2411	'\n'.join(imported))
2412	if len(not_imported) > 0:
2413	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2414	'\n'.join(not_imported))
2415	#if mode.endswith('e'):
2416	# mode = mode[:-1]
2417	em = "Finished importing in %(mode)s mode from %(filename)s.csv: %(total_imported)d imported, %(total_not_imported)d not imported (of total %(total)d)" % vars()
2418	logger.info(em)
2419	return em
2420	###)
2421
2422	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2423	def moveImagesToFS(self,student_id="O738726"):
2424	"move the images to the filesystem"
2425	images_dir = getImagesDir(student_id)
2426	#images_dir = os.path.join("%s" % images_base,student_id)
2427	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2428	stool = getToolByName(self, 'portal_schemas')
2429	schemas = ['student_application',
2430	'student_clearance',
2431	]
2432	created = False
2433	for schema_id in schemas:
2434	schema = stool._getOb(schema_id)
2435	object = getattr(student_folder,schema_id[len('student_'):],None)
2436	if object is None:
2437	continue
2438	doc = object.getContent()
2439	for key in schema.keys():
2440	if schema[key].meta_type != "CPS Image Field":
2441	continue
2442	#import pdb;pdb.set_trace()
2443	image = getattr(doc,key,None)
2444	if not image or not hasattr(image,"data"):
2445	continue
2446	if not created:
2447	if not os.path.exists(images_dir):
2448	os.mkdir(images_dir)
2449	created = True
2450	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2451	open(filename,"wb").write(str(image.data))
2452	###)
2453
2454	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2455	def movePassportToFS(self,student_id="O738726"):
2456	"move the passports to the filesystem"
2457	images_dir = os.path.join("%s" % i_home,'passports')
2458	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2459	stool = getToolByName(self, 'portal_schemas')
2460	schemas = ['student_application',
2461	#'student_clearance',
2462	]
2463	created = False
2464	for schema_id in schemas:
2465	schema = stool._getOb(schema_id)
2466	object = getattr(student_folder,schema_id[len('student_'):],None)
2467	if object is None:
2468	continue
2469	doc = object.getContent()
2470	for key in schema.keys():
2471	if schema[key].meta_type != "CPS Image Field":
2472	continue
2473	#import pdb;pdb.set_trace()
2474	image = getattr(doc,key)
2475	if not hasattr(image,"data"):
2476	continue
2477	if not created:
2478	if not os.path.exists(images_dir):
2479	os.mkdir(images_dir)
2480	created = True
2481	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2482	open(filename,"wb").write(str(image.data))
2483	###)
2484
2485	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: