Context navigation

WAeUPTool.py @ 3133

Last change on this file since 3133 was 3133, checked in by joachim, 17 years ago

implement new functionality im importData:

for verdicts and course_results now records, which return an "id" and

"error" in their mass_edit-function are collected into files named:

verdict_pending.csv for verdicts
or course_result_pending.csv for course_results

These files are checked first, when importing new data, and importable
records are imported and removed from this file.

There are also the options:

Pending Verdicts only
Pending CourseResults? only

to import the pending records only.

The input file is irrelevant then.

Property svn:keywords set to Id

File size: 103.8 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3133 2008-02-09 09:15:05Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	import DateTime,time
46	import logging
47	import transaction
48	import csv,re,os,sys
49	from shutil import copy2
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	p_home = package_home(globals())
53	i_home = INSTANCE_HOME
54	images_base = os.path.join(i_home,"images")
55
56	def getImagesDir(student_id):
57	return os.path.join("%s" % images_base,student_id[0],student_id)
58
59	def getObject(object,name):
60	if object.hasObject(name):
61	return getattr(object,name)
62	return None
63
64
65	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66	"""WAeUP tool"""
67
68	id = 'waeup_tool'
69	meta_type = 'WAeUP Tool'
70	_actions = ()
71	security = ClassSecurityInfo()
72	security.declareObjectProtected(View)
73	manage_options = ( ActionProviderBase.manage_options
74	+ SimpleItem.manage_options
75	)
76
77	security.declareProtected(View,'re_split') ###(
78	def re_split(self,split_string,string):
79	return re.split(split_string,string)
80	###)
81
82	security.declareProtected(View,'difference') ###(
83	def difference(self,l1,l2):
84	return set(l1).difference(set(l2))
85	###)
86
87	def rwrite(self,s): ###(
88	response = self.REQUEST.RESPONSE
89	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
90	response.write("%s<br />\r\n" % s)
91	###)
92
93	def addtodict(self,d,key,item): ###(
94	d[key].append(item)
95	return d[key]
96	###)
97
98	def sleep(self,secs): ###(
99	"sleep"
100	import time
101	time.sleep(secs)
102	return
103
104	###)
105
106	security.declareProtected(View,'updateRoleMappingsFor') ###(
107	def updateRoleMappingsFor(self,wf_definition,ob):
108	"do so for public"
109	wf_def = getattr(self.portal_workflow,wf_definition)
110	wf_def.updateRoleMappingsFor(ob)
111	###)
112
113	security.declareProtected(View,'getStatesLgas') ###(
114	def getStatesLgas(self):
115	"""return lga info"""
116	voc = getattr(self.portal_vocabularies,'local_gov_areas')
117	states = []
118	lgas = []
119	d = {}
120	wd = {}
121	for k,v in voc.items():
122	parts = v.split(' / ')
123	if len(parts) == 1:
124	state = parts[0].lower()
125	lga = ""
126	elif len(parts) == 2:
127	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
128	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
129	else:
130	continue
131	if state not in states:
132	states.append(state)
133	if lga not in lgas:
134	lgas.append(lga)
135	words = re.split('[^a-zA-Z0-9/]',k)
136	words.sort()
137	wd[k] = words
138	d[k] = v
139	mapping = {}
140	mapping['word_dict'] = wd
141	mapping['lga_dict'] = d
142	mapping['states'] = states
143	mapping['lgas'] = lgas
144	return mapping
145	###)
146
147	security.declareProtected(View,'findLga') ###(
148	def findLga(self,words,words_dict):
149	words = re.split('[^a-zA-Z0-9/]',words)
150	lga_words = []
151	for word in words:
152	if word:
153	lga_words += word.strip().lower(),
154	lga_words.sort()
155	state_lga = ''
156	while not state_lga:
157	for k,l in words_dict.items():
158	if lga_words == l:
159	state_lga = k
160	break
161	break
162	return state_lga
163	###)
164	security.declareProtected(View,'getAccessInfo') ###(
165	def getAccessInfo(self,context):
166	"return a dict with access_info"
167	logger = logging.getLogger('WAeUPTool.getAccessInfo')
168	mtool = self.portal_membership
169	member = mtool.getAuthenticatedMember()
170	member_id = str(member)
171	info = {}
172	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
173	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
174	is_staff = info['is_staff'] = not is_anonymous and not is_student
175	roles = member.getRolesInContext(context)
176	is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
177	"SectionManager" in roles or
178	"Manager" in roles)
179	is_allowed = info['is_allowed'] = not is_anonymous
180	requested_id = context.getStudentId()
181	student_id = None
182	if is_allowed:
183	if not is_student and requested_id:
184	student_id = requested_id
185	elif is_anonymous or member_id != requested_id:
186	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
187	else:
188	student_id = member_id
189	info['student_id'] = student_id
190	return info
191	###)
192
193	security.declareProtected(ModifyPortalContent,'openLog') ###(
194	def openLog(self,name):
195	"""open a log file"""
196	version = 1
197	path = "%s/log/%s_%d.log" % (i_home,name,version)
198	while os.path.exists(path):
199	version += 1
200	path = "%s/log/%s_%d.log" % (i_home,name,version)
201	log = open(path,"w")
202	return log
203	###)
204
205	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
206	def bypassQueueCatalog(self,enable=True):
207	"""bypass the QueueCatalog by setting all indexes to process imediate,
208	if enable is True (default) the old settings are restored
209	"""
210
211	###)
212
213	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
214	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
215	"""measure Object access Time"""
216	import random
217	if hasattr(self,'portal_catalog_real'):
218	aq_portal = self.portal_catalog_real.evalAdvancedQuery
219	else:
220	aq_portal = self.portal_catalog.evalAdvancedQuery
221	nr_pts = int(nr_pts)
222	probe = int(probe)
223	intervall = probe/10
224	objects = ("application","clearance","personal")
225	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
226	#i = random.randrange(num_objects)
227	count = 0
228	found = 0
229	not_found = 0
230	t_found = 0
231	t_not_found = 0
232	time_found = time_not_found = 0.0
233	t_time_found = t_time_not_found = 0.0
234	accessed = []
235	t_min = 1000
236	t_max = 0
237	#import pdb;pdb.set_trace()
238	students = self.portal_catalog(portal_type="Student")
239	num_students = len(students)
240	if method == "d":
241	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
242	res = aq_portal(query)
243	brains = {}
244	for r in res:
245	sid = r.relative_path.split('/')[-2]
246	if brains.has_key(sid):
247	brains[sid][r.portal_type] = r
248	else:
249	brains[sid] = {r.portal_type : r}
250	brains_list = brains.keys()
251	num_objects = len(brains_list)
252	else:
253	num_objects = num_students
254	print "="*40
255	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
256	probe,
257	nr_pts,
258	num_objects)
259	print "nr found/not time found/not min/max"
260	elapse = time.time()
261	i_elapse = time.time()
262	c_elapse = time.clock()
263	for c in range(1,probe + 1):
264	i = random.randrange(num_objects)
265	if method in ('a','b','c'):
266	student_brain = students[i]
267	elif method == "d":
268	#import pdb;pdb.set_trace()
269	student_brain = brains[brains_list[i]]
270	if method == "c":
271	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
272	res = aq_portal(query)
273	this_portal_types = [r.portal_type for r in res]
274	for i in range(nr_pts):
275	oid = objects[i]
276	if method == "a":
277	try:
278	student_path = student_brain.getPath()
279	path = "%s/%s" % (student_path,oid)
280	doc = self.unrestrictedTraverse(path).getContent()
281	found += 1
282	i_time = time.time() - i_elapse
283	time_found += i_time
284	except:
285	not_found += 1
286	i_time = time.time() - i_elapse
287	time_not_found += i_time
288	pass
289	elif method == "b":
290	try:
291	student_object = student_brain.getObject()
292	doc = getattr(student_object,oid).getContent()
293	found += 1
294	i_time = time.time() - i_elapse
295	time_found += i_time
296	except:
297	i_time = time.time() - i_elapse
298	time_not_found += i_time
299	not_found += 1
300	pass
301	elif method == "c":
302	if portal_types[i] in this_portal_types:
303	found += 1
304	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
305	i_time = time.time() - i_elapse
306	time_found += i_time
307	else:
308	not_found += 1
309	i_time = time.time() - i_elapse
310	time_not_found += i_time
311	elif method == "d":
312	if student_brain.has_key(portal_types[i]):
313	found += 1
314	doc = student_brain[portal_types[i]].getObject().getContent()
315	i_time = time.time() - i_elapse
316	time_found += i_time
317	else:
318	not_found += 1
319	i_time = time.time() - i_elapse
320	time_not_found += i_time
321	i_elapse = time.time()
322	if c and (c % intervall == 0):
323	#i_time = time.time() - i_elapse
324	t_per = 0.0
325	if found:
326	t_per = time_found/found
327	if t_per > t_max:
328	t_max = t_per
329	if t_per > 0.0 and t_per < t_min:
330	t_min = t_per
331	itf = 0.0
332	if found:
333	itf = time_found/found
334	itnf = 0.0
335	if not_found :
336	itnf = time_not_found / not_found
337	interval_time = time_found + time_not_found
338	s = "%(c)d: %(found)d/%(not_found)d " % vars()
339	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
340	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
341	print s
342	t_found += found
343	t_not_found += not_found
344	t_time_found += time_found
345	t_time_not_found += time_not_found
346	time_found = time_not_found = 0.0
347	found = not_found = 0
348	# t_found += found
349	# t_not_found += not_found
350	elapse = time.time() - elapse
351	itf = 0.0
352	if t_found:
353	itf = t_time_found/t_found
354	itnf = 0.0
355	if t_not_found:
356	itnf = t_time_not_found / t_not_found
357	#c_elapse = time.clock() - c_elapse
358	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
359	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
360	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
361	print "-"*40
362	print s
363	rel_found = float(t_found)/probe
364	rel_not_found = float(t_not_found)/probe
365	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
366	print estimated_total_time
367	###)
368
369	security.declareProtected(ModifyPortalContent,'writeLog') ###(
370	def writeLog(self,logfile,s):
371	"""write to the log file"""
372	logfile.write(s)
373
374	###)
375
376	def generateStudentId(self,letter): ###(
377	import random
378	r = random
379	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
380	if letter == '?':
381	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
382	sid = "%c%d" % (letter,r.randint(99999,1000000))
383	students = self.portal_url.getPortalObject().campus.students
384	## while hasattr(students, sid):
385	## sid = "%c%d" % (letter,r.randint(99999,1000000))
386	while self.students_catalog(id = sid):
387	sid = "%c%d" % (letter,r.randint(99999,1000000))
388	return sid
389	###)
390
391	def generatePassword(self,s=None): ###(
392	import random
393	r = random
394	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
395	if s is None:
396	s = 'abcdefghklmnpqrstuvwxy23456789'
397	pw = ''
398	while len(pw) < 6:
399	pw += r.choice(s)
400	return pw
401	###)
402
403	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
404	def dumpSchoolfeePayments(self):
405	"dump paid schoolfees"
406	mtool = self.portal_membership
407	member = mtool.getAuthenticatedMember()
408	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
409	aq_student = self.students_catalog.evalAdvancedQuery
410	query = In('review_state',('schoolfee_paid',
411	'courses_registered',
412	'courses_validated',
413	))
414	res = aq_student(query)
415	#import pdb;pdb.set_trace()
416	l = []
417	logger.info("start for %d" % len(res))
418	count = 1
419	log_after = 100
420	for student in res:
421	if not count % log_after:
422	logger.info("processed %d total %d" % (log_after,count))
423	count += 1
424	fee_dict =self.getSchoolFee(student)
425	fulltime = student.mode.endswith('_ft')
426	d = {}
427	d['student_id'] = student.id
428	d['name'] = student.name
429	d['amount'] = fee_dict.get(new_returning)
430	l += d,
431	csv_name = self.dumpListToCSV(l,'payments')
432	logger.info('%s dumped payments to %s' % (member,export_file))
433	###)
434
435	security.declarePublic('dumpListToCSV') ###(
436	def dumpListToCSV(self,l,filename,fields=None):
437	"""dump a list of dicts to a CSV file"""
438	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
439	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
440	if fields is None:
441	fields = l[0].keys()
442	headline = ','.join(fields)
443	out = open(export_file,"wb")
444	out.write(headline +'\n')
445	out.close()
446	out = open(export_file,"a")
447	csv_writer = csv.DictWriter(out,fields,)
448	csv_writer.writerows(l)
449	return export_file
450	###)
451
452	security.declareProtected(ManagePortal, 'listMembers') ###(
453	def listMembers(self):
454	"list all members"
455	mtool = self.portal_membership
456	member = mtool.getAuthenticatedMember()
457	logger = logging.getLogger('WAeUPTool.listMembers')
458	if str(member) not in ('admin','joachim'):
459	logger.info('%s tried to list members' % (member))
460	return None
461	members = self.portal_directories.members
462	all = members.listEntryIdsAndTitles()
463	l = []
464	for user_id,name in all:
465	d = {}
466	d['user_id'] = user_id
467	d['name'] = name
468	d['pw'] = getattr(getattr(members,user_id),'password')
469	d['email'] = getattr(getattr(members,user_id),'email')
470	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
471	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
472	l += d,
473	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
474	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
475	logger.info('%s dumped memberlist to %s' % (member,export_file))
476	fields = l[0].keys()
477	headline = ','.join(fields)
478	out = open(export_file,"wb")
479	out.write(headline +'\n')
480	out.close()
481	out = open(export_file,"a")
482	csv_writer = csv.DictWriter(out,fields,)
483	csv_writer.writerows(l)
484	###)
485
486	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
487	def removeDeletedDocIds(self, max=1000):
488	"""
489	remove deleted docids from repository commit after max
490	"""
491	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
492	repository = getToolByName(self, 'portal_repository')
493	pxtool = getToolByName(self, 'portal_proxies')
494	logger.info('start')
495	pxtool_infos = pxtool.getRevisionsUsed()
496	logger.info('found %d used revisions ' % (len(pxtool_infos)))
497
498	nb_revs = 0
499	docids_d = {} # all docids
500	unused_docids_d = {} # all docids that are unused
501	ids_unused_revs_docids = [] # ids for revs of unused docids
502	unused_ids = [] # ids for unused revs
503	total = 0
504	idlist = repository.objectIds()
505	to_delete = 0
506	found = False
507	for id in idlist:
508	docid, rev = repository._splitId(id)
509	if docid is None:
510	logger.info("invalid doc_id %s" % docid)
511	continue
512	nb_revs += 1
513	if not pxtool_infos.has_key(docid):
514	found = True
515	to_delete += 1
516	unused_ids.append(id)
517	elif not pxtool_infos[docid].has_key(rev):
518	found = True
519	to_delete += 1
520	unused_ids.append(id)
521	if found and not to_delete % max:
522	found = False
523	#import pdb;pdb.set_trace()
524	repository.manage_delObjects(unused_ids)
525	transaction.commit()
526	logger.info('removed %d total %d unused docids ' % (max,to_delete))
527	else:
528	if unused_ids:
529	repository.manage_delObjects(unused_ids)
530	transaction.commit()
531	logger.info('finished removing %d unused docids ' % (to_delete))
532
533
534	###)
535
536	security.declareProtected(View,'getCredential') ###(
537	def getCredential(self,student_id):
538	student_entry = getattr(self.portal_directories.students,student_id,None)
539	if not self.isStaff():
540	mtool = self.portal_membership
541	member = mtool.getAuthenticatedMember()
542	logger = logging.getLogger('WAeUPTool.getCredential')
543	logger.info('%s tried to access password of %s' % (member,student_id))
544	return None
545	if student_entry is None:
546	return None
547	return getattr(student_entry,"password","not set")
548	###)
549
550	security.declarePublic('checkPassword') ###(
551	def checkPassword(self,student_id,password):
552	student_entry = getattr(self.portal_directories.students,student_id,None)
553	if student_entry is None:
554	return False
555	return getattr(student_entry,"password","not set") == password
556	###)
557
558	security.declarePublic('checkGenericPassword') ###(
559	def checkGenericPassword(self,member_id):
560	member_entry = getattr(self.portal_directories.members,member_id,None)
561	if member_entry is None:
562	return False
563	ltool = getToolByName(self, 'portal_layouts')
564	unsecure_words = ltool._getOb('members')['w__password'].check_words
565	password = getattr(member_entry,"password","not set")
566	is_unsecure = password in unsecure_words
567	if is_unsecure:
568	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
569	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
570	return is_unsecure
571	###)
572
573	security.declareProtected(ModifyPortalContent,'editPassword') ###(
574	def editPassword(self,student_id,password):
575	"edit a student password"
576	student_entry = getattr(self.portal_directories.students,student_id,None)
577	if student_entry is None:
578	return
579	setattr(student_entry,'password',password)
580	###)
581
582	security.declareProtected(ModifyPortalContent,'doCommit') ###(
583	def doCommit(self,logger=None):
584	"commit some transactions"
585	transaction.commit()
586	###)
587
588	security.declarePublic('loadStudentFoto') ###(
589	def loadStudentFoto(self,student,filename,folder):
590	"return a student passport picture"
591	#import pdb;pdb.set_trace()
592	picture ="%s/import/%s/%s" % (i_home,folder,filename)
593	student_id = student.getId()
594	images_dir = getImagesDir(student_id)
595	if not os.path.exists(images_dir):
596	os.mkdir(images_dir)
597	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
598	for extension in ('.jpg','.JPG'):
599	fullname = "%(picture)s%(extension)s" % vars()
600	if os.path.exists(fullname):
601	copy2(fullname,image_name)
602	return "successfully copied passport picture"
603	return "passport picture not found: %s.jpg or .JPG" % picture
604	###)
605
606	def old____loadStudentFoto(self,student,filename,folder): ###(
607	"return a student passport picture"
608	app = student.application
609	app_doc = app.getContent()
610	#clear = student.clearance
611	#clear_doc = clear.getContent()
612	#matric_no = clear_doc.matric_no.upper()
613	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
614	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
615	#import pdb;pdb.set_trace()
616	if os.path.exists(picture1):
617	file = open(picture1)
618	elif os.path.exists(picture2):
619	file = open(picture2)
620	else:
621	return "passport picture not found %s" % picture1
622	reopened = False
623	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
624	self.portal_workflow.doActionFor(app,'open')
625	reopened = True
626	outfile = file.read()
627	app_doc.manage_addFile('passport',
628	file=outfile,
629	title="%s.jpg" % filename)
630	if reopened:
631	self.portal_workflow.doActionFor(app,'close')
632	return "successfully loaded passport picture"
633	###)
634
635	security.declareProtected(ModifyPortalContent,'createOne') ###(
636	def createOne(self,students_folder,student_brain,letter,commit=False):
637	sid = self.waeup_tool.generateStudentId(letter)
638	students_folder.invokeFactory('Student', sid)
639	student = getattr(students_folder,sid)
640	self.portal_workflow.doActionFor(student,'return')
641	student.manage_setLocalRoles(sid, ['Owner',])
642	matric_no = student_brain.matric_no
643	jamb_reg_no = student_brain.Entryregno
644	self.students_catalog.addRecord(id = sid,
645	matric_no = matric_no,
646	jamb_reg_no = jamb_reg_no,
647	sex = student_brain.Sex == "F",
648	name = "%s %s %s" % (student_brain.Firstname,
649	student_brain.Middlename,
650	student_brain.Lastname)
651	)
652	if commit:
653	transaction.commit()
654	return sid,jamb_reg_no
655	###)
656
657	security.declareProtected(ModifyPortalContent,'addStudent') ###(
658	def addStudent(self,dict):
659	students_folder = self.portal_url.getPortalObject().campus.students
660	sid = self.waeup_tool.generateStudentId('?')
661	students_folder.invokeFactory('Student', sid)
662	student_obj = getattr(students_folder,sid)
663	f2t = self.field2types_student
664	#from pdb import set_trace; set_trace()
665	d = {}
666	#d['jamb_sex'] = 'M'
667	#if dict.get('sex'):
668	# d['jamb_sex'] = 'F'
669
670	entry_session = dict.get('entry_session')
671	if entry_session == self.getSessionId()[0]:
672	wfaction = 'admit'
673	wft = 'wf_transition_admit'
674	password = None
675	else:
676	wfaction = 'return'
677	wft = 'wf_transition_return'
678	password = self.generatePassword()
679	self.makeStudentMember(sid,password)
680
681	for pt in f2t.keys():
682	student_obj.invokeFactory(pt,f2t[pt]['id'])
683	sub_obj = getattr(student_obj,f2t[pt]['id'])
684	sub_doc = sub_obj.getContent()
685	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
686	d['Title'] = f2t[pt]['title']
687	for field in f2t[pt]['fields']:
688	d[field] = dict.get(field,'')
689	sub_doc.edit(mapping = d)
690	new_state = f2t[pt][wft]
691	if new_state != "remain":
692	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
693	self.portal_workflow.doActionFor(student_obj,wfaction)
694	student_obj.manage_setLocalRoles(sid, ['Owner',])
695	return sid,password
696	###)
697
698	security.declarePublic('getCertificateBrain') ###(
699	def getCertificateBrain(self,cert_id):
700	"do it"
701	res = ZCatalog.searchResults(self.portal_catalog_real,
702	{'portal_type':"Certificate",
703	'id': cert_id})
704	if res:
705	return res[0]
706	return None
707	###)
708
709	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
710	def get_csv_filenames(self):
711	"do it"
712	files = [file for file in os.listdir("%s/import/" % (i_home))
713	if file.endswith('.csv') and file.find('imported') == -1]
714	return files
715	###)
716
717	security.declarePublic('findStudentByMatricelNo') ###(
718	def findStudentByMatricelNo(self,matric_no):
719	"do it"
720	res = ZCatalog.searchResults(self.portal_catalog_real,
721	{'portal_type':"StudentClearance",
722	'SearchableText': matric_no})
723	if res:
724	return res[0]
725	return None
726	###)
727
728	security.declarePublic('makeStudentMember') ###(
729	def makeStudentMember(self,sid,password='uNsEt'):
730	"""make the student a member"""
731	membership = self.portal_membership
732	membership.addMember(sid,
733	password ,
734	roles=('Member',
735	'Student',
736	),
737	domains='',
738	properties = {'memberareaCreationFlag': False,
739	'homeless': True},)
740	member = membership.getMemberById(sid)
741	self.portal_registration.afterAdd(member, sid, password, None)
742	#self.manage_setLocalRoles(sid, ['Owner',])
743	###)
744
745	security.declareProtected(View,'makeStudentData') ###(
746	def makeStudentData(self,student_id,email=None,phone_nr=None):
747	"create Datastructure for a returning Student"
748	#import pdb;pdb.set_trace()
749	logger = logging.getLogger('WAeUPTool.makeStudentData')
750	students_folder = self.portal_url.getPortalObject().campus.students
751	#res = self.students_catalog(id=student_id)
752	#if res:
753	# st = res[0]
754	#res = self.returning_import(matric_no = st.matric_no)
755	res = self.returning_import(id = student_id)
756	if res:
757	student = res[0]
758	else:
759	logger.info('Id %s not found in returning_import' % student_id)
760	return
761	logger.info('%s creates data structure' % student_id)
762	s_results = self.results_import(matric_no = student.matric_no)
763	if s_results:
764	lnr = self.getLevelFromResultsCosCode(s_results)
765	level = "%d00" % lnr
766	verdict,eligible = self.getVerdict(s_results[0].Verdict)
767	#if eligible:
768	# level = "%d00" % (lnr + 1)
769	else:
770	logger.info('matric_no %s not found in results_import' % student.matric_no)
771	level = ''
772	verdict = ''
773	#student should not be allowed to perform this transition
774	#wftool = self.portal_workflow
775	#wftool.doActionFor(student,'return')
776	certcode_org = student.Coursemajorcode
777	certcode = makeCertificateCode(certcode_org)
778	certificate_brain = self.getCertificateBrain(certcode)
779	if not certificate_brain:
780	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
781	logger.info(em)
782	matric_no = student.matric_no
783	sid = student_id
784	student_obj = getattr(students_folder,sid)
785	if not getattr(student_obj,'application'):
786	student_obj.invokeFactory('StudentApplication','application')
787	application = student_obj.application
788	self.portal_workflow.doActionFor(application,'open',dest_container=application)
789	da = {'Title': 'Application Data'}
790	student_obj.invokeFactory('StudentPersonal','personal')
791	da['jamb_reg_no'] = student.Entryregno
792	em = self.getEntryMode(student.Entryregno)
793	da['entry_mode'] = em
794	personal = student_obj.personal
795	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
796	dp = {'Title': 'Personal Data'}
797	student_obj.invokeFactory('StudentClearance','clearance')
798	clearance = student_obj.clearance
799	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
800	dc = {'Title': 'Clearance/Eligibility Record'}
801	dc['matric_no'] = matric_no
802	state = student.State
803	lga = student.LGA
804	if state and lga:
805	lga = state + ' / ' + lga
806	else:
807	lga = "None"
808	da['jamb_lga'] = dc['lga'] = lga
809	da['app_email'] = dp['email'] = email
810	da['app_mobile'] = dp['phone'] = phone_nr
811	dp['firstname'] = student.Firstname
812	dp['middlename'] = student.Middlename
813	dp['lastname'] = student.Lastname
814	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
815	da['jamb_sex'] = student.Sex
816	dp['sex'] = student.Sex == 'F'
817	dp['perm_address'] = student.Permanent_Address
818	application.getContent().edit(mapping=da)
819	self.portal_workflow.doActionFor(application,'close',dest_container=application)
820	personal.getContent().edit(mapping=dp)
821	clearance.getContent().edit(mapping=dc)
822	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
823	#
824	# Study Course
825	#
826	student_obj.invokeFactory('StudentStudyCourse','study_course')
827	studycourse = student_obj.study_course
828	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
829	dsc = {}
830	dsc['study_course'] = certcode
831	dsc['current_level'] = level
832	dsc['current_verdict'] = verdict
833	dsc['current_mode'] = em
834	dsc['current_session'] = '05'
835	studycourse.getContent().edit(mapping=dsc)
836	#
837	# Level
838	#
839	## l = getattr(studycourse,level,None)
840	## if l is None:
841	## studycourse.invokeFactory('StudentStudyLevel', level)
842	## l = getattr(studycourse, level)
843	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
844	## l.getContent().edit(mapping={'Title': "Level %s" % level})
845	###)
846
847	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
848	def admitOneStudent(self,brain,entry_session,pin_password):
849	"create Datastructure for an admitted Student"
850	#import pdb;pdb.set_trace()
851	if not hasattr(self,"_v_certificates"):
852	self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
853	students_folder = self.portal_url.getPortalObject().campus.students
854	logger = logging.getLogger('WAeUPTool.admitOneStudent')
855	if brain.status != "admitted":
856	logger.info('status of %s is %s' % (brain.reg_no,brain.status))
857	return
858	pin_parts = brain.pin.split('-')
859	if pin_parts and len(pin_parts) != 3:
860	logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
861	return
862	if brain.course_admitted not in self._v_certificates:
863	logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
864	return
865	if brain.sex not in (True,False):
866	logger.info('sex of %s not available' % (brain.reg_no))
867	return
868	student_id = self.generateStudentId('?')
869	students_folder.invokeFactory('Student', student_id)
870	student_object = getattr(students_folder,student_id)
871	if pin_password:
872	password = pin_parts[2]
873	self.makeStudentMember(student_id,password = password)
874	student_object.manage_setLocalRoles(student_id, ['Owner',])
875	#logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
876	#
877	# application
878	#
879	student_object.invokeFactory('StudentApplication','application')
880	application = student_object.application
881	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
882	da = {'Title': 'Application Data'}
883	da['jamb_reg_no'] = brain.reg_no
884
885	sex = 'M'
886	if brain.sex:
887	sex = 'F'
888	da['jamb_sex'] = sex
889	da['jamb_age'] = brain.jamb_age
890	#da['app_ac_pin'] = brain.pin
891	da['jamb_lga'] = brain.jamb_lga
892	da['jamb_state'] = brain.jamb_state
893	da['jamb_score'] = brain.aggregate
894	da['app_email'] = brain.email
895	da['app_mobile'] = brain.phone
896	if brain.entry_mode:
897	da['entry_mode'] = brain.entry_mode
898	elif brain.screening_type == 'pume':
899	da['entry_mode'] = 'ume_ft'
900	elif brain.screening_type == 'pde':
901	da['entry_mode'] = 'de_ft'
902	elif brain.screening_type == 'pce':
903	da['entry_mode'] = 'pce'
904	elif brain.screening_type == 'prence':
905	da['entry_mode'] = 'prence'
906	else:
907	da['entry_mode'] = ''
908	da['entry_session'] = entry_session
909	da['jamb_lastname'] = brain.lastname
910	da['jamb_middlename'] = brain.middlenames # different field names!
911	da['jamb_firstname'] = brain.firstname
912	da['screening_application_date'] = brain.application_date
913	da['date_of_birth'] = brain.date_of_birth
914	da['jamb_first_cos'] = brain.course1
915	da['jamb_second_cos'] = brain.course2
916	da['course3'] = brain.course3
917	da['screening_type'] = brain.screening_type
918	da['screening_score'] = brain.screening_score
919	da['screening_date'] = brain.screening_date
920	da['hq_type'] = brain.hq_type
921	da['hq_grade'] = brain.hq_grade
922	da['aos'] = brain.aos
923
924	application.getContent().edit(mapping=da)
925	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
926	#
927	# personal
928	#
929	student_object.invokeFactory('StudentPersonal','personal')
930	personal = student_object.personal
931	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
932	dp = {'Title': 'Personal Data'}
933	dp['sex'] = brain.sex
934	dp['email'] = brain.email
935	dp['phone'] = brain.phone
936	dp['lastname'] = brain.lastname
937	dp['middlename'] = brain.middlenames # different field names!
938	dp['firstname'] = brain.firstname
939	personal.getContent().edit(mapping=dp)
940	#
941	# clearance
942	#
943	student_object.invokeFactory('StudentClearance','clearance')
944	clearance = student_object.clearance
945	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
946	dc = {'Title': 'Clearance/Eligibility Record'}
947	dc['lga'] = brain.lga
948	dc['birthday'] = brain.date_of_birth
949	clearance.getContent().edit(mapping=dc)
950	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
951	#
952	# study Course
953	#
954	student_object.invokeFactory('StudentStudyCourse','study_course')
955	studycourse = student_object.study_course
956	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
957	dsc = {}
958	dsc['study_course'] = brain.course_admitted
959	dsc['current_verdict'] = ''
960	dsc['current_mode'] = da['entry_mode']
961	if da['entry_mode'].startswith('de'):
962	dsc['current_level'] = '200'
963	elif da['entry_mode'].startswith('pre'):
964	dsc['current_level'] = '000'
965	else:
966	dsc['current_level'] = '100'
967	dsc['current_session'] = entry_session
968	studycourse.getContent().edit(mapping=dsc)
969	#
970	# payments folder
971	student_object.invokeFactory('PaymentsFolder','payments')
972	payments = getattr(student_object,'payments')
973	dpay = {}
974	dpay['Title'] = 'Payments'
975	payments.getContent().edit(mapping=dpay)
976	self.portal_workflow.doActionFor(payments,'open')
977	#
978	# passport foto
979	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
980	brain.screening_type,
981	brain.reg_no)
982	images_dir = getImagesDir(student_id)
983	#images_dir = os.path.join("%s" % images_base,student_id)
984	letter_dir,student_dir = os.path.split(images_dir)
985	if not os.path.exists(letter_dir):
986	os.mkdir(letter_dir)
987	if not os.path.exists(images_dir):
988	os.mkdir(images_dir)
989	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
990	if os.path.exists(app_picture):
991	copy2(app_picture,image_name)
992	else:
993	logger.info('passport of %s/%s not found: %s' % (student_id,
994	brain.reg_no,
995	app_picture))
996
997	return student_id
998	###)
999
1000	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1001	def makeStudentLevel(self,student_id):
1002	"create the StudyLevel for a returning Student"
1003	#import pdb;pdb.set_trace()
1004	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1005	students_folder = self.portal_url.getPortalObject().campus.students
1006	res = self.students_catalog(id=student_id)
1007	if res:
1008	st = res[0]
1009	course = st.course
1010	matric_no = st.matric_no
1011	level = st.level
1012	res = self.results_import(matric_no = matric_no)
1013	if res:
1014	results = res
1015	logger.info('%s creating Level %s' % (student_id,level))
1016	#
1017	# Level
1018	#
1019	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1020	studycourse = getattr(student_obj,"study_course",None)
1021	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1022	l = getattr(studycourse,level,None)
1023	if l is None:
1024	studycourse.invokeFactory('StudentStudyLevel', level)
1025	l = getattr(studycourse, level)
1026	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1027	l.getContent().edit(mapping={'Title': "Level %s" % level})
1028	###)
1029
1030	security.declarePublic('getHallInfo') ###(
1031	def getHallInfo(self,bed):
1032	"""return Hall Info"""
1033	info = {}
1034	hall,block,room,letter = bed.split('_')
1035	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1036	if res and len(res) == 1:
1037	hall_brain = res[0]
1038	hall_doc = hall_brain.getObject().getContent()
1039	else:
1040	return info
1041	info['hall_title'] = hall_brain.Title
1042	info['maintenance_code'] = hall_doc.maintenance_code
1043	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1044	batch_doc = None
1045	for brain in res:
1046	if brain.id.startswith(info['maintenance_code']):
1047	batch_doc = brain.getObject().getContent()
1048	break
1049	if batch_doc is None:
1050	info['maintenance_fee'] = ''
1051	else:
1052	info['maintenance_fee'] = batch_doc.cost
1053	return info
1054	###)
1055
1056	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1057	def removePictureFolder(self,student_id):
1058	"""remove picture_folder by renaming it"""
1059	path = 'images'
1060	picture_path = os.path.join(i_home,path,student_id)
1061	if not os.path.exists(picture_path):
1062	return False
1063	os.rename(picture_path,picture_path + "_removed")
1064	return True
1065	###)
1066
1067	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1068	def restorePictureFolder(self,student_id):
1069	"""restore picture_folder by renaming it"""
1070	path = 'images'
1071	picture_path = os.path.join(i_home,path,student_id)
1072	if not os.path.exists(picture_path + "_removed"):
1073	return False
1074	os.rename(picture_path + "_removed",picture_path)
1075	return True
1076	###)
1077
1078	security.declarePublic('picturesExist') ###(
1079	def picturesExist(self, ids,student_id=None):
1080	"""check if pictures exist in the filesystem"""
1081	if student_id is None:
1082	student_id = self.getStudentId()
1083	if student_id is None:
1084	return False
1085	picture_path = getImagesDir(student_id)
1086	#picture_path = os.path.join(images_base,student_id)
1087	if not os.path.exists(picture_path):
1088	return False
1089	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1090	return set(ids).issubset(set(pictures))
1091	###)
1092
1093	security.declarePublic('picturesList') ###(
1094	def picturesList(self):
1095	"""check if pictures exist in the filesystem"""
1096	path = 'images'
1097	student_id = self.getStudentId()
1098	#picture_path = os.path.join(i_home,path,student_id)
1099	picture_path = getImagesDir(student_id)
1100	if not os.path.exists(picture_path):
1101	return []
1102	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1103	###)
1104
1105	security.declarePublic('showFsPicture') ###(
1106	def showFsPicture(self,path):
1107	"""return a picture from the filesystem"""
1108	#picture_path = os.path.join(i_home,path)
1109	picture_path = os.path.join(images_base,path)
1110	response = self.REQUEST.RESPONSE
1111	#import pdb;pdb.set_trace()
1112	registry = getToolByName(self, 'mimetypes_registry')
1113	mimetype = str(registry.lookupExtension(path.lower()) or
1114	registry.lookupExtension('file.bin'))
1115	if os.path.exists(picture_path):
1116	response.setHeader('Content-type',mimetype)
1117	return open(picture_path).read()
1118	picture_path = os.path.join(i_home,'import',path)
1119	if os.path.exists(picture_path):
1120	return open(picture_path).read()
1121	###)
1122
1123	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1124	def deleteAllCourses(self,department="All"):
1125	''' delete the courses'''
1126	pm = self.portal_membership
1127	member = pm.getAuthenticatedMember()
1128
1129	if str(member) not in ("henrik","joachim"):
1130	return "not possible"
1131	if department == "All":
1132	res = self.portal_catalog({'meta_type': 'Department'})
1133	if len(res) < 1:
1134	return "No Departments found"
1135
1136	deleted = []
1137	for dep in res:
1138	cf = dep.getObject().courses
1139	if cf:
1140	cf.manage_delObjects(ids=cf.objectIds())
1141	deleted.append("deleted Courses in %s" % dep.getId)
1142	return "\r".join(deleted)
1143	###)
1144
1145	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1146	def getLogfileLines(self,filename="event.log",numlines=20):
1147	"""Get last NUMLINES lines of logfile FILENAME.
1148
1149	Return last lines' of a file in the instances logfile directory as
1150	a list. The number of returned lines equals `numlines' or less. If
1151	less than `numlines' lines are available, the whole file ist
1152	returned. If the file can not be opened or some other error
1153	occurs, empty list is returend.
1154	"""
1155	result = []
1156	lines_hit = 0
1157
1158	# We only handle files in instances' log directory...
1159	logpath = os.path.join(i_home, "log")
1160	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1161	if not filename.startswith( logpath ):
1162	# Attempt to access file outside log-dir...
1163	return []
1164
1165	try:
1166	fd = file( filename, "rb" )
1167	except IOError:
1168	return []
1169	if not fd:
1170	return []
1171
1172	if os.linesep == None:
1173	linesep = '\n'
1174	else:
1175	linesep = os.linesep
1176
1177	# Try to find 'numlines' times a lineseparator, searching from end
1178	# and moving to the beginning of file...
1179	fd.seek( 0, 2) # Move to end of file...
1180	while lines_hit < numlines:
1181	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1182	# one step forward...
1183	lines_hit += 1
1184	try:
1185	fd.seek( -2, 1) # Go two bytes back from current pos...
1186	except IOError:
1187	# We cannot go back two bytes. Maybe the file is too small...
1188	break
1189	fd.seek(2,1)
1190
1191	# Read all lines from current position...
1192	result = fd.readlines()
1193	# Remove line endings...
1194	result = [x.strip() for x in result]
1195	fd.close()
1196	return result
1197	###)
1198
1199	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1200	def getCallbacksFromLog(self,filename):
1201	"""fix Online Payment Transactions from Z2.log entries"""
1202	import transaction
1203	import random
1204	from cgi import parse_qs
1205	from urlparse import urlparse
1206	#from pdb import set_trace
1207	wftool = self.portal_workflow
1208	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1209	students_folder = self.portal_url.getPortalObject().campus.students
1210	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1211	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1212	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1213	data = re.compile(s)
1214	start = True
1215	tr_count = 1
1216	total = 0
1217	#name = 'pume_results'
1218	#name = 'epaymentsuccessful_z2log2'
1219	name = filename
1220	no_import = []
1221	imported = []
1222	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1223	try:
1224	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1225	except:
1226	logger.error('Error reading %s' % name)
1227	return
1228	tas = []
1229	for line in transactions:
1230	dict = {}
1231	items = data.search(line)
1232	dict['idict'] = idict = items.groupdict()
1233	#print idict
1234	#from pdb import set_trace;set_trace()
1235	urlparsed = urlparse(idict['get'][4:])
1236	#print urlparsed
1237	path = urlparsed[2].split('/')
1238	dict['student_id'] = student_id = path[8]
1239	dict['payment_id'] = payment_id = path[10]
1240	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1241	tas.append(dict)
1242	tr_count += 1
1243	return tas
1244	###)
1245
1246	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1247	def importOnlinePaymentTransactions(self):
1248	"""load Online Payment Transactions from CSV values"""
1249	import transaction
1250	import random
1251	#from pdb import set_trace
1252	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1253	opt = self.online_payments_import
1254	students_folder = self.portal_url.getPortalObject().campus.students
1255	start = True
1256	tr_count = 1
1257	total = 0
1258	#name = 'pume_results'
1259	name = 'OnlineTransactions'
1260	no_import = []
1261	imported = []
1262	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1263	try:
1264	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1265	except:
1266	logger.error('Error reading %s.csv' % name)
1267	return
1268	for pay_transaction in transactions:
1269	if start:
1270	start = False
1271	logger.info('Start loading from %s.csv' % name)
1272	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1273	no_import.append('%s,"Error"' % s)
1274	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1275	format_error = format + ',"%(Error)s"'
1276	data = {}
1277
1278	# format of the first file sent by Tayo
1279	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1280	#data['student_id'] = student_id = pay_transaction['Payer ID']
1281	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1282	#data['response_code'] = response_code = pay_transaction['Resp Code']
1283	#data['amount'] = amount = pay_transaction['Amount']
1284
1285	# format of the second file sent by Tayo
1286	#data['datetime'] = date = 0
1287	#data['student_id'] = student_id = pay_transaction['Payer ID']
1288	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1289	#data['response_code'] = response_code = '00'
1290	#data['amount'] = amount = pay_transaction['Amount']
1291
1292	# format of the third file sent by Kehinde
1293	data['datetime'] = date = 0
1294	data['student_id'] = student_id = pay_transaction['customer_id']
1295	data['order_id'] = order_id = pay_transaction['merchant_reference']
1296	data['response_code'] = response_code = '00'
1297	data['amount'] = amount = pay_transaction['Amount']
1298
1299	dup = False
1300	if response_code == "12":
1301	continue
1302	try:
1303	opt.addRecord(**data)
1304	except ValueError:
1305	dup = True
1306	#from pdb import set_trace;set_trace()
1307	if dup:
1308	if response_code == "00":
1309	try:
1310	opt.modifyRecord(**data)
1311	except:
1312	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1313	continue
1314	else:
1315	pay_transaction['Error'] = "Duplicate order_id"
1316	no_import.append( format_error % pay_transaction)
1317	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1318	continue
1319	tr_count += 1
1320	if tr_count > 1000:
1321	if len(no_import) > 0:
1322	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1323	'\n'.join(no_import) + '\n')
1324	no_import = []
1325	em = '%d transactions committed\n' % (tr_count)
1326	transaction.commit()
1327	regs = []
1328	logger.info(em)
1329	total += tr_count
1330	tr_count = 0
1331	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1332	'\n'.join(no_import))
1333	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1334	###)
1335
1336	security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1337	def mass_create_faculty(self,mapping):
1338	"create a faculty"
1339	logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1340	academics_folder = self.portal_url.getPortalObject().campus.academics
1341	fid = mapping['code']
1342	if getattr(academics_folder,fid,None) is not None:
1343	return '', "Faculty with ID: %s exists" % fid
1344	logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1345	try:
1346	academics_folder.invokeFactory('Faculty', fid)
1347	except BadRequest,E:
1348	return '', "%s" % E
1349	f = getattr(academics_folder,fid,None)
1350	f.getContent().edit(mapping=mapping)
1351	return fid,''
1352	###)
1353
1354	security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1355	def mass_edit_faculty(self,mapping):
1356	"edit a faculty"
1357	logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1358	academics_folder = self.portal_url.getPortalObject().campus.academics
1359	fid = mapping['code']
1360	f = getattr(academics_folder,fid,None)
1361	if f is None:
1362	return '', "Faculty with ID: %s does not exist" % fid
1363	logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1364	f.getContent().edit(mapping=mapping)
1365	return fid,''
1366	###)
1367
1368	security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1369	def mass_create_department(self,mapping):
1370	"create a department in the correct faculty"
1371	logger = logging.getLogger('WAeUPTool.mass_create_department')
1372	fid = mapping['faculty_code']
1373	if getattr(self,'_v_faculties',None) is None:
1374	res = self.portal_catalog(portal_type = "Faculty")
1375	self._v_faculties = {}
1376	for f in res:
1377	self._v_faculties[f.getId] = f.getObject()
1378	f = self._v_faculties.get(fid,None)
1379	if f is None:
1380	return '', "No Faculty with ID: %s" % fid
1381	else:
1382	did = mapping.get('code')
1383	d = getattr(f,did,None)
1384	if d is None or d.portal_type == "Faculty":
1385	logger.info('Creating Department %(code)s, %(title)s' % mapping)
1386	try:
1387	f.invokeFactory('Department', did)
1388	except BadRequest,E:
1389	return '', "%s" % E
1390	d = getattr(f,did)
1391	d.invokeFactory('CoursesFolder','courses')
1392	courses = getattr(d,'courses')
1393	dict = {'Title': 'Courses'}
1394	courses.getContent().edit(mapping=dict)
1395	d.invokeFactory('CertificatesFolder','certificates')
1396	certificates = getattr(d,'certificates')
1397	dict = {'Title': 'Certificates'}
1398	certificates.getContent().edit(mapping=dict)
1399	d.getContent().edit(mapping=mapping)
1400	return did,''
1401	###)
1402
1403	security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1404	def mass_edit_department(self,mapping):
1405	"create a department in the correct faculty"
1406	logger = logging.getLogger('WAeUPTool.mass_create_department')
1407	academics_folder = self.portal_url.getPortalObject().campus.academics
1408	fid = mapping['faculty_code']
1409	did = mapping.get('code')
1410	try:
1411	d = getattr(getattr(academics_folder,fid),did,None)
1412	except KeyError:
1413	return '', "Department %s or Faculty %s wrong" % (did,fid)
1414	else:
1415	if d is None or d.portal_type == "Faculty":
1416	logger.info('Editing Department %(code)s, %(title)s' % mapping)
1417	d.getContent().edit(mapping=mapping)
1418	return did,''
1419	###)
1420
1421	security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1422	def mass_create_course(self,mapping):
1423	#import pdb;pdb.set_trace()
1424	if getattr(self,'_v_course_list',None) is None:
1425	self._v_course_list = []
1426	if getattr(self,'_v_departments',None) is None:
1427	res = self.portal_catalog(portal_type = "Department")
1428	self._v_department_courses = {}
1429	for d in res:
1430	self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1431	did = mapping['department_code']
1432	d = self._v_department_courses.get(did,None)
1433	if d is None:
1434	return '', "No Department with ID: %s" % did
1435	course_id = mapping.get('code')
1436	if course_id in self._v_course_list:
1437	return '', "Duplicate Course ID: %s" % did
1438	c = getattr(d,course_id,None)
1439	if c is not None:
1440	return '', "Duplicate Course ID: %s" % did
1441	try:
1442	d.invokeFactory('Course', course_id)
1443	except BadRequest,E:
1444	return '', "%s" % E
1445	self._v_course_list.append(course_id)
1446	c = getattr(d,course_id)
1447	c.getContent().edit(mapping=mapping)
1448	return course_id,''
1449	###)
1450
1451	security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1452	def mass_edit_course(self,mapping):
1453	#import pdb;pdb.set_trace()
1454	course_id = mapping.get('code')
1455	res = self.portal_catalog(id=course_id)
1456	if not res:
1457	return '', "No Course with ID: %s" % course_id
1458	c = res[0].getObject()
1459	c.getContent().edit(mapping=mapping)
1460	return course_id,''
1461	###)
1462
1463	security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1464	def mass_create_certificate(self,mapping):
1465	if getattr(self,'_v_certificate_list',None) is None:
1466	self._v_certificate_list = []
1467	if getattr(self,'_v_department_certificates',None) is None:
1468	res = self.portal_catalog(portal_type = "Department")
1469	self._v_department_certificates = {}
1470	for d in res:
1471	self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1472	did = mapping['department_code']
1473	d = self._v_department_certificates.get(did,None)
1474	if d is None:
1475	return '', "No Department with ID: %s" % did
1476	certificate_id = mapping.get('code')
1477	if certificate_id in self._v_certificate_list:
1478	return '', "Duplicate Certificate ID: %s" % did
1479	c = getattr(d,certificate_id,None)
1480	if c is not None:
1481	return '', "Duplicate Certificate ID: %s" % did
1482	try:
1483	d.invokeFactory('Certificate', certificate_id)
1484	except BadRequest,E:
1485	return '', "%s" % E
1486	self._v_certificate_list.append(certificate_id)
1487	c = getattr(d,certificate_id)
1488	c.getContent().edit(mapping=mapping)
1489	return certificate_id,''
1490	###)
1491
1492	security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1493	def mass_edit_certificate(self,mapping):
1494	#import pdb;pdb.set_trace()
1495	certificate_id = mapping.get('code')
1496	res = self.portal_catalog(id=certificate_id)
1497	if not res:
1498	return '', "No Certificate with ID: %s" % certificate_id
1499	c = res[0].getObject()
1500	c.getContent().edit(mapping=mapping)
1501	return certificate_id,''
1502	###)
1503
1504	security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1505	def mass_create_application(self,mapping):
1506	#import pdb;pdb.set_trace()
1507	reg_no = mapping.get('reg_no')
1508	try:
1509	self.applicants_catalog.addRecord(**mapping)
1510	except ValueError:
1511	return '', "applicant record with reg_no %s already exists" % reg_no
1512	return reg_no,''
1513	###)
1514
1515	security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1516	def mass_edit_application(self,mapping):
1517	#import pdb;pdb.set_trace()
1518	reg_no = mapping.get('reg_no')
1519	status = mapping.get('status')
1520	res = self.applicants_catalog(reg_no = reg_no)
1521	if len(res):
1522	if res[0].status == 'created' and status != 'created':
1523	return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1524	elif status == 'created' and res[0].status != 'created':
1525	return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1526	else:
1527	self.applicants_catalog.modifyRecord(**mapping)
1528	return reg_no,''
1529	else:
1530	return '', "applicant record with reg_no %s does not exist" % reg_no
1531
1532	###)
1533
1534	security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1535	def mass_create_course_result(self,mapping):
1536	#import pdb;pdb.set_trace()
1537	students_folder = self.portal_url.getPortalObject().campus.students
1538	if getattr(self,'_v_courses',None) is None:
1539	res = self.courses_catalog()
1540	self._v_courses = {}
1541	for brain in res:
1542	self._v_courses[brain.code] = brain
1543	if getattr(self,'_v_level_created',None) is None:
1544	self._v_level_created = []
1545	course_id = mapping.get('code')
1546	if course_id not in self._v_courses.keys():
1547	return '', "No course with ID: %s" % course_id
1548	id_key = ''
1549	for id_key in ('student_id','matric_no'):
1550	id_field = mapping.get(id_key,None)
1551	if id_field is not None:
1552	student_id = id_field
1553	break
1554	query = Eq(id_key,id_field)
1555	res = self.students_catalog.evalAdvancedQuery(query)
1556	if not res:
1557	return '', "no student with %(id_key)s %(id_field)s" % vars()
1558	student_rec = res[0]
1559	if id_field != "student_id":
1560	mapping['student_id'] = student_rec.id
1561	student_id = student_rec.id
1562	level_id = mapping['level_id']
1563	code = mapping['code']
1564	if student_id not in self._v_level_created:
1565	try:
1566	context = getattr(getattr(students_folder,
1567	"%(student_id)s" % vars()),
1568	'study_course')
1569	except:
1570	return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1571	if level_id not in context.objectIds():
1572	context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1573	level = getattr(context,"%(level_id)s" % vars())
1574	self.portal_workflow.doActionFor(level,'open')
1575	# the session string must not be copied into the level object
1576	current_verdict = getattr(student_rec,'current_verdict','')
1577	current_session = getattr(student_rec,'current_session','')
1578	if current_verdict and student_rec.current_level == level_id:
1579	level.getContent().edit(mapping={'verdict': "%s" %
1580	current_verdict,
1581	'session': "%s" %
1582	current_session,
1583	})
1584	self.portal_workflow.doActionFor(level,'close')
1585	self._v_level_created += student_id,
1586	mapping['key'] = key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1587	for k in ('semester','credits',):
1588	mapping[k] = getattr(self._v_courses[course_id],k)
1589	try:
1590	self.course_results.addRecord(**mapping)
1591	except ValueError:
1592	return '', "course result already exists: %s" % key
1593	return key,''
1594	###)
1595
1596	security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1597	def mass_edit_course_result(self,mapping):
1598	while True:
1599	result_id = mapping.get('id',None)
1600	if result_id is not None:
1601	mapping['key'] = key = result_id
1602	break
1603	for id_key in ('student_id','matric_no'):
1604	id_field = mapping.get(id_key,None)
1605	if id_field is not None:
1606	student_id = id_field
1607	break
1608	query = Eq(id_key,id_field)
1609	res = self.students_catalog.evalAdvancedQuery(query)
1610	if not res:
1611	return '', "No student with %(id_key)s %(id_field)s" % vars()
1612	if id_field != "student_id":
1613	mapping['student_id'] = res[0].id
1614	if not res:
1615	return '', "No student with %(id_key)s %(id_field)s" % vars()
1616	student_rec = res[0]
1617	student_id = student_rec.id
1618	level_id = mapping['level_id']
1619	code = mapping['code']
1620	mapping['key'] = key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1621	break
1622	try:
1623	self.course_results.modifyRecord(**mapping)
1624	except KeyError:
1625	return key, "No course result to edit: %s" % key
1626	return key,''
1627	###)
1628
1629	security.declareProtected(ModifyPortalContent,'mass_delete_course_result') ###(
1630	def mass_delete_course_result(self,mapping):
1631	id_key = ''
1632	for id_key in ('student_id','matric_no'):
1633	id_field = mapping.get(id_key,None)
1634	if id_field is not None:
1635	student_id = id_field
1636	break
1637	query = Eq(id_key,id_field)
1638	res = self.students_catalog.evalAdvancedQuery(query)
1639	if not res:
1640	return '', "No student with %(id_field)s: %(id_key)s" % vars()
1641	if id_field != "student_id":
1642	mapping['student_id'] = res[0].id
1643	if not res:
1644	return '', "no student with %(id_key)s %(id_field)s" % vars()
1645	student_rec = res[0]
1646	student_id = student_rec.id
1647	level_id = mapping['level_id']
1648	code = mapping['code']
1649	key = "%(student_id)s\|%(level_id)s\|%(code)s" % vars()
1650	# for k in ('semester',):
1651	# mapping[k] = getattr(self._v_courses[course_id],k)
1652	if self.course_results.getRecordByKey(key) is None:
1653	return '', "no course-result with %(key)s" % vars()
1654	self.course_results.deleteRecord(key)
1655	return key,''
1656	###)
1657
1658	security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1659	def mass_create_certificate_course(self,mapping):
1660	if getattr(self,'_v_courses',None) is None:
1661	res = self.courses_catalog()
1662	self._v_courses= [course.code for course in res]
1663	if getattr(self,'_v_ceritficates',None) is None:
1664	res = self.portal_catalog(portal_type = "Certificate")
1665	self._v_certificates = {}
1666	for cert in res:
1667	self._v_certificates[cert.getId] = cert.getObject()
1668	certificate_course_id = mapping.get('code')
1669	if certificate_course_id not in self._v_courses:
1670	return '', "No Course with ID: %s" % certificate_course_id
1671	cert_id = mapping['certificate_code']
1672	cert = self._v_certificates.get(cert_id,None)
1673	if cert is None:
1674	return '', "No Certificate with ID: %s" % cert_id
1675	level_id = mapping.get('level')
1676	level = getattr(cert,level_id,None)
1677	if level is None:
1678	cert.invokeFactory('StudyLevel', level_id)
1679	level = getattr(cert,level_id,None)
1680	elif hasattr(level,certificate_course_id):
1681	return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1682	(certificate_course_id,cert_id,level_id)
1683	level.invokeFactory('CertificateCourse', certificate_course_id)
1684	c = getattr(level,certificate_course_id)
1685	c.getContent().edit(mapping=mapping)
1686	return certificate_course_id,''
1687	###)
1688
1689	field2types_student = { ###(
1690	'StudentApplication':
1691	{'id': 'application',
1692	'title': 'Application Data',
1693	'wf_transition_return': 'close',
1694	'wf_transition_admit': 'remain',
1695	'fields':
1696	('jamb_reg_no',
1697	'entry_mode',
1698	'entry_session',
1699	'jamb_score',
1700	'app_email',
1701	'jamb_age',
1702	'jamb_state',
1703	'jamb_lga',
1704	'jamb_sex',
1705	)
1706	},
1707	#'StudentPume':
1708	# {'id': 'pume',
1709	# 'title': 'Pume Data',
1710	# 'wf_transition_return': 'close',
1711	# 'wf_transition_admit': 'close',
1712	# 'fields':
1713	# ('pume_score',
1714	# )
1715	# },
1716	'StudentClearance':
1717	{'id': 'clearance',
1718	'title': 'Clearance/Eligibility Record',
1719	'wf_transition_return': 'close',
1720	'wf_transition_admit': 'remain',
1721	'fields':
1722	('matric_no',
1723	'nationality',
1724	'lga',
1725	'birthday',
1726	)
1727	},
1728	'StudentPersonal':
1729	{'id': 'personal',
1730	'title': 'Personal Data',
1731	'wf_transition_return': 'open',
1732	'wf_transition_admit': 'remain',
1733	'fields':
1734	('firstname',
1735	'middlename',
1736	'lastname',
1737	'sex',
1738	'email',
1739	'phone',
1740	'perm_address',
1741	)
1742	},
1743	'StudentStudyCourse':
1744	{'id': 'study_course',
1745	'title': 'Study Course',
1746	'wf_transition_return': 'open',
1747	'wf_transition_admit': 'remain',
1748	'fields':
1749	('study_course',
1750	'current_level',
1751	'current_session',
1752	'current_mode',
1753	'current_verdict',
1754	'previous_verdict',
1755	)
1756	},
1757	# 'StudentStudyLevel':
1758	# {'id': 'current_level',
1759	# 'title': '',
1760	# 'wf_transition_return': 'open',
1761	# 'wf_transition_admit': 'remain',
1762	# 'fields':
1763	# ('verdict',
1764	# 'session',
1765	# )
1766	# },
1767	'PaymentsFolder':
1768	{'id': 'payments',
1769	'title': 'Payments',
1770	'wf_transition_return': 'open',
1771	'wf_transition_admit': 'open',
1772	'fields':
1773	()
1774	},
1775	}
1776	###)
1777
1778	security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1779	def mass_create_student(self,mapping):
1780	"create student records due import"
1781	logger = logging.getLogger('WAeUPTool.mass_create_student')
1782	students_folder = self.portal_url.getPortalObject().campus.students
1783	jamb_reg_no = mapping.get('jamb_reg_no',None)
1784	if jamb_reg_no:
1785	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1786	if res:
1787	return '',"jamb_reg_no exists"
1788	matric_no = mapping.get('matric_no',None)
1789	if matric_no:
1790	res = self.students_catalog(matric_no = matric_no)
1791	if res:
1792	return '',"matric_no exists"
1793	sid = self.waeup_tool.generateStudentId('?')
1794	students_folder.invokeFactory('Student', sid)
1795	student_obj = getattr(students_folder,sid)
1796	f2t = self.field2types_student
1797	d = {}
1798	transition = mapping.get('reg_transition','admit')
1799	if transition not in ('admit','return'):
1800	return '',"no valid transition provided"
1801	for pt in f2t.keys():
1802	student_obj.invokeFactory(pt,f2t[pt]['id'])
1803	sub_obj = getattr(student_obj,f2t[pt]['id'])
1804	sub_doc = sub_obj.getContent()
1805	d['Title'] = f2t[pt]['title']
1806	for field in f2t[pt]['fields']:
1807	d[field] = mapping.get(field,'')
1808
1809	if pt == "StudentApplication":
1810	#d['jamb_sex'] = 'M'
1811	#if mapping.get('sex'):
1812	# d['jamb_sex'] = 'F'
1813	d['jamb_firstname'] = mapping.get('firstname',None)
1814	d['jamb_middlename'] = mapping.get('middlename',None)
1815	d['jamb_lastname'] = mapping.get('lastname',None)
1816
1817	# if pt == "StudyCourse":
1818	# for von,zu in (('entry_mode','current_mode'),
1819	# ('entry_session','current_session')):
1820	# if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1821	# d[zu] = mapping[von]
1822	sub_doc.edit(mapping = d)
1823
1824	#import pdb;pdb.set_trace()
1825	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1826	if new_state != "remain":
1827	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1828	self.portal_workflow.doActionFor(student_obj,transition)
1829	student_obj.manage_setLocalRoles(sid, ['Owner',])
1830	return sid,''
1831	###)
1832
1833	security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1834	def mass_edit_student(self,mapping):
1835	wftool = self.portal_workflow
1836	"edit student records due import"
1837	logger = logging.getLogger('WAeUPTool.mass_edit_student')
1838	students_folder = self.portal_url.getPortalObject().campus.students
1839	sid = mapping.get('id',None)
1840	jamb_reg_no = mapping.get('jamb_reg_no',None)
1841	matric_no = mapping.get('matric_no',None)
1842	editable_keys = mapping.keys()
1843	if sid:
1844	res = self.students_catalog(id = sid)
1845	if not res:
1846	return '',"no student with id %s" % sid
1847	if matric_no and res[0].matric_no and\
1848	matric_no != res[0].matric_no:
1849	logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1850	if jamb_reg_no and res[0].jamb_reg_no and\
1851	jamb_reg_no != res[0].jamb_reg_no:
1852	logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1853	elif jamb_reg_no:
1854	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1855	if not res:
1856	return '',"no student with jamb_reg_no %s" % jamb_reg_no
1857	editable_keys.remove('jamb_reg_no')
1858	elif matric_no:
1859	res = self.students_catalog(matric_no = matric_no)
1860	if not res:
1861	return '',"no student with matric_no %s" % matric_no
1862	editable_keys.remove('matric_no')
1863
1864	## included only to change wf state from admitted to returning
1865	#if res[0].review_state not in ('admitted','objection_raised'):
1866	# return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1867	## end inclusion
1868
1869	sid = res[0].id
1870	student_obj = getattr(students_folder,sid)
1871	f2t = self.field2types_student
1872	d = {}
1873	#import pdb;pdb.set_trace()
1874	any_change = False
1875	#special treatment for StudentStudyLevel
1876	d['verdict'] = mapping.get('current_verdict','')
1877	d['session'] = mapping.get('current_session','')
1878	current_level = mapping.get('current_level','')
1879	while d['session'] and d['verdict'] and current_level:
1880	sub_obj = getattr(student_obj,'study_course',None)
1881	if sub_obj is None:
1882	break
1883	level_obj = getattr(sub_obj,current_level,None)
1884	if level_obj is None:
1885	break
1886	any_change = True
1887	level_obj.getContent().edit(mapping = d)
1888	try:
1889	wftool.doActionFor(level_obj,'close')
1890	except:
1891	pass
1892	break
1893	for pt in f2t.keys():
1894	#if pt == "StudentApplication":
1895	# d['jamb_sex'] = 'M'
1896	# if mapping.get('sex'):
1897	# d['jamb_sex'] = 'F'
1898	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1899	if intersect and pt not in ('StudentStudyLevel',):
1900	object_id = f2t[pt]['id']
1901	sub_obj = getattr(student_obj,object_id,None)
1902	if sub_obj is None:
1903	try:
1904	student_obj.invokeFactory(pt,object_id)
1905	except:
1906	continue
1907	sub_obj = getattr(student_obj,object_id)
1908	if f2t[pt]['title'] != '':
1909	d['Title'] = f2t[pt]['title']
1910	sub_doc = sub_obj.getContent()
1911	for field in intersect:
1912	changed = False
1913	if getattr(sub_doc,field,None) != mapping.get(field,''):
1914	any_change = True
1915	changed = True
1916	d[field] = mapping.get(field,'')
1917	if changed:
1918	sub_doc.edit(mapping = d)
1919
1920
1921	## included only to change wf state from admitted to returning
1922	# if res[0].review_state in ('admitted','objection_raised'):
1923	# new_state = f2t[pt]['wf_transition_return']
1924	# sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1925	# if sub_obj and new_state != "remain":
1926	# try:
1927	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1928	# except:
1929	# #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1930	# pass
1931	#if res[0].review_state in ('admitted','objection_raised'):
1932	# wfaction = 'return'
1933	# try:
1934	# self.portal_workflow.doActionFor(student_obj,wfaction)
1935	# logger.info('%s, wf state changed' % sid)
1936	# any_change = True
1937	# except:
1938	# logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1939	# pass
1940	## end inclusion
1941
1942
1943	if any_change:
1944	return sid,''
1945	else:
1946	return sid,'not modified'
1947	###)
1948
1949	security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
1950	def mass_edit_verdict(self,mapping):
1951	wftool = self.portal_workflow
1952	"edit student verdicts"
1953	logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
1954	students_folder = self.portal_url.getPortalObject().campus.students
1955	student_id = mapping.get('id',None)
1956	matric_no = mapping.get('matric_no',None)
1957	editable_keys = mapping.keys()
1958	if student_id:
1959	student_record = self.students_catalog.getRecordByKey(student_id)
1960	if student_record is None:
1961	return '',"no student with id %s" % student_id
1962	if matric_no and student_record.matric_no and\
1963	matric_no != student_record.matric_no:
1964	return '', 'student %s: matric_no %s does not match %s' % (student_record.id,
1965	student_record.matric_no,
1966	matric_no)
1967	elif matric_no:
1968	res = self.students_catalog(matric_no = matric_no)
1969	if not res:
1970	return '',"no student with matric_no %s" % matric_no
1971	student_record = res[0]
1972	editable_keys.remove('matric_no')
1973	else:
1974	return '',"no id or matric_no specified"
1975	student_id = student_record.id
1976	student_obj = getattr(students_folder,student_id)
1977	f2t = self.field2types_student
1978	d = {}
1979	#import pdb;pdb.set_trace()
1980	any_change = False
1981	#special treatment for StudentStudyLevel
1982	current_session = d['session'] = mapping.get('current_session','')
1983	if current_session and student_record.session != current_session:
1984	return student_id,'student %s: imported session %s does not match current_session %s' % (student_id,
1985	current_session,
1986	student_record.session)
1987	current_level = mapping.get('current_level','')
1988	if not current_level.isdigit():
1989	return '','student %s: imported level is empty' % (student_id,)
1990	if current_level and student_record.level != current_level:
1991	return student_id,'student %s: imported level %s does not match current_level %s' % (student_id,
1992	current_level,
1993	student_record.level)
1994	student_review_state = student_record.review_state
1995	if student_review_state not in ('courses_validated','returning'):
1996	return student_id,"student %s in wrong review_state %s" % (student_id, student_review_state)
1997	study_course_obj = getattr(student_obj,'study_course',None)
1998	if study_course_obj is None:
1999	return '','student %s: no study_course object' % student_id
2000	level_obj = getattr(study_course_obj,current_level,None)
2001	if level_obj is None:
2002	return '','student %s: no study_level object for level %s' % (student_id,
2003	current_level)
2004	verdict = d['verdict'] = d['current_verdict'] = mapping.get('current_verdict','')
2005
2006	#if verdict == student_record.verdict:
2007	# return '','student %s: verdict already set to %s' % (student_id,
2008	# verdict)
2009
2010	level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
2011	if level_review_state != "closed":
2012	wftool.doActionFor(level_obj,'close')
2013	# return '','student %s: level %s is not closed' % (student_id,
2014	# current_level)
2015
2016	study_course_obj.getContent().edit(mapping = d)
2017	level_obj.getContent().edit(mapping = d)
2018	if student_review_state != "returning":
2019	wftool.doActionFor(student_obj,'return')
2020	# try:
2021	# wftool.doActionFor(level_obj,'close')
2022	# except:
2023	# pass
2024	return student_id,''
2025	###)
2026
2027	security.declareProtected(ModifyPortalContent,"importData")###(
2028	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
2029	"""load data from CSV values"""
2030	import transaction
2031	import random
2032	member = self.portal_membership.getAuthenticatedMember()
2033	logger = logging.getLogger('WAeUPTool.importData')
2034	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2035	students_folder = self.portal_url.getPortalObject().campus.students
2036	start = True
2037	tr_count = 0
2038	total_imported = 0
2039	total_not_imported = 0
2040	total = 0
2041	pending_only = False
2042	pend_str = '_pending'
2043	if name.endswith(pend_str):
2044	pending_only = True
2045	name = name[:-len(pend_str)]
2046	iname = "import_%s" % name
2047	if name in ('application','course_result',):
2048	commit_after = 2000
2049	else:
2050	commit_after = 100
2051	stool = getToolByName(self, 'portal_schemas')
2052	ltool = getToolByName(self, 'portal_layouts')
2053	schema = stool._getOb(iname)
2054	if schema is None:
2055	em = 'No such schema %s' % iname
2056	logger.error('No such schema %s' % iname)
2057	return em
2058	layout = ltool._getOb(iname)
2059	if layout is None:
2060	em = 'No such layout %s' % iname
2061	logger.error(em)
2062	return em
2063	validators = {}
2064	for widget in layout.keys():
2065	validators[widget] = layout[widget].validate
2066	mode = "create"
2067	if edit:
2068	if filename.endswith('_toDelete'):
2069	mode = "delete"
2070	else:
2071	mode = "edit"
2072	importer_name = "mass_%(mode)s_%(name)s" % vars()
2073	importer = getattr(self, '%s' % importer_name,None)
2074	if importer is None:
2075	em = 'No importer function %s' % importer_name
2076	logger.error(em)
2077	return em
2078	pending_fn = "%s/import/%s_pending.csv" % (i_home,name)
2079	if pending_only:
2080	import_source_fn = pending_fn
2081	imported_fn = "%s/import/%s_pending_imported%s.csv" % (i_home,name,current)
2082	not_imported_fn = "%s/import/%s_pending_not_imported%s.csv" % (i_home,name,current)
2083	attrs = csv.reader(open(pending_fn,"rb")).next()
2084	else:
2085	import_source_fn = "%s/import/%s.csv" % (i_home,filename)
2086	imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
2087	not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
2088	attrs = csv.reader(open(import_source_fn,"rb")).next()
2089	import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
2090	or k.strip() == 'Error')]
2091	diff2schema = set(import_keys).difference(set(schema.keys()))
2092	diff2layout = set(import_keys).difference(set(layout.keys()))
2093	if diff2schema and diff2schema != set(['id',]):
2094	#import pdb;pdb.set_trace()
2095	msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2096	return msg
2097	if mode == "create":
2098	required_keys = [layout.getIdUnprefixed(id)
2099	for id,widget in layout.objectItems()
2100	if widget.is_required]
2101	if not set(required_keys).issubset(set(import_keys)):
2102	diff2import = set(required_keys).difference(set(import_keys))
2103	msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2104	return msg
2105	#
2106	# not_imported
2107	#
2108	not_imported_keys = import_keys + ["Error",]
2109	not_imported_file = open(not_imported_fn,"a")
2110	not_imported_csv_writer = csv.DictWriter(not_imported_file,
2111	not_imported_keys,
2112	extrasaction='ignore')
2113	mapping = dict((k,k) for k in not_imported_keys)
2114	not_imported_csv_writer.writerow(mapping)
2115	not_imported = []
2116	#
2117	# imported
2118	#
2119	imported_keys = import_keys[:]
2120	if 'id' not in imported_keys:
2121	imported_keys.insert(0,"id")
2122	imported_file = open(imported_fn,"a")
2123	imported_csv_writer = csv.DictWriter(imported_file,
2124	imported_keys,
2125	extrasaction='ignore')
2126	mapping = dict((k,k) for k in imported_keys)
2127	imported_csv_writer.writerow(mapping)
2128	imported = []
2129	#
2130	# pending
2131	#
2132	pending_records = {}
2133	imports_pending = 0
2134	total_pending = 0
2135	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2136	format_error = format + ',"%(Error)s"'
2137	#format = '"%(id)s",'+ format
2138	imported = []
2139	if name in ('verdict','course_result',):
2140	#global pending_records
2141	pending_keys = imported_keys[:]
2142	pending_keys += "Error",
2143	pending_records = {}
2144	if os.path.exists(pending_fn):
2145	pending_imports_file = open(pending_fn)
2146	pending_imports = csv.DictReader(pending_imports_file,
2147	fieldnames=pending_keys)
2148	if pending_imports:
2149	pending_records = dict((row['id'],row) for row in pending_imports
2150	if row['id'] != 'id')
2151	pending_imports_file.close()
2152	new_imports_pending = 0
2153	pending_imported = []
2154	for item in pending_records.values():
2155	id,error = importer(item)
2156	if error:
2157	continue
2158	else:
2159	if item.has_key('Error'):
2160	del item['Error']
2161	imported.append(item)
2162	msg = format % item
2163	logger.info("imported from %(pending_fn)s %(msg)s" % vars())
2164	pending_imported += id,
2165	continue
2166	total_imported = len(pending_imported)
2167	logger.info("imported %d pending imports" % len(pending_imported))
2168	for id in pending_imported:
2169	del pending_records[id]
2170	pending_imports_file = open(pending_fn,'w')
2171	pending_csv_writer = csv.DictWriter(pending_imports_file,
2172	pending_keys,
2173	extrasaction='ignore')
2174	mapping = dict((k,k) for k in pending_keys)
2175	pending_csv_writer.writerow(mapping)
2176	#
2177	if pending_only:
2178	items = []
2179	else:
2180	try:
2181	items = csv.DictReader(open(import_source_fn,"rb"))
2182	except:
2183	msg = 'Error reading %s.csv' % filename
2184	logger.error(msg)
2185	return msg
2186	not_imported = []
2187	for item in items:
2188	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2189	if start:
2190	start = False
2191	adapters = [MappingStorageAdapter(schema, item)]
2192	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2193	dm = DataModel(item, adapters,context=self)
2194	ds = DataStructure(data=item,datamodel=dm)
2195	error_string = ""
2196	total += 1
2197	error_count = 0
2198	for k in import_keys:
2199	if not validators[k](ds,mode=mode):
2200	if error_count:
2201	error_string += ' ++ '
2202	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
2203	ds.getErrorMapping(k)))
2204	error_count += 1
2205	if error_string:
2206	item['Error'] = error_string
2207	not_imported.append(item)
2208	total_not_imported += 1
2209	continue
2210	temp_item = item.copy()
2211	temp_item.update(dm)
2212	id,error = importer(temp_item)
2213	if error:
2214	item['Error'] = error
2215	if id and id not in pending_records.keys():
2216	item['id'] = id
2217	pending_records[id] = item
2218	msg = format_error % item
2219	logger.info("%(id)s added to pending %(msg)s" % vars())
2220	new_imports_pending += 1
2221	continue
2222	else:
2223	not_imported.append(item)
2224	total_not_imported += 1
2225	continue
2226	item = temp_item
2227	item['id'] = id
2228	imported.append(item)
2229	tr_count += 1
2230	total_imported += 1
2231	msg = format % item
2232	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
2233	if total and not total % commit_after:
2234	transaction.commit()
2235	if len(imported) > 0:
2236	imported_csv_writer.writerows(imported)
2237	imported = []
2238	if len(not_imported) > 0:
2239	not_imported_csv_writer.writerows(not_imported)
2240	not_imported = []
2241	if len(pending_records) > 0:
2242	pending_csv_writer.writerows(pending_records.values())
2243	total_pending += len(pending_records)
2244	pending_records = {}
2245	msg = '%d transactions committed\n' % (tr_count)
2246	regs = []
2247	logger.info(msg)
2248	tr_count = 0
2249	if len(imported) > 0:
2250	imported_csv_writer.writerows(imported)
2251	if len(not_imported) > 0:
2252	not_imported_csv_writer.writerows(not_imported)
2253	if len(pending_records) > 0:
2254	pending_csv_writer.writerows(pending_records.values())
2255	total_pending += len(pending_records)
2256	msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
2257	msg += "%(total_imported)d imported, %(total_not_imported)d not imported " % vars()
2258	if total_pending:
2259	msg += "%(new_imports_pending)d pending added %(total_pending)d total pending in %(pending_fn)s " % vars()
2260	msg += "(of total %(total)d)" % vars()
2261	logger.info(msg)
2262	return msg
2263	###)
2264
2265	def old_importData(self,filename,name,edit=False,bypass_queue_catalog=False):###(
2266	"""load data from CSV values"""
2267	import transaction
2268	import random
2269
2270	pm = self.portal_membership
2271	member = pm.getAuthenticatedMember()
2272
2273	logger = logging.getLogger('WAeUPTool.importData')
2274	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2275	students_folder = self.portal_url.getPortalObject().campus.students
2276	start = True
2277	tr_count = 0
2278	total_imported = 0
2279	total_not_imported = 0
2280	total = 0
2281	iname = "import_%s" % name
2282	if iname in ('import_application','import_course_result',):
2283	commit_after = 2000
2284	else:
2285	commit_after = 100
2286	stool = getToolByName(self, 'portal_schemas')
2287	ltool = getToolByName(self, 'portal_layouts')
2288	schema = stool._getOb(iname)
2289	if schema is None:
2290	em = 'No such schema %s' % iname
2291	logger.error('No such schema %s' % iname)
2292	return em
2293	layout = ltool._getOb(iname)
2294	if layout is None:
2295	em = 'No such layout %s' % iname
2296	logger.error(em)
2297	return em
2298	validators = {}
2299	for widget in layout.keys():
2300	validators[widget] = layout[widget].validate
2301	mode = "create"
2302	if edit:
2303	if filename.endswith('_toDelete'):
2304	mode = "delete"
2305	else:
2306	mode = "edit"
2307	importer_name = "mass_%(mode)s_%(name)s" % vars()
2308	importer = getattr(self, '%s' % importer_name,None)
2309	if importer is None:
2310	em = 'No importer function %s' % importer_name
2311	logger.error(em)
2312	return em
2313	not_imported = []
2314	imported = []
2315	try:
2316	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
2317	except:
2318	em = 'Error reading %s.csv' % filename
2319	logger.error(em)
2320	return em
2321	for item in items:
2322	#import pdb;pdb.set_trace()
2323	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2324	if start:
2325	start = False
2326	adapters = [MappingStorageAdapter(schema, item)]
2327	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2328	attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
2329	import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
2330	diff2schema = set(import_keys).difference(set(schema.keys()))
2331	diff2layout = set(import_keys).difference(set(layout.keys()))
2332	if diff2schema:
2333	em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2334	return em
2335	if mode == "create":
2336	required_keys = [layout.getIdUnprefixed(id)
2337	for id,widget in layout.objectItems()
2338	if widget.is_required]
2339	if not set(required_keys).issubset(set(import_keys)):
2340	diff2import = set(required_keys).difference(set(import_keys))
2341	em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2342	return em
2343	s = ','.join(['"%s"' % fn for fn in import_keys])
2344	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
2345	s = '"id",' + s
2346	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
2347	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2348	format_error = format + ',"%(Error)s"'
2349	format = '"%(id)s",'+ format
2350
2351	dm = DataModel(item, adapters,context=self)
2352	ds = DataStructure(data=item,datamodel=dm)
2353	error_string = ""
2354	total += 1
2355	error_count = 0
2356	for k in import_keys:
2357	if not validators[k](ds,mode=mode):
2358	if error_count:
2359	error_string += ' ++ '
2360	error_string += "%s: %s" % (k,
2361	self.translation_service(ds.getError(k),
2362	ds.getErrorMapping(k)))
2363	error_count += 1
2364	if not error_string:
2365	temp_item = item.copy()
2366	temp_item.update(dm)
2367	temp_item['id'],error = importer(temp_item)
2368	if error:
2369	error_string += error
2370	else:
2371	item = temp_item
2372	if error_string:
2373	item['Error'] = error_string
2374	not_imported.append(format_error % item)
2375	total_not_imported += 1
2376	else:
2377	em = format % item
2378	imported.append(em)
2379	tr_count += 1
2380	total_imported += 1
2381	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(em)s" % vars())
2382
2383	if total and not total % commit_after:
2384	transaction.commit()
2385	if len(not_imported) > 0:
2386	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2387	'\n'.join(not_imported) + '\n')
2388	not_imported = []
2389	if len(imported) > 0:
2390	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2391	'\n'.join(imported) + '\n')
2392	imported = []
2393	em = '%d transactions committed\n' % (tr_count)
2394	regs = []
2395	logger.info(em)
2396	tr_count = 0
2397	#if total > 100:
2398	# return
2399	if len(imported) > 0:
2400	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2401	'\n'.join(imported))
2402	if len(not_imported) > 0:
2403	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2404	'\n'.join(not_imported))
2405	#if mode.endswith('e'):
2406	# mode = mode[:-1]
2407	em = "Finished importing in %(mode)s mode from %(filename)s.csv: %(total_imported)d imported, %(total_not_imported)d not imported (of total %(total)d)" % vars()
2408	logger.info(em)
2409	return em
2410	###)
2411
2412	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2413	def moveImagesToFS(self,student_id="O738726"):
2414	"move the images to the filesystem"
2415	images_dir = getImagesDir(student_id)
2416	#images_dir = os.path.join("%s" % images_base,student_id)
2417	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2418	stool = getToolByName(self, 'portal_schemas')
2419	schemas = ['student_application',
2420	'student_clearance',
2421	]
2422	created = False
2423	for schema_id in schemas:
2424	schema = stool._getOb(schema_id)
2425	object = getattr(student_folder,schema_id[len('student_'):],None)
2426	if object is None:
2427	continue
2428	doc = object.getContent()
2429	for key in schema.keys():
2430	if schema[key].meta_type != "CPS Image Field":
2431	continue
2432	#import pdb;pdb.set_trace()
2433	image = getattr(doc,key,None)
2434	if not image or not hasattr(image,"data"):
2435	continue
2436	if not created:
2437	if not os.path.exists(images_dir):
2438	os.mkdir(images_dir)
2439	created = True
2440	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2441	open(filename,"wb").write(str(image.data))
2442	###)
2443
2444	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2445	def movePassportToFS(self,student_id="O738726"):
2446	"move the passports to the filesystem"
2447	images_dir = os.path.join("%s" % i_home,'passports')
2448	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2449	stool = getToolByName(self, 'portal_schemas')
2450	schemas = ['student_application',
2451	#'student_clearance',
2452	]
2453	created = False
2454	for schema_id in schemas:
2455	schema = stool._getOb(schema_id)
2456	object = getattr(student_folder,schema_id[len('student_'):],None)
2457	if object is None:
2458	continue
2459	doc = object.getContent()
2460	for key in schema.keys():
2461	if schema[key].meta_type != "CPS Image Field":
2462	continue
2463	#import pdb;pdb.set_trace()
2464	image = getattr(doc,key)
2465	if not hasattr(image,"data"):
2466	continue
2467	if not created:
2468	if not os.path.exists(images_dir):
2469	os.mkdir(images_dir)
2470	created = True
2471	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2472	open(filename,"wb").write(str(image.data))
2473	###)
2474
2475	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3133

Download in other formats: