Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 3962

Last change on this file since 3962 was 3961, checked in by Henrik Bettermann, 16 years ago
Widgets.admitOneStudent: entry_session and session must start with '0' fixSessionKey.py: fix students_catalog, won't be used, because we will import fresh student data with session and entry_session columns. Only 08 students are affected.
Property svn:keywords set to `Id`
File size: 75.5 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3961 2009-02-12 07:35:52Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108	###)
109
110	security.declareProtected(View,'updateRoleMappingsFor') ###(
111	def updateRoleMappingsFor(self,wf_definition,ob):
112	"do so for public"
113	wf_def = getattr(self.portal_workflow,wf_definition)
114	wf_def.updateRoleMappingsFor(ob)
115	###)
116
117	security.declareProtected(View,'getStatesLgas') ###(
118	def getStatesLgas(self):
119	"""return lga info"""
120	voc = getattr(self.portal_vocabularies,'local_gov_areas')
121	states = []
122	lgas = []
123	d = {}
124	wd = {}
125	for k,v in voc.items():
126	parts = v.split(' / ')
127	if len(parts) == 1:
128	state = parts[0].lower()
129	lga = ""
130	elif len(parts) == 2:
131	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133	else:
134	continue
135	if state not in states:
136	states.append(state)
137	if lga not in lgas:
138	lgas.append(lga)
139	words = re.split('[^a-zA-Z0-9/]',k)
140	words.sort()
141	wd[k] = words
142	d[k] = v
143	mapping = {}
144	mapping['word_dict'] = wd
145	mapping['lga_dict'] = d
146	mapping['states'] = states
147	mapping['lgas'] = lgas
148	return mapping
149	###)
150
151	security.declareProtected(View,'findLga') ###(
152	def findLga(self,words,words_dict):
153	words = re.split('[^a-zA-Z0-9/]',words)
154	lga_words = []
155	for word in words:
156	if word:
157	lga_words += word.strip().lower(),
158	lga_words.sort()
159	state_lga = ''
160	while not state_lga:
161	for k,l in words_dict.items():
162	if lga_words == l:
163	state_lga = k
164	break
165	break
166	return state_lga
167	###)
168	security.declareProtected(View,'getAccessInfo') ###(
169	def getAccessInfo(self,context):
170	"return a dict with access_info"
171	logger = logging.getLogger('WAeUPTool.getAccessInfo')
172	mtool = self.portal_membership
173	member = mtool.getAuthenticatedMember()
174	member_id = str(member)
175	info = {}
176	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178	is_staff = info['is_staff'] = not is_anonymous and not is_student
179	roles = member.getRolesInContext(context)
180	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181	"SectionManager" in roles or
182	"Manager" in roles)
183	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184	#is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if not is_anonymous and requested_id:
188	if (is_student and member_id == requested_id) or is_staff:
189	student_id = requested_id
190	else:
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	info['student_id'] = student_id
193	return info
194	###)
195
196	security.declareProtected(ModifyPortalContent,'openLog') ###(
197	def openLog(self,name):
198	"""open a log file"""
199	version = 1
200	path = "%s/log/%s_%d.log" % (i_home,name,version)
201	while os.path.exists(path):
202	version += 1
203	path = "%s/log/%s_%d.log" % (i_home,name,version)
204	log = open(path,"w")
205	return log
206	###)
207
208	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
209	def bypassQueueCatalog(self,enable=True):
210	"""bypass the QueueCatalog by setting all indexes to process imediate,
211	if enable is True (default) the old settings are restored
212	"""
213
214	###)
215
216	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
217	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
218	"""measure Object access Time"""
219	import random
220	if hasattr(self,'portal_catalog_real'):
221	aq_portal = self.portal_catalog_real.evalAdvancedQuery
222	else:
223	aq_portal = self.portal_catalog.evalAdvancedQuery
224	nr_pts = int(nr_pts)
225	probe = int(probe)
226	intervall = probe/10
227	objects = ("application","clearance","personal")
228	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
229	#i = random.randrange(num_objects)
230	count = 0
231	found = 0
232	not_found = 0
233	t_found = 0
234	t_not_found = 0
235	time_found = time_not_found = 0.0
236	t_time_found = t_time_not_found = 0.0
237	accessed = []
238	t_min = 1000
239	t_max = 0
240	#import pdb;pdb.set_trace()
241	students = self.portal_catalog(portal_type="Student")
242	num_students = len(students)
243	if method == "d":
244	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
245	res = aq_portal(query)
246	brains = {}
247	for r in res:
248	sid = r.relative_path.split('/')[-2]
249	if brains.has_key(sid):
250	brains[sid][r.portal_type] = r
251	else:
252	brains[sid] = {r.portal_type : r}
253	brains_list = brains.keys()
254	num_objects = len(brains_list)
255	else:
256	num_objects = num_students
257	print "="*40
258	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
259	probe,
260	nr_pts,
261	num_objects)
262	print "nr found/not time found/not min/max"
263	elapse = time.time()
264	i_elapse = time.time()
265	c_elapse = time.clock()
266	for c in range(1,probe + 1):
267	i = random.randrange(num_objects)
268	if method in ('a','b','c'):
269	student_brain = students[i]
270	elif method == "d":
271	#import pdb;pdb.set_trace()
272	student_brain = brains[brains_list[i]]
273	if method == "c":
274	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
275	res = aq_portal(query)
276	this_portal_types = [r.portal_type for r in res]
277	for i in range(nr_pts):
278	oid = objects[i]
279	if method == "a":
280	try:
281	student_path = student_brain.getPath()
282	path = "%s/%s" % (student_path,oid)
283	doc = self.unrestrictedTraverse(path).getContent()
284	found += 1
285	i_time = time.time() - i_elapse
286	time_found += i_time
287	except:
288	not_found += 1
289	i_time = time.time() - i_elapse
290	time_not_found += i_time
291	pass
292	elif method == "b":
293	try:
294	student_object = student_brain.getObject()
295	doc = getattr(student_object,oid).getContent()
296	found += 1
297	i_time = time.time() - i_elapse
298	time_found += i_time
299	except:
300	i_time = time.time() - i_elapse
301	time_not_found += i_time
302	not_found += 1
303	pass
304	elif method == "c":
305	if portal_types[i] in this_portal_types:
306	found += 1
307	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
308	i_time = time.time() - i_elapse
309	time_found += i_time
310	else:
311	not_found += 1
312	i_time = time.time() - i_elapse
313	time_not_found += i_time
314	elif method == "d":
315	if student_brain.has_key(portal_types[i]):
316	found += 1
317	doc = student_brain[portal_types[i]].getObject().getContent()
318	i_time = time.time() - i_elapse
319	time_found += i_time
320	else:
321	not_found += 1
322	i_time = time.time() - i_elapse
323	time_not_found += i_time
324	i_elapse = time.time()
325	if c and (c % intervall == 0):
326	#i_time = time.time() - i_elapse
327	t_per = 0.0
328	if found:
329	t_per = time_found/found
330	if t_per > t_max:
331	t_max = t_per
332	if t_per > 0.0 and t_per < t_min:
333	t_min = t_per
334	itf = 0.0
335	if found:
336	itf = time_found/found
337	itnf = 0.0
338	if not_found :
339	itnf = time_not_found / not_found
340	interval_time = time_found + time_not_found
341	s = "%(c)d: %(found)d/%(not_found)d " % vars()
342	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
343	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
344	print s
345	t_found += found
346	t_not_found += not_found
347	t_time_found += time_found
348	t_time_not_found += time_not_found
349	time_found = time_not_found = 0.0
350	found = not_found = 0
351	# t_found += found
352	# t_not_found += not_found
353	elapse = time.time() - elapse
354	itf = 0.0
355	if t_found:
356	itf = t_time_found/t_found
357	itnf = 0.0
358	if t_not_found:
359	itnf = t_time_not_found / t_not_found
360	#c_elapse = time.clock() - c_elapse
361	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
362	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
363	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
364	print "-"*40
365	print s
366	rel_found = float(t_found)/probe
367	rel_not_found = float(t_not_found)/probe
368	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
369	print estimated_total_time
370	###)
371
372	security.declareProtected(ModifyPortalContent,'writeLog') ###(
373	def writeLog(self,logfile,s):
374	"""write to the log file"""
375	logfile.write(s)
376	###)
377
378	def generateStudentId(self,letter): ###(
379	import random
380	logger = logging.getLogger('WAeUPTool.generateStudentId')
381	r = random
382	if letter == '?':
383	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
384	sid = "%c%d" % (letter,r.randint(99999,1000000))
385	students = self.portal_url.getPortalObject().campus.students
386	while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid):
387	if self.waeup_tool.picturePathExists(sid):
388	logger.info('picture path %s exists' % sid)
389	sid = "%c%d" % (letter,r.randint(99999,1000000))
390	return sid
391	###)
392
393	def generatePassword(self,s=None): ###(
394	import random
395	r = random
396	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
397	if s is None:
398	s = 'abcdefghklmnpqrstuvwxy23456789'
399	pw = ''
400	while len(pw) < 6:
401	pw += r.choice(s)
402	return pw
403	###)
404
405	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
406	def dumpSchoolfeePayments(self):
407	"dump paid schoolfees"
408	mtool = self.portal_membership
409	member = mtool.getAuthenticatedMember()
410	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
411	aq_student = self.students_catalog.evalAdvancedQuery
412	query = In('review_state',('schoolfee_paid',
413	'courses_registered',
414	'courses_validated',
415	))
416	res = aq_student(query)
417	#import pdb;pdb.set_trace()
418	l = []
419	logger.info("start for %d" % len(res))
420	count = 1
421	log_after = 100
422	for student in res:
423	if not count % log_after:
424	logger.info("processed %d total %d" % (log_after,count))
425	count += 1
426	fee_dict =self.getSchoolFee(student)
427	fulltime = student.mode.endswith('_ft')
428	d = {}
429	d['student_id'] = student.id
430	d['name'] = student.name
431	d['amount'] = fee_dict.get(new_returning)
432	l += d,
433	csv_name = self.dumpListToCSV(l,'payments')
434	logger.info('%s dumped payments to %s' % (member,export_file))
435	###)
436
437	security.declarePublic('dumpListToCSV') ###(
438	def dumpListToCSV(self,l,filename,fields=None):
439	"""dump a list of dicts to a CSV file"""
440	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
441	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
442	if fields is None:
443	fields = l[0].keys()
444	headline = ','.join(fields)
445	out = open(export_file,"wb")
446	out.write(headline +'\n')
447	out.close()
448	out = open(export_file,"a")
449	csv_writer = csv.DictWriter(out,fields,)
450	csv_writer.writerows(l)
451	return export_file
452	###)
453
454	security.declareProtected(ManagePortal, 'listMembers') ###(
455	def listMembers(self):
456	"list all members"
457	mtool = self.portal_membership
458	member = mtool.getAuthenticatedMember()
459	logger = logging.getLogger('WAeUPTool.listMembers')
460	if str(member) not in ('admin','joachim'):
461	logger.info('%s tried to list members' % (member))
462	return None
463	members = self.portal_directories.members
464	all = members.listEntryIdsAndTitles()
465	l = []
466	for user_id,name in all:
467	d = {}
468	d['user_id'] = user_id
469	d['name'] = name
470	d['pw'] = getattr(getattr(members,user_id),'password')
471	d['email'] = getattr(getattr(members,user_id),'email')
472	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
473	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
474	l += d,
475	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
476	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
477	logger.info('%s dumped member list to %s' % (member,export_file))
478	fields = l[0].keys()
479	headline = ','.join(fields)
480	out = open(export_file,"wb")
481	out.write(headline +'\n')
482	out.close()
483	out = open(export_file,"a")
484	csv_writer = csv.DictWriter(out,fields,)
485	csv_writer.writerows(l)
486	###)
487
488	security.declareProtected(ManagePortal, 'listStudents') ###(
489	def listStudents(self):
490	"list all students"
491	mtool = self.portal_membership
492	member = mtool.getAuthenticatedMember()
493	logger = logging.getLogger('WAeUPTool.listStudents')
494	if str(member) not in ('admin','joachim'):
495	logger.info('%s tried to list students' % (member))
496	return None
497	students = self.portal_directories.students
498	all = students.listEntryIdsAndTitles()
499	l = []
500	for user_id,name in all:
501	d = {}
502	d['user_id'] = user_id
503	d['name'] = name
504	d['pw'] = getattr(getattr(students,user_id),'password')
505	d['email'] = getattr(getattr(students,user_id),'email')
506	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
507	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
508	l += d,
509	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
510	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
511	logger.info('%s dumped student list to %s' % (member,export_file))
512	fields = l[0].keys()
513	headline = ','.join(fields)
514	out = open(export_file,"wb")
515	out.write(headline +'\n')
516	out.close()
517	out = open(export_file,"a")
518	csv_writer = csv.DictWriter(out,fields,)
519	csv_writer.writerows(l)
520	###)
521
522	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
523	def removeDeletedDocIds(self, max=1000):
524	"""
525	remove deleted docids from repository commit after max
526	"""
527	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
528	repository = getToolByName(self, 'portal_repository')
529	pxtool = getToolByName(self, 'portal_proxies')
530	logger.info('start')
531	pxtool_infos = pxtool.getRevisionsUsed()
532	logger.info('found %d used revisions ' % (len(pxtool_infos)))
533
534	nb_revs = 0
535	docids_d = {} # all docids
536	unused_docids_d = {} # all docids that are unused
537	ids_unused_revs_docids = [] # ids for revs of unused docids
538	unused_ids = [] # ids for unused revs
539	total = 0
540	idlist = repository.objectIds()
541	to_delete = 0
542	found = False
543	for id in idlist:
544	docid, rev = repository._splitId(id)
545	if docid is None:
546	logger.info("invalid doc_id %s" % docid)
547	continue
548	nb_revs += 1
549	if not pxtool_infos.has_key(docid):
550	found = True
551	to_delete += 1
552	unused_ids.append(id)
553	elif not pxtool_infos[docid].has_key(rev):
554	found = True
555	to_delete += 1
556	unused_ids.append(id)
557	if found and not to_delete % max:
558	found = False
559	#import pdb;pdb.set_trace()
560	repository.manage_delObjects(unused_ids)
561	transaction.commit()
562	logger.info('removed %d total %d unused docids ' % (max,to_delete))
563	else:
564	if unused_ids:
565	repository.manage_delObjects(unused_ids)
566	transaction.commit()
567	logger.info('finished removing %d unused docids ' % (to_delete))
568	###)
569
570	security.declarePublic('getCredential') ###(
571	def getCredential(self,student_id):
572	student_entry = getattr(self.portal_directories.students,student_id,None)
573	#import pdb;pdb.set_trace()
574	#if not self.isStaff():
575	# mtool = self.portal_membership
576	# member = mtool.getAuthenticatedMember()
577	# logger = logging.getLogger('WAeUPTool.getCredential')
578	# logger.info('%s tried to access password of %s' % (member,student_id))
579	# return None
580	if student_entry is None:
581	return None
582	return getattr(student_entry,"password","not set")
583	###)
584
585	security.declarePublic('checkPassword') ###(
586	def checkPassword(self,student_id,password):
587	student_entry = getattr(self.portal_directories.students,student_id,None)
588	if student_entry is None:
589	return False
590	return getattr(student_entry,"password","not set") == password
591	###)
592
593	security.declarePublic('checkGenericPassword') ###(
594	def checkGenericPassword(self,member_id):
595	member_entry = getattr(self.portal_directories.members,member_id,None)
596	if member_entry is None:
597	return False
598	ltool = getToolByName(self, 'portal_layouts')
599	unsecure_words = ltool._getOb('members')['w__password'].check_words
600	password = getattr(member_entry,"password","not set")
601	is_unsecure = password in unsecure_words
602	if is_unsecure:
603	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
604	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
605	return is_unsecure
606	###)
607
608	security.declareProtected(ModifyPortalContent,'editPassword') ###(
609	def editPassword(self,student_id,password):
610	"edit a student password"
611	student_entry = getattr(self.portal_directories.students,student_id,None)
612	if student_entry is None:
613	return
614	setattr(student_entry,'password',password)
615	###)
616
617	security.declareProtected(ModifyPortalContent,'doCommit') ###(
618	def doCommit(self,logger=None):
619	"commit some transactions"
620	transaction.commit()
621	###)
622
623	security.declarePublic('loadStudentFoto') ###(
624	def loadStudentFoto(self,student,filename,folder):
625	"return a student passport picture"
626	#import pdb;pdb.set_trace()
627	picture ="%s/import/%s/%s" % (i_home,folder,filename)
628	student_id = student.getId()
629	images_dir = getImagesDir(student_id)
630	if not os.path.exists(images_dir):
631	os.mkdir(images_dir)
632	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
633	for extension in ('.jpg','.JPG'):
634	fullname = "%(picture)s%(extension)s" % vars()
635	if os.path.exists(fullname):
636	copy2(fullname,image_name)
637	return "successfully copied passport picture"
638	return "passport picture not found: %s.jpg or .JPG" % picture
639	###)
640
641
642	security.declareProtected(ModifyPortalContent,'createOne') ###(
643	def createOne(self,students_folder,student_brain,letter,commit=False):
644	sid = self.waeup_tool.generateStudentId(letter)
645	students_folder.invokeFactory('Student', sid)
646	student = getattr(students_folder,sid)
647	self.portal_workflow.doActionFor(student,'return')
648	student.manage_setLocalRoles(sid, ['Owner',])
649	matric_no = student_brain.matric_no
650	jamb_reg_no = student_brain.Entryregno
651	self.students_catalog.addRecord(id = sid,
652	matric_no = matric_no,
653	jamb_reg_no = jamb_reg_no,
654	sex = student_brain.Sex == "F",
655	name = "%s %s %s" % (student_brain.Firstname,
656	student_brain.Middlename,
657	student_brain.Lastname)
658	)
659	if commit:
660	transaction.commit()
661	return sid,jamb_reg_no
662	###)
663
664	# Functions without docstring cannot be called from outside (via the URL),
665	# thus we can declare addStudent public.
666	security.declarePublic('addStudent') ###(
667	def addStudent(self,dict):
668	students_folder = self.portal_url.getPortalObject().campus.students
669	sid = self.waeup_tool.generateStudentId('?')
670	students_folder.invokeFactory('Student', sid)
671	student_obj = getattr(students_folder,sid)
672	f2t = StudentImport.field2types_student
673	#from pdb import set_trace; set_trace()
674	d = {}
675	#d['jamb_sex'] = 'M'
676	#if dict.get('sex'):
677	# d['jamb_sex'] = 'F'
678
679	entry_session = dict.get('entry_session')
680	if entry_session == self.getSessionId()[0]:
681	wfaction = 'admit'
682	wft = 'wf_transition_admit'
683	password = None
684	else:
685	wfaction = 'return'
686	wft = 'wf_transition_return'
687	password = self.generatePassword()
688	self.makeStudentMember(sid,password)
689
690	for pt in f2t.keys():
691	student_obj.invokeFactory(pt,f2t[pt]['id'])
692	sub_obj = getattr(student_obj,f2t[pt]['id'])
693	sub_doc = sub_obj.getContent()
694	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
695	#d['Title'] = f2t[pt]['title']
696	for field in f2t[pt]['fields']:
697	d[field] = dict.get(field,'')
698	sub_doc.edit(mapping = d)
699	new_state = f2t[pt][wft]
700	if new_state != "remain":
701	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
702	self.portal_workflow.doActionFor(student_obj,wfaction)
703	student_obj.manage_setLocalRoles(sid, ['Owner',])
704	return sid,password
705	###)
706
707	security.declarePublic('getCertificateBrain') ###(
708	def getCertificateBrain(self,cert_id):
709	"do it"
710	res = ZCatalog.searchResults(self.portal_catalog_real,
711	{'portal_type':"Certificate",
712	'id': cert_id})
713	if res:
714	return res[0]
715	return None
716	###)
717
718	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
719	def get_csv_filenames(self):
720	"do it"
721	files = [file for file in os.listdir("%s/import/" % (i_home))
722	if file.endswith('.csv') and (file.find('imported') == -1 and
723	file.find('pending') == -1)]
724	return files
725	###)
726
727	security.declarePublic('findStudentByMatricelNo') ###(
728	def findStudentByMatricelNo(self,matric_no):
729	"do it"
730	res = ZCatalog.searchResults(self.portal_catalog_real,
731	{'portal_type':"StudentClearance",
732	'SearchableText': matric_no})
733	if res:
734	return res[0]
735	return None
736	###)
737
738
739	security.declarePublic('getOfficerName') ###(
740	def getOfficerName(self,mid):
741	"""get the real name of a member"""
742	membership = self.portal_membership
743	member_record = membership.getMemberById(mid)
744	return member_record.getProperty('fullname',None)
745	###)
746
747
748	security.declarePublic('makeStudentMember') ###(
749	def makeStudentMember(self,sid,password='uNsEt'):
750	"""make the student a member"""
751	membership = self.portal_membership
752	membership.addMember(sid,
753	password ,
754	roles=('Member',
755	'Student',
756	),
757	domains='',
758	properties = {'memberareaCreationFlag': False,
759	'homeless': True},)
760	member = membership.getMemberById(sid)
761	self.portal_registration.afterAdd(member, sid, password, None)
762	#self.manage_setLocalRoles(sid, ['Owner',])
763	###)
764
765	security.declareProtected(View,'makeStudentData') ###(
766	def makeStudentData(self,student_id,email=None,phone_nr=None):
767	"create Datastructure for a returning Student"
768	#import pdb;pdb.set_trace()
769	logger = logging.getLogger('WAeUPTool.makeStudentData')
770	students_folder = self.portal_url.getPortalObject().campus.students
771	#res = self.students_catalog(id=student_id)
772	#if res:
773	# st = res[0]
774	#res = self.returning_import(matric_no = st.matric_no)
775	res = self.returning_import(id = student_id)
776	if res:
777	student = res[0]
778	else:
779	logger.info('Id %s not found in returning_import' % student_id)
780	return
781	logger.info('%s creates data structure' % student_id)
782	s_results = self.results_import(matric_no = student.matric_no)
783	if s_results:
784	lnr = self.getLevelFromResultsCosCode(s_results)
785	level = "%d00" % lnr
786	verdict,eligible = self.getVerdict(s_results[0].Verdict)
787	#if eligible:
788	# level = "%d00" % (lnr + 1)
789	else:
790	logger.info('matric_no %s not found in results_import' % student.matric_no)
791	level = ''
792	verdict = ''
793	#student should not be allowed to perform this transition
794	#wftool = self.portal_workflow
795	#wftool.doActionFor(student,'return')
796	certcode_org = student.Coursemajorcode
797	certcode = makeCertificateCode(certcode_org)
798	certificate_brain = self.getCertificateBrain(certcode)
799	if not certificate_brain:
800	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
801	logger.info(em)
802	matric_no = student.matric_no
803	sid = student_id
804	student_obj = getattr(students_folder,sid)
805	if not getattr(student_obj,'application'):
806	student_obj.invokeFactory('StudentApplication','application')
807	application = student_obj.application
808	self.portal_workflow.doActionFor(application,'open',dest_container=application)
809	da = {'Title': 'Application Data'}
810	student_obj.invokeFactory('StudentPersonal','personal')
811	da['jamb_reg_no'] = student.Entryregno
812	em = self.getEntryMode(student.Entryregno)
813	da['entry_mode'] = em
814	personal = student_obj.personal
815	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
816	dp = {'Title': 'Personal Data'}
817	student_obj.invokeFactory('StudentClearance','clearance')
818	clearance = student_obj.clearance
819	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
820	dc = {'Title': 'Clearance/Eligibility Record'}
821	dc['matric_no'] = matric_no
822	state = student.State
823	lga = student.LGA
824	if state and lga:
825	lga = state + ' / ' + lga
826	else:
827	lga = "None"
828	da['jamb_lga'] = dc['lga'] = lga
829	da['app_email'] = dp['email'] = email
830	da['app_mobile'] = dp['phone'] = phone_nr
831	dp['firstname'] = student.Firstname
832	dp['middlename'] = student.Middlename
833	dp['lastname'] = student.Lastname
834	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
835	da['jamb_sex'] = student.Sex
836	dp['sex'] = student.Sex == 'F'
837	dp['perm_address'] = student.Permanent_Address
838	application.getContent().edit(mapping=da)
839	self.portal_workflow.doActionFor(application,'close',dest_container=application)
840	personal.getContent().edit(mapping=dp)
841	clearance.getContent().edit(mapping=dc)
842	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
843	#
844	# Study Course
845	#
846	student_obj.invokeFactory('StudentStudyCourse','study_course')
847	studycourse = student_obj.study_course
848	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
849	dsc = {}
850	dsc['study_course'] = certcode
851	dsc['current_level'] = level
852	dsc['current_verdict'] = verdict
853	dsc['current_mode'] = em #no longer used
854	dsc['current_session'] = '05'
855	studycourse.getContent().edit(mapping=dsc)
856	#
857	# Level
858	#
859	# l = getattr(studycourse,level,None)
860	# if l is None:
861	# studycourse.invokeFactory('StudentStudyLevel', level)
862	# l = getattr(studycourse, level)
863	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
864	# l.getContent().edit(mapping={'Title': "Level %s" % level})
865	###)
866
867	def init_timing(self): ###(
868	if self.with_timing:
869	if not hasattr(self,'_v_step_times'):
870	self._v_timer_count = 0
871	self._v_total = 0
872	self._v_step_times = {}
873	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
874	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
875	self.timer_step = 0
876	self.total_time = 0
877	self.elapse = time.time()
878	self.i_elapse = time.time()
879	self.c_elapse = time.clock()
880	###)
881
882	def do_timing(self): ###(
883	if self.with_timing:
884	try:
885	raise 'dummy'
886	except:
887	frame = sys.exc_traceback.tb_frame.f_back
888	locals = frame.f_locals
889	globals = frame.f_globals
890	functionname = frame.f_code.co_name
891	filename = os.path.basename(frame.f_code.co_filename)
892	lineno = frame.f_lineno
893	mod_line = "%(functionname)s:%(lineno)s" % vars()
894	i_time = time.time() - self.i_elapse
895	td = {}
896	if self._v_step_times.has_key(mod_line):
897	a_time = self._v_step_times[mod_line]['a_time'] + i_time
898	td['a_time'] = a_time
899	else:
900	td['a_time'] = i_time
901	td['i_time'] = i_time
902	self._v_step_times[mod_line] = td
903	self.i_time = i_time
904	self.total_time += i_time
905	self.timer_step +=1
906	self.i_elapse = time.time()
907	###)
908
909	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
910	def print_timing(self):
911	if self.with_timing:
912	l = []
913	timer_count = self._v_timer_count + 1
914	mod_lines = self._v_step_times.keys()
915	mod_lines.sort(cmp,reverse=0)
916	for mod_line in mod_lines:
917	td = self._v_step_times[mod_line]
918	i_time = td['i_time']
919	a_time = td['a_time']/(self._v_timer_count + 1)
920	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
921	total_time = self.total_time
922	total_avarage = self._v_total / timer_count
923	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
924	print "\r\n".join(l)
925	out = open(self._v_timer_file,'a')
926	out.write("\r\n".join(l))
927	out.close()
928	###)
929
930	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
931	def get_timing_data(self):
932	if self.with_timing:
933	timer_count = self._v_timer_count + 1
934	results = {}
935	for k,d in self._v_step_times.items():
936	dd = {}
937	dd['a_time'] = d['a_time']/timer_count
938	dd['i_time'] = d['i_time']
939	dd['count'] = timer_count
940	results[k] = dd
941	dd = {}
942	dd['a_time'] = self._v_total / timer_count
943	dd['i_time'] = self.total_time
944	dd['count'] = timer_count
945	results["total"] = dd
946	return results
947	###)
948
949	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
950	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
951	"create Datastructure for an admitted Student"
952	#import pdb;pdb.set_trace()
953	logger = logging.getLogger('WAeUPTool.admitOneStudent')
954	self.with_timing = with_timing
955
956	if brain.screening_type in ('cest','sandwich',):
957	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
958	else:
959	reg_no = brain.reg_no
960
961	#ignore argument entry_session
962	if not brain.entry_session:
963	logger.info('no entry_session for %s provided' % (reg_no))
964	return
965	else:
966	es = brain.entry_session
967	if len(es) == 1:
968	es = '0%c' % es
969
970	if not hasattr(self,"_v_certificates"):
971	self._v_certificates = self.getCertificatesDict()
972	students_folder = self.portal_url.getPortalObject().campus.students
973
974	res = self.students_catalog(jamb_reg_no = reg_no)
975	if res:
976	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
977	return
978	if brain.status != "admitted":
979	logger.info('status of %s is %s' % (reg_no,brain.status))
980	return
981	pin_parts = brain.pin.split('-')
982	if pin_parts and len(pin_parts) != 3:
983	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
984	return
985	if not brain.course_admitted:
986	logger.info('no course_admitted provided for %s' % (reg_no))
987	return
988	if brain.course_admitted not in self._v_certificates:
989	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
990	return
991	if brain.sex not in (True,False):
992	logger.info('sex of %s not available' % (reg_no))
993	return
994	self.init_timing()
995	student_id = self.generateStudentId('?')
996	students_folder.invokeFactory('Student', student_id)
997	student_object = getattr(students_folder,student_id)
998	self.do_timing()
999	if pin_password:
1000	password = pin_parts[2]
1001	self.makeStudentMember(student_id,password = password)
1002	student_object.manage_setLocalRoles(student_id, ['Owner',])
1003	self.do_timing()
1004	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1005	#
1006	# application
1007	#
1008	student_object.invokeFactory('StudentApplication','application')
1009	application = student_object.application
1010	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1011	#self.do_timing()
1012	da = {'Title': 'Application Data'}
1013	da['jamb_reg_no'] = reg_no
1014
1015	sex = 'M'
1016	if brain.sex:
1017	sex = 'F'
1018	da['jamb_sex'] = sex
1019	da['jamb_age'] = brain.jamb_age
1020	da['app_reg_pin'] = brain.pin
1021	da['jamb_lga'] = brain.jamb_lga
1022	da['jamb_state'] = brain.jamb_state
1023	da['jamb_score'] = brain.aggregate
1024	da['app_email'] = brain.email
1025	da['app_mobile'] = brain.phone
1026
1027	# entry_mode cannot be retrieved from the certtificate
1028	# because ug_ft has two different entry modes
1029
1030	if brain.entry_mode: # does not happen because import_application
1031	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
1032	elif brain.screening_type == 'pume':
1033	da['entry_mode'] = 'ume_ft'
1034	elif brain.screening_type == 'pde':
1035	da['entry_mode'] = 'de_ft'
1036	else:
1037	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1038
1039	#elif brain.screening_type == 'pce':
1040	# da['entry_mode'] = 'pce'
1041	#elif brain.screening_type == 'prence':
1042	# da['entry_mode'] = 'prence'
1043	#else:
1044	# da['entry_mode'] = ''
1045
1046
1047	#da['entry_session'] = entry_session
1048	da['entry_session'] = es
1049	da['jamb_lastname'] = brain.lastname
1050	da['jamb_middlename'] = brain.middlenames # different field names!
1051	da['jamb_firstname'] = brain.firstname
1052	da['screening_application_date'] = brain.application_date
1053	da['date_of_birth'] = brain.date_of_birth
1054	da['jamb_first_cos'] = brain.course1
1055	da['jamb_second_cos'] = brain.course2
1056	da['course3'] = brain.course3
1057	da['screening_type'] = brain.screening_type
1058	da['screening_score'] = brain.screening_score
1059	da['screening_date'] = brain.screening_date
1060	da['hq_type'] = brain.hq_type
1061	da['hq_grade'] = brain.hq_grade
1062	da['aos'] = brain.aos
1063
1064	application.getContent().edit(mapping=da)
1065	self.do_timing()
1066	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1067	#
1068	# personal
1069	#
1070	student_object.invokeFactory('StudentPersonal','personal')
1071	personal = student_object.personal
1072	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1073	#self.do_timing()
1074	dp = {'Title': 'Personal Data'}
1075	dp['sex'] = brain.sex
1076	dp['email'] = brain.email
1077	dp['phone'] = brain.phone
1078	dp['lastname'] = brain.lastname
1079	dp['middlename'] = brain.middlenames # different field names!
1080	dp['firstname'] = brain.firstname
1081	personal.getContent().edit(mapping=dp)
1082	self.do_timing()
1083	#
1084	# clearance
1085	#
1086	student_object.invokeFactory('StudentClearance','clearance')
1087	clearance = student_object.clearance
1088	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1089	dc = {'Title': 'Clearance/Eligibility Record'}
1090	dc['lga'] = brain.lga
1091	dc['birthday'] = brain.date_of_birth
1092	clearance.getContent().edit(mapping=dc)
1093	self.do_timing()
1094	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1095	#
1096	# study Course
1097	#
1098	student_object.invokeFactory('StudentStudyCourse','study_course')
1099	studycourse = student_object.study_course
1100	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1101	#self.do_timing()
1102	dsc = {}
1103	dsc['study_course'] = brain.course_admitted
1104	dsc['current_verdict'] = ''
1105	dsc['current_mode'] = da['entry_mode'] # no longer used
1106	if da['entry_mode'].startswith('de'):
1107	dsc['current_level'] = '200'
1108	elif da['entry_mode'].startswith('pre'):
1109	dsc['current_level'] = '000'
1110	else:
1111	dsc['current_level'] = '100'
1112	dsc['current_session'] = es
1113	studycourse.getContent().edit(mapping=dsc)
1114	self.do_timing()
1115	#
1116	# payments folder
1117	student_object.invokeFactory('PaymentsFolder','payments')
1118	payments = getattr(student_object,'payments')
1119	#self.do_timing()
1120	dpay = {}
1121	dpay['Title'] = 'Payments'
1122	payments.getContent().edit(mapping=dpay)
1123	self.portal_workflow.doActionFor(payments,'open')
1124	self.do_timing()
1125	#
1126	# passport foto
1127	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1128	brain.screening_type,
1129	brain.reg_no)
1130	images_dir = getImagesDir(student_id)
1131	#images_dir = os.path.join("%s" % images_base,student_id)
1132	letter_dir,student_dir = os.path.split(images_dir)
1133	if not os.path.exists(letter_dir):
1134	os.mkdir(letter_dir)
1135	if not os.path.exists(images_dir):
1136	os.mkdir(images_dir)
1137	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1138	if os.path.exists(app_picture):
1139	copy2(app_picture,image_name)
1140	else:
1141	logger.info('passport of %s/%s not found: %s' % (student_id,
1142	brain.reg_no,
1143	app_picture))
1144
1145	self.do_timing()
1146	self.print_timing()
1147	if with_timing:
1148	self.timer_step = 0
1149	self._v_timer_count += 1
1150	self._v_total += self.total_time
1151	return student_id
1152	###)
1153
1154	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1155	def makeStudentLevel(self,student_id):
1156	"create the StudyLevel for a returning Student"
1157	#import pdb;pdb.set_trace()
1158	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1159	students_folder = self.portal_url.getPortalObject().campus.students
1160	res = self.students_catalog(id=student_id)
1161	if res:
1162	st = res[0]
1163	course = st.course
1164	matric_no = st.matric_no
1165	level = st.level
1166	res = self.results_import(matric_no = matric_no)
1167	if res:
1168	results = res
1169	logger.info('%s creating Level %s' % (student_id,level))
1170	#
1171	# Level
1172	#
1173	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1174	studycourse = getattr(student_obj,"study_course",None)
1175	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1176	l = getattr(studycourse,level,None)
1177	if l is None:
1178	studycourse.invokeFactory('StudentStudyLevel', level)
1179	l = getattr(studycourse, level)
1180	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1181	l.getContent().edit(mapping={'Title': "Level %s" % level})
1182	###)
1183
1184	security.declarePublic('getHallInfo') ###(
1185	def getHallInfo(self,bed):
1186	"""return Hall Info"""
1187	info = {}
1188	bedsplit = bed.split('_')
1189	if len(bedsplit) == 4:
1190	hall,block,room,letter = bed.split('_')
1191	else:
1192	info['maintenance_code'] = 'None'
1193	return info
1194	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1195	if res and len(res) == 1:
1196	hall_brain = res[0]
1197	hall_doc = hall_brain.getObject().getContent()
1198	else:
1199	return info
1200	info['hall_title'] = hall_brain.Title
1201	info['maintenance_code'] = hall_doc.maintenance_code
1202	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1203	batch_doc = None
1204	for brain in res:
1205	if brain.id.startswith(info['maintenance_code']):
1206	batch_doc = brain.getObject().getContent()
1207	break
1208	if batch_doc is None:
1209	info['maintenance_fee'] = ''
1210	else:
1211	info['maintenance_fee'] = batch_doc.cost
1212	return info
1213	###)
1214
1215	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1216	def removePictureFolder(self,student_id):
1217	"""remove picture_folder by renaming it"""
1218	picture_path = getImagesDir(student_id)
1219	dest_path = os.path.join("%s" % images_base,'removed',student_id)
1220	dest_path = dest_path + "_removed"
1221	if os.path.exists(dest_path) or not os.path.exists(picture_path):
1222	return False
1223	os.rename(picture_path,dest_path)
1224	return True
1225	###)
1226
1227	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1228	def restorePictureFolder(self,student_id):
1229	"""restore picture_folder by renaming it"""
1230	picture_path = getImagesDir(student_id)
1231	orig_path = os.path.join("%s" % images_base,'removed',student_id)
1232	orig_path = orig_path + "_removed"
1233	if os.path.exists(picture_path) or not os.path.exists(orig_path):
1234	return False
1235	os.rename(picture_path + "_removed",picture_path)
1236	return True
1237	###)
1238
1239	security.declarePublic('picturesExist') ###(
1240	def picturesExist(self, ids,student_id=None):
1241	"""check if pictures exist in the filesystem"""
1242	if student_id is None:
1243	student_id = self.getStudentId()
1244	if student_id is None:
1245	return False
1246	picture_path = getImagesDir(student_id)
1247	#picture_path = os.path.join(images_base,student_id)
1248	if not os.path.exists(picture_path):
1249	return False
1250	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1251	return set(ids).issubset(set(pictures))
1252	###)
1253
1254	security.declarePublic('picturePathExists') ###(
1255	def picturePathExists(self, student_id=None):
1256	"""check if picture path exists in the filesystem"""
1257	if student_id is None:
1258	student_id = self.getStudentId()
1259	if student_id is None:
1260	return False
1261	picture_path = getImagesDir(student_id)
1262	if os.path.exists(picture_path):
1263	return True
1264	return False
1265	###)
1266
1267
1268	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1269	def removeUnusedImageFolders(self):
1270	"""check if an unused image folders exists in the filesystem"""
1271	mtool = self.portal_membership
1272	member = mtool.getAuthenticatedMember()
1273	member_id = str(member)
1274	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1275	abc = os.listdir(images_base)
1276	ifolders = []
1277	for i in abc:
1278	picture_path = os.path.join(images_base,i)
1279	ifolders.extend(os.listdir(picture_path))
1280	unused_ids = []
1281	for id in ifolders:
1282	res = self.students_catalog(id=id)
1283	if not res:
1284	unused_ids.append(id)
1285	#import pdb;pdb.set_trace()
1286	if not id.endswith('removed'):
1287	removed = self.waeup_tool.removePictureFolder(id)
1288	if removed:
1289	logger.info('%s: image folder %s successfully removed' % (member_id,id))
1290	else:
1291	logger.info('%s: image folder %s could not be removed' % (member_id,id))
1292	return
1293
1294	###)
1295
1296	security.declarePublic('picturesList') ###(
1297	def picturesList(self):
1298	"""check if pictures exist in the filesystem"""
1299	path = 'images'
1300	student_id = self.getStudentId()
1301	#picture_path = os.path.join(i_home,path,student_id)
1302	picture_path = getImagesDir(student_id)
1303	if not os.path.exists(picture_path):
1304	return []
1305	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1306	###)
1307
1308	security.declarePublic('showFsPicture') ###(
1309	def showFsPicture(self,path):
1310	"""return a picture from the filesystem"""
1311	#picture_path = os.path.join(i_home,path)
1312	picture_path = os.path.join(images_base,path)
1313	response = self.REQUEST.RESPONSE
1314	#import pdb;pdb.set_trace()
1315	registry = getToolByName(self, 'mimetypes_registry')
1316	mimetype = str(registry.lookupExtension(path.lower()) or
1317	registry.lookupExtension('file.bin'))
1318	if os.path.exists(picture_path):
1319	response.setHeader('Content-type',mimetype)
1320	return open(picture_path).read()
1321	picture_path = os.path.join(i_home,'import',path)
1322	if os.path.exists(picture_path):
1323	return open(picture_path).read()
1324	###)
1325
1326	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1327	def deleteAllCourses(self,department="All"):
1328	''' delete the courses'''
1329	pm = self.portal_membership
1330	member = pm.getAuthenticatedMember()
1331
1332	if str(member) not in ("henrik","joachim"):
1333	return "not possible"
1334	if department == "All":
1335	res = self.portal_catalog({'meta_type': 'Department'})
1336	if len(res) < 1:
1337	return "No Departments found"
1338
1339	deleted = []
1340	for dep in res:
1341	cf = dep.getObject().courses
1342	if cf:
1343	cf.manage_delObjects(ids=cf.objectIds())
1344	deleted.append("deleted Courses in %s" % dep.getId)
1345	return "\r".join(deleted)
1346	###)
1347
1348	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1349	def getLogfileLines(self,filename="event.log",numlines=20):
1350	"""Get last NUMLINES lines of logfile FILENAME.
1351
1352	Return last lines' of a file in the instances logfile directory as
1353	a list. The number of returned lines equals `numlines' or less. If
1354	less than `numlines' lines are available, the whole file ist
1355	returned. If the file can not be opened or some other error
1356	occurs, empty list is returend.
1357	"""
1358	result = []
1359	lines_hit = 0
1360
1361	# We only handle files in instances' log directory...
1362	logpath = os.path.join(i_home, "log")
1363	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1364	if not filename.startswith( logpath ):
1365	# Attempt to access file outside log-dir...
1366	return []
1367
1368	try:
1369	fd = file( filename, "rb" )
1370	except IOError:
1371	return []
1372	if not fd:
1373	return []
1374
1375	if os.linesep == None:
1376	linesep = '\n'
1377	else:
1378	linesep = os.linesep
1379
1380	# Try to find 'numlines' times a lineseparator, searching from end
1381	# and moving to the beginning of file...
1382	fd.seek( 0, 2) # Move to end of file...
1383	while lines_hit < numlines:
1384	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1385	# one step forward...
1386	lines_hit += 1
1387	try:
1388	fd.seek( -2, 1) # Go two bytes back from current pos...
1389	except IOError:
1390	# We cannot go back two bytes. Maybe the file is too small...
1391	break
1392	fd.seek(2,1)
1393
1394	# Read all lines from current position...
1395	result = fd.readlines()
1396	# Remove line endings...
1397	result = [x.strip() for x in result]
1398	fd.close()
1399	return result
1400	###)
1401
1402	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1403	def getCallbacksFromLog(self,filename):
1404	"""fix Online Payment Transactions from Z2.log entries"""
1405	import transaction
1406	import random
1407	from cgi import parse_qs
1408	from urlparse import urlparse
1409	#from pdb import set_trace
1410	wftool = self.portal_workflow
1411	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1412	students_folder = self.portal_url.getPortalObject().campus.students
1413	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1414	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1415	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1416	data = re.compile(s)
1417	start = True
1418	tr_count = 1
1419	total = 0
1420	#name = 'pume_results'
1421	#name = 'epaymentsuccessful_z2log2'
1422	name = filename
1423	no_import = []
1424	imported = []
1425	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1426	try:
1427	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1428	except:
1429	logger.error('Error reading %s' % name)
1430	return
1431	tas = []
1432	for line in transactions:
1433	dict = {}
1434	items = data.search(line)
1435	dict['idict'] = idict = items.groupdict()
1436	#print idict
1437	#from pdb import set_trace;set_trace()
1438	urlparsed = urlparse(idict['get'][4:])
1439	#print urlparsed
1440	path = urlparsed[2].split('/')
1441	dict['student_id'] = student_id = path[8]
1442	dict['payment_id'] = payment_id = path[10]
1443	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1444	tas.append(dict)
1445	tr_count += 1
1446	return tas
1447	###)
1448
1449	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1450	def importOnlinePaymentTransactions(self):
1451	"""load Online Payment Transactions from CSV values"""
1452	import transaction
1453	import random
1454	#from pdb import set_trace
1455	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1456	opt = self.online_payments_import
1457	students_folder = self.portal_url.getPortalObject().campus.students
1458	start = True
1459	tr_count = 1
1460	total = 0
1461	#name = 'pume_results'
1462	name = 'OnlineTransactions'
1463	no_import = []
1464	imported = []
1465	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1466	try:
1467	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1468	except:
1469	logger.error('Error reading %s.csv' % name)
1470	return
1471	for pay_transaction in transactions:
1472	if start:
1473	start = False
1474	logger.info('Start loading from %s.csv' % name)
1475	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1476	no_import.append('%s,"Error"' % s)
1477	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1478	format_error = format + ',"%(Error)s"'
1479	data = {}
1480
1481	# format of the first file sent by Tayo
1482	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1483	#data['student_id'] = student_id = pay_transaction['Payer ID']
1484	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1485	#data['response_code'] = response_code = pay_transaction['Resp Code']
1486	#data['amount'] = amount = pay_transaction['Amount']
1487
1488	# format of the second file sent by Tayo
1489	#data['datetime'] = date = 0
1490	#data['student_id'] = student_id = pay_transaction['Payer ID']
1491	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1492	#data['response_code'] = response_code = '00'
1493	#data['amount'] = amount = pay_transaction['Amount']
1494
1495	# format of the third file sent by Kehinde
1496	data['datetime'] = date = 0
1497	data['student_id'] = student_id = pay_transaction['customer_id']
1498	data['order_id'] = order_id = pay_transaction['merchant_reference']
1499	data['response_code'] = response_code = '00'
1500	data['amount'] = amount = pay_transaction['Amount']
1501
1502	dup = False
1503	if response_code == "12":
1504	continue
1505	try:
1506	opt.addRecord(**data)
1507	except ValueError:
1508	dup = True
1509	#from pdb import set_trace;set_trace()
1510	if dup:
1511	if response_code == "00":
1512	try:
1513	opt.modifyRecord(**data)
1514	except:
1515	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1516	continue
1517	else:
1518	pay_transaction['Error'] = "Duplicate order_id"
1519	no_import.append( format_error % pay_transaction)
1520	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1521	continue
1522	tr_count += 1
1523	if tr_count > 1000:
1524	if len(no_import) > 0:
1525	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1526	'\n'.join(no_import) + '\n')
1527	no_import = []
1528	em = '%d transactions committed\n' % (tr_count)
1529	transaction.commit()
1530	regs = []
1531	logger.info(em)
1532	total += tr_count
1533	tr_count = 0
1534	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1535	'\n'.join(no_import))
1536	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1537	###)
1538
1539	security.declareProtected(ModifyPortalContent,"importData")###(
1540	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1541	"""load data from CSV values"""
1542	import transaction
1543	import random
1544	students_folder = self.portal_url.getPortalObject().campus.students
1545	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1546	pending_only = False
1547	pend_str = '--'
1548	elapse = time.time()
1549	#
1550	# preparations
1551	#
1552	if filename == pend_str:
1553	pending_only = True
1554	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1555	importer = eval("%sImport" % importer_name)(self)
1556	logger = importer.logger
1557	if importer.init_errors:
1558	logger.info(importer.init_errors)
1559	return importer.init_errors
1560	member = importer.member
1561	#current = importer.current
1562	import_date = importer.import_date
1563	#
1564	# not_imported
1565	#
1566	info = importer.info
1567	data_keys = importer.data_keys
1568	csv_keys = importer.csv_keys
1569	#csv_keys.extend(info.keys())
1570	headline_mapping = dict((k,k) for k in csv_keys)
1571	#
1572	# pending
1573	#
1574	pending_path = importer.pending_path
1575	pending_tmp = importer.pending_tmp
1576	pending_backup = importer.pending_backup
1577	pending_fn = importer.pending_fn
1578	imported_path = importer.imported_path
1579	imported_fn = importer.imported_fn
1580	commit_after = importer.commit_after
1581	pending = []
1582	pending_digests = []
1583	#total_added_to_pending = 0
1584	if not pending_only:
1585	pending,pending_digests = importer.makeIdLists()
1586	pending_at_start = len(pending)
1587	datafile = open(pending_tmp,"w")
1588	pending_csv_writer = csv.DictWriter(datafile,
1589	csv_keys,
1590	extrasaction='ignore')
1591	pending_csv_writer.writerow(headline_mapping)
1592	datafile.close()
1593	#
1594	# imported
1595	#
1596	if not os.path.exists(imported_path):
1597	datafile = open(imported_path,"w")
1598	imported_csv_writer = csv.DictWriter(datafile,
1599	csv_keys,
1600	extrasaction='ignore')
1601	imported_csv_writer.writerow(headline_mapping)
1602	datafile.close()
1603	start = True
1604	tr_count = 0
1605	total = 0
1606	total_added_to_imported = 0
1607	total_pending = 0
1608	import_source_done = ""
1609	if pending_only:
1610	import_source_path = pending_path
1611	else:
1612	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1613	import_source_done = "%s/import/%s.done" % (i_home,filename)
1614	if not os.path.exists(import_source_path):
1615	fn = os.path.split(import_source_path)[1]
1616	em = 'no import file %(fn)s' % vars()
1617	return em
1618	import_source_fn = os.path.split(import_source_path)[1]
1619	if not pending_only:
1620	info['imported_from'] = import_source_fn
1621	headline = csv.reader(open(import_source_path,"rb")).next()
1622	if "import_mode" not in headline:
1623	msg = 'import_mode must be in heading'
1624	return msg
1625	invalid_keys = importer.checkHeadline(headline)
1626	if invalid_keys:
1627	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1628
1629	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1630	or k.strip() in info.keys())]
1631	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1632	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1633	# if diff2schema and diff2schema != set(['id',]):
1634	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1635	# return msg
1636	#
1637	# start importing
1638	#
1639	try:
1640	reader = csv.DictReader(open(import_source_path,"rb"))
1641	except:
1642	msg = 'Error reading %s.csv' % filename
1643	logger.error(msg)
1644	return msg
1645	items = [item for item in reader]
1646	total_to_import = len(items)
1647	tti_float = float(total_to_import)
1648	if pending_only:
1649	pending_at_start = total_to_import
1650	count = 0
1651	imported = []
1652	old_commit_count = 0
1653	error_count = imported_count = 0
1654	already_in = 0
1655	for record in items:
1656	item = {}
1657	empty_value_keys = []
1658	for k,v in record.items():
1659	if k is None:
1660	continue
1661	if v:
1662	if v == EMPTY:
1663	empty_value_keys += k,
1664	v = ''
1665	item[k.strip()] = v.strip()
1666	count += 1
1667	if start:
1668	start = False
1669	adapters = [MappingStorageAdapter(importer.schema, item)]
1670	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1671	dm = DataModel(item, adapters,context=self)
1672	ds = DataStructure(data=item,datamodel=dm)
1673	error_string = ""
1674	total += 1
1675	import_mode = item.get('import_mode','')
1676	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1677	if import_method is None:
1678	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1679	elif (import_mode in importer.required_modes and
1680	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1681	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1682	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1683	else:
1684	for k in import_keys:
1685	if k not in item.keys() or k not in importer.validators.keys():
1686	continue
1687	if not importer.validators[k](ds,mode=import_mode):
1688	error_string += ' ++ '
1689	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1690	ds.getErrorMapping(k)))
1691	if error_string:
1692	error = error_string
1693	id = ''
1694	mapping = item
1695	else:
1696	temp_item = item.copy()
1697	temp_item.update(dm)
1698	results = import_method(temp_item)
1699	id = results[0]
1700	error = results[1]
1701	mapping = results[2]
1702	#
1703	# restore some values
1704	#
1705	for k in empty_value_keys:
1706	mapping[k] = EMPTY
1707	if mapping.has_key('sex'):
1708	#import pdb;pdb.set_trace()
1709	if mapping['sex'] == True:
1710	mapping['sex'] = 'F'
1711	elif mapping['sex'] == False:
1712	mapping['sex'] = 'M'
1713	if pending_only:
1714	info['imported_from'] = item['imported_from']
1715	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1716	info['error'] = error
1717	info['import_record_no'] = count + 1
1718	mapping.update(info)
1719	log_list = []
1720	if error:
1721	error_count += 1
1722	digest = makeDigest(mapping,data_keys)
1723	if digest not in pending_digests:
1724	pending_digests += digest,
1725	pending.append(mapping)
1726	if not pending_only:
1727	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1728	else:
1729	already_in += 1
1730	pass
1731	else:
1732	imported_count += 1
1733	imported += mapping,
1734	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1735	if log_list:
1736	time_till_now = time.time() - elapse
1737	percent_finished = (error_count + imported_count)/tti_float*100
1738	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1739	logger.info(' '.join(log_list))
1740	finished = count > total_to_import - 1
1741	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1742	if must_commit:
1743	old_commit_count = imported_count
1744
1745	if must_commit or finished:
1746	if len(imported):
1747	transaction.commit()
1748	datafile = open(imported_path,"a")
1749	writer = csv.DictWriter(datafile,
1750	csv_keys,
1751	extrasaction='ignore')
1752	writer.writerows(imported)
1753	datafile.close()
1754	total_added_to_imported += len(imported)
1755	imported = []
1756	if len(pending) > 0:
1757	datafile = open(pending_tmp,"a")
1758	writer = csv.DictWriter(datafile,
1759	csv_keys,
1760	extrasaction='ignore')
1761	writer.writerows(pending)
1762	datafile.close()
1763	total_pending += len(pending)
1764	#total_added_to_pending += len(pending)
1765	pending = []
1766	if not finished:
1767	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1768	logger.info(msg)
1769	elapse = time.time() - elapse
1770	if os.path.exists(pending_path):
1771	copy2(pending_path,pending_backup)
1772	copy2(pending_tmp,pending_path)
1773	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1774	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1775	if pending_only:
1776	removed_pending = pending_at_start - total_pending
1777	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1778	else:
1779	added_pending = total_pending - pending_at_start
1780	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1781	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1782	logger.info(msg)
1783	if import_source_done:
1784	copy(import_source_path,import_source_done)
1785	os.remove(import_source_path)
1786	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1787	if upload is not None:
1788	upload_doc = upload.getContent()
1789	mapping = {}
1790	#mapping['import_date'] = DateTime.DateTime()
1791	mapping['import_date'] = import_date
1792	mapping['imported_by'] = importer.imported_by
1793	mapping['import_message'] = msg
1794	upload_doc.edit(mapping = mapping)
1795	os.remove(pending_tmp)
1796	return msg
1797	###)
1798
1799	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1800	def moveImagesToFS(self,student_id="O738726"):
1801	"move the images to the filesystem"
1802	images_dir = getImagesDir(student_id)
1803	#images_dir = os.path.join("%s" % images_base,student_id)
1804	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1805	stool = getToolByName(self, 'portal_schemas')
1806	schemas = ['student_application',
1807	'student_clearance',
1808	]
1809	created = False
1810	for schema_id in schemas:
1811	schema = stool._getOb(schema_id)
1812	object = getattr(student_folder,schema_id[len('student_'):],None)
1813	if object is None:
1814	continue
1815	doc = object.getContent()
1816	for key in schema.keys():
1817	if schema[key].meta_type != "CPS Image Field":
1818	continue
1819	#import pdb;pdb.set_trace()
1820	image = getattr(doc,key,None)
1821	if not image or not hasattr(image,"data"):
1822	continue
1823	if not created:
1824	if not os.path.exists(images_dir):
1825	os.mkdir(images_dir)
1826	created = True
1827	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1828	open(filename,"wb").write(str(image.data))
1829	###)
1830
1831	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1832	def movePassportToFS(self,student_id="O738726"):
1833	"move the passports to the filesystem"
1834	images_dir = os.path.join("%s" % i_home,'passports')
1835	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1836	stool = getToolByName(self, 'portal_schemas')
1837	schemas = ['student_application',
1838	#'student_clearance',
1839	]
1840	created = False
1841	for schema_id in schemas:
1842	schema = stool._getOb(schema_id)
1843	object = getattr(student_folder,schema_id[len('student_'):],None)
1844	if object is None:
1845	continue
1846	doc = object.getContent()
1847	for key in schema.keys():
1848	if schema[key].meta_type != "CPS Image Field":
1849	continue
1850	#import pdb;pdb.set_trace()
1851	image = getattr(doc,key)
1852	if not hasattr(image,"data"):
1853	continue
1854	if not created:
1855	if not os.path.exists(images_dir):
1856	os.mkdir(images_dir)
1857	created = True
1858	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1859	open(filename,"wb").write(str(image.data))
1860	###)
1861
1862
1863	security.declarePublic('getConfigParams')
1864	def getConfigParams(self,conf_id="configuration"):
1865	conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
1866	conf_obj = conf.getContent()
1867	stool = getToolByName(self, 'portal_schemas')
1868	schema = stool._getOb('configuration')
1869	d = {}
1870	for key in schema.keys():
1871	d[key] = getattr(conf_obj,key,None)
1872	return d
1873
1874
1875	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: