Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 3811

Last change on this file since 3811 was 3807, checked in by Henrik Bettermann, 16 years ago
resolve ticket #589 part 2
Property svn:keywords set to `Id`
File size: 75.3 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3807 2008-12-08 16:12:01Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108	###)
109
110	security.declareProtected(View,'updateRoleMappingsFor') ###(
111	def updateRoleMappingsFor(self,wf_definition,ob):
112	"do so for public"
113	wf_def = getattr(self.portal_workflow,wf_definition)
114	wf_def.updateRoleMappingsFor(ob)
115	###)
116
117	security.declareProtected(View,'getStatesLgas') ###(
118	def getStatesLgas(self):
119	"""return lga info"""
120	voc = getattr(self.portal_vocabularies,'local_gov_areas')
121	states = []
122	lgas = []
123	d = {}
124	wd = {}
125	for k,v in voc.items():
126	parts = v.split(' / ')
127	if len(parts) == 1:
128	state = parts[0].lower()
129	lga = ""
130	elif len(parts) == 2:
131	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133	else:
134	continue
135	if state not in states:
136	states.append(state)
137	if lga not in lgas:
138	lgas.append(lga)
139	words = re.split('[^a-zA-Z0-9/]',k)
140	words.sort()
141	wd[k] = words
142	d[k] = v
143	mapping = {}
144	mapping['word_dict'] = wd
145	mapping['lga_dict'] = d
146	mapping['states'] = states
147	mapping['lgas'] = lgas
148	return mapping
149	###)
150
151	security.declareProtected(View,'findLga') ###(
152	def findLga(self,words,words_dict):
153	words = re.split('[^a-zA-Z0-9/]',words)
154	lga_words = []
155	for word in words:
156	if word:
157	lga_words += word.strip().lower(),
158	lga_words.sort()
159	state_lga = ''
160	while not state_lga:
161	for k,l in words_dict.items():
162	if lga_words == l:
163	state_lga = k
164	break
165	break
166	return state_lga
167	###)
168	security.declareProtected(View,'getAccessInfo') ###(
169	def getAccessInfo(self,context):
170	"return a dict with access_info"
171	logger = logging.getLogger('WAeUPTool.getAccessInfo')
172	mtool = self.portal_membership
173	member = mtool.getAuthenticatedMember()
174	member_id = str(member)
175	info = {}
176	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178	is_staff = info['is_staff'] = not is_anonymous and not is_student
179	roles = member.getRolesInContext(context)
180	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181	"SectionManager" in roles or
182	"Manager" in roles)
183	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184	is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if is_allowed:
188	if not is_student and requested_id:
189	student_id = requested_id
190	elif not is_allowed and (not is_staff or member_id != requested_id):
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	else:
193	student_id = member_id
194	info['student_id'] = student_id
195	return info
196	###)
197
198	security.declareProtected(ModifyPortalContent,'openLog') ###(
199	def openLog(self,name):
200	"""open a log file"""
201	version = 1
202	path = "%s/log/%s_%d.log" % (i_home,name,version)
203	while os.path.exists(path):
204	version += 1
205	path = "%s/log/%s_%d.log" % (i_home,name,version)
206	log = open(path,"w")
207	return log
208	###)
209
210	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211	def bypassQueueCatalog(self,enable=True):
212	"""bypass the QueueCatalog by setting all indexes to process imediate,
213	if enable is True (default) the old settings are restored
214	"""
215
216	###)
217
218	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220	"""measure Object access Time"""
221	import random
222	if hasattr(self,'portal_catalog_real'):
223	aq_portal = self.portal_catalog_real.evalAdvancedQuery
224	else:
225	aq_portal = self.portal_catalog.evalAdvancedQuery
226	nr_pts = int(nr_pts)
227	probe = int(probe)
228	intervall = probe/10
229	objects = ("application","clearance","personal")
230	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231	#i = random.randrange(num_objects)
232	count = 0
233	found = 0
234	not_found = 0
235	t_found = 0
236	t_not_found = 0
237	time_found = time_not_found = 0.0
238	t_time_found = t_time_not_found = 0.0
239	accessed = []
240	t_min = 1000
241	t_max = 0
242	#import pdb;pdb.set_trace()
243	students = self.portal_catalog(portal_type="Student")
244	num_students = len(students)
245	if method == "d":
246	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247	res = aq_portal(query)
248	brains = {}
249	for r in res:
250	sid = r.relative_path.split('/')[-2]
251	if brains.has_key(sid):
252	brains[sid][r.portal_type] = r
253	else:
254	brains[sid] = {r.portal_type : r}
255	brains_list = brains.keys()
256	num_objects = len(brains_list)
257	else:
258	num_objects = num_students
259	print "="*40
260	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261	probe,
262	nr_pts,
263	num_objects)
264	print "nr found/not time found/not min/max"
265	elapse = time.time()
266	i_elapse = time.time()
267	c_elapse = time.clock()
268	for c in range(1,probe + 1):
269	i = random.randrange(num_objects)
270	if method in ('a','b','c'):
271	student_brain = students[i]
272	elif method == "d":
273	#import pdb;pdb.set_trace()
274	student_brain = brains[brains_list[i]]
275	if method == "c":
276	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277	res = aq_portal(query)
278	this_portal_types = [r.portal_type for r in res]
279	for i in range(nr_pts):
280	oid = objects[i]
281	if method == "a":
282	try:
283	student_path = student_brain.getPath()
284	path = "%s/%s" % (student_path,oid)
285	doc = self.unrestrictedTraverse(path).getContent()
286	found += 1
287	i_time = time.time() - i_elapse
288	time_found += i_time
289	except:
290	not_found += 1
291	i_time = time.time() - i_elapse
292	time_not_found += i_time
293	pass
294	elif method == "b":
295	try:
296	student_object = student_brain.getObject()
297	doc = getattr(student_object,oid).getContent()
298	found += 1
299	i_time = time.time() - i_elapse
300	time_found += i_time
301	except:
302	i_time = time.time() - i_elapse
303	time_not_found += i_time
304	not_found += 1
305	pass
306	elif method == "c":
307	if portal_types[i] in this_portal_types:
308	found += 1
309	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310	i_time = time.time() - i_elapse
311	time_found += i_time
312	else:
313	not_found += 1
314	i_time = time.time() - i_elapse
315	time_not_found += i_time
316	elif method == "d":
317	if student_brain.has_key(portal_types[i]):
318	found += 1
319	doc = student_brain[portal_types[i]].getObject().getContent()
320	i_time = time.time() - i_elapse
321	time_found += i_time
322	else:
323	not_found += 1
324	i_time = time.time() - i_elapse
325	time_not_found += i_time
326	i_elapse = time.time()
327	if c and (c % intervall == 0):
328	#i_time = time.time() - i_elapse
329	t_per = 0.0
330	if found:
331	t_per = time_found/found
332	if t_per > t_max:
333	t_max = t_per
334	if t_per > 0.0 and t_per < t_min:
335	t_min = t_per
336	itf = 0.0
337	if found:
338	itf = time_found/found
339	itnf = 0.0
340	if not_found :
341	itnf = time_not_found / not_found
342	interval_time = time_found + time_not_found
343	s = "%(c)d: %(found)d/%(not_found)d " % vars()
344	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
346	print s
347	t_found += found
348	t_not_found += not_found
349	t_time_found += time_found
350	t_time_not_found += time_not_found
351	time_found = time_not_found = 0.0
352	found = not_found = 0
353	# t_found += found
354	# t_not_found += not_found
355	elapse = time.time() - elapse
356	itf = 0.0
357	if t_found:
358	itf = t_time_found/t_found
359	itnf = 0.0
360	if t_not_found:
361	itnf = t_time_not_found / t_not_found
362	#c_elapse = time.clock() - c_elapse
363	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
366	print "-"*40
367	print s
368	rel_found = float(t_found)/probe
369	rel_not_found = float(t_not_found)/probe
370	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
371	print estimated_total_time
372	###)
373
374	security.declareProtected(ModifyPortalContent,'writeLog') ###(
375	def writeLog(self,logfile,s):
376	"""write to the log file"""
377	logfile.write(s)
378	###)
379
380	def generateStudentId(self,letter): ###(
381	import random
382	r = random
383	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
384	if letter == '?':
385	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
386	sid = "%c%d" % (letter,r.randint(99999,1000000))
387	students = self.portal_url.getPortalObject().campus.students
388	# while hasattr(students, sid):
389	# sid = "%c%d" % (letter,r.randint(99999,1000000))
390	while self.students_catalog(id = sid):
391	sid = "%c%d" % (letter,r.randint(99999,1000000))
392	return sid
393	###)
394
395	def generatePassword(self,s=None): ###(
396	import random
397	r = random
398	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
399	if s is None:
400	s = 'abcdefghklmnpqrstuvwxy23456789'
401	pw = ''
402	while len(pw) < 6:
403	pw += r.choice(s)
404	return pw
405	###)
406
407	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
408	def dumpSchoolfeePayments(self):
409	"dump paid schoolfees"
410	mtool = self.portal_membership
411	member = mtool.getAuthenticatedMember()
412	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
413	aq_student = self.students_catalog.evalAdvancedQuery
414	query = In('review_state',('schoolfee_paid',
415	'courses_registered',
416	'courses_validated',
417	))
418	res = aq_student(query)
419	#import pdb;pdb.set_trace()
420	l = []
421	logger.info("start for %d" % len(res))
422	count = 1
423	log_after = 100
424	for student in res:
425	if not count % log_after:
426	logger.info("processed %d total %d" % (log_after,count))
427	count += 1
428	fee_dict =self.getSchoolFee(student)
429	fulltime = student.mode.endswith('_ft')
430	d = {}
431	d['student_id'] = student.id
432	d['name'] = student.name
433	d['amount'] = fee_dict.get(new_returning)
434	l += d,
435	csv_name = self.dumpListToCSV(l,'payments')
436	logger.info('%s dumped payments to %s' % (member,export_file))
437	###)
438
439	security.declarePublic('dumpListToCSV') ###(
440	def dumpListToCSV(self,l,filename,fields=None):
441	"""dump a list of dicts to a CSV file"""
442	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
443	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
444	if fields is None:
445	fields = l[0].keys()
446	headline = ','.join(fields)
447	out = open(export_file,"wb")
448	out.write(headline +'\n')
449	out.close()
450	out = open(export_file,"a")
451	csv_writer = csv.DictWriter(out,fields,)
452	csv_writer.writerows(l)
453	return export_file
454	###)
455
456	security.declareProtected(ManagePortal, 'listMembers') ###(
457	def listMembers(self):
458	"list all members"
459	mtool = self.portal_membership
460	member = mtool.getAuthenticatedMember()
461	logger = logging.getLogger('WAeUPTool.listMembers')
462	if str(member) not in ('admin','joachim'):
463	logger.info('%s tried to list members' % (member))
464	return None
465	members = self.portal_directories.members
466	all = members.listEntryIdsAndTitles()
467	l = []
468	for user_id,name in all:
469	d = {}
470	d['user_id'] = user_id
471	d['name'] = name
472	d['pw'] = getattr(getattr(members,user_id),'password')
473	d['email'] = getattr(getattr(members,user_id),'email')
474	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
475	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
476	l += d,
477	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
478	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
479	logger.info('%s dumped member list to %s' % (member,export_file))
480	fields = l[0].keys()
481	headline = ','.join(fields)
482	out = open(export_file,"wb")
483	out.write(headline +'\n')
484	out.close()
485	out = open(export_file,"a")
486	csv_writer = csv.DictWriter(out,fields,)
487	csv_writer.writerows(l)
488	###)
489
490	security.declareProtected(ManagePortal, 'listStudents') ###(
491	def listStudents(self):
492	"list all students"
493	mtool = self.portal_membership
494	member = mtool.getAuthenticatedMember()
495	logger = logging.getLogger('WAeUPTool.listStudents')
496	if str(member) not in ('admin','joachim'):
497	logger.info('%s tried to list students' % (member))
498	return None
499	students = self.portal_directories.students
500	all = students.listEntryIdsAndTitles()
501	l = []
502	for user_id,name in all:
503	d = {}
504	d['user_id'] = user_id
505	d['name'] = name
506	d['pw'] = getattr(getattr(students,user_id),'password')
507	d['email'] = getattr(getattr(students,user_id),'email')
508	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
509	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
510	l += d,
511	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
512	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
513	logger.info('%s dumped student list to %s' % (member,export_file))
514	fields = l[0].keys()
515	headline = ','.join(fields)
516	out = open(export_file,"wb")
517	out.write(headline +'\n')
518	out.close()
519	out = open(export_file,"a")
520	csv_writer = csv.DictWriter(out,fields,)
521	csv_writer.writerows(l)
522	###)
523
524	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
525	def removeDeletedDocIds(self, max=1000):
526	"""
527	remove deleted docids from repository commit after max
528	"""
529	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
530	repository = getToolByName(self, 'portal_repository')
531	pxtool = getToolByName(self, 'portal_proxies')
532	logger.info('start')
533	pxtool_infos = pxtool.getRevisionsUsed()
534	logger.info('found %d used revisions ' % (len(pxtool_infos)))
535
536	nb_revs = 0
537	docids_d = {} # all docids
538	unused_docids_d = {} # all docids that are unused
539	ids_unused_revs_docids = [] # ids for revs of unused docids
540	unused_ids = [] # ids for unused revs
541	total = 0
542	idlist = repository.objectIds()
543	to_delete = 0
544	found = False
545	for id in idlist:
546	docid, rev = repository._splitId(id)
547	if docid is None:
548	logger.info("invalid doc_id %s" % docid)
549	continue
550	nb_revs += 1
551	if not pxtool_infos.has_key(docid):
552	found = True
553	to_delete += 1
554	unused_ids.append(id)
555	elif not pxtool_infos[docid].has_key(rev):
556	found = True
557	to_delete += 1
558	unused_ids.append(id)
559	if found and not to_delete % max:
560	found = False
561	#import pdb;pdb.set_trace()
562	repository.manage_delObjects(unused_ids)
563	transaction.commit()
564	logger.info('removed %d total %d unused docids ' % (max,to_delete))
565	else:
566	if unused_ids:
567	repository.manage_delObjects(unused_ids)
568	transaction.commit()
569	logger.info('finished removing %d unused docids ' % (to_delete))
570	###)
571
572	security.declareProtected(View,'getCredential') ###(
573	def getCredential(self,student_id):
574	student_entry = getattr(self.portal_directories.students,student_id,None)
575	if not self.isStaff():
576	mtool = self.portal_membership
577	member = mtool.getAuthenticatedMember()
578	logger = logging.getLogger('WAeUPTool.getCredential')
579	logger.info('%s tried to access password of %s' % (member,student_id))
580	return None
581	if student_entry is None:
582	return None
583	return getattr(student_entry,"password","not set")
584	###)
585
586	security.declarePublic('checkPassword') ###(
587	def checkPassword(self,student_id,password):
588	student_entry = getattr(self.portal_directories.students,student_id,None)
589	if student_entry is None:
590	return False
591	return getattr(student_entry,"password","not set") == password
592	###)
593
594	security.declarePublic('checkGenericPassword') ###(
595	def checkGenericPassword(self,member_id):
596	member_entry = getattr(self.portal_directories.members,member_id,None)
597	if member_entry is None:
598	return False
599	ltool = getToolByName(self, 'portal_layouts')
600	unsecure_words = ltool._getOb('members')['w__password'].check_words
601	password = getattr(member_entry,"password","not set")
602	is_unsecure = password in unsecure_words
603	if is_unsecure:
604	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
605	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
606	return is_unsecure
607	###)
608
609	security.declareProtected(ModifyPortalContent,'editPassword') ###(
610	def editPassword(self,student_id,password):
611	"edit a student password"
612	student_entry = getattr(self.portal_directories.students,student_id,None)
613	if student_entry is None:
614	return
615	setattr(student_entry,'password',password)
616	###)
617
618	security.declareProtected(ModifyPortalContent,'doCommit') ###(
619	def doCommit(self,logger=None):
620	"commit some transactions"
621	transaction.commit()
622	###)
623
624	security.declarePublic('loadStudentFoto') ###(
625	def loadStudentFoto(self,student,filename,folder):
626	"return a student passport picture"
627	#import pdb;pdb.set_trace()
628	picture ="%s/import/%s/%s" % (i_home,folder,filename)
629	student_id = student.getId()
630	images_dir = getImagesDir(student_id)
631	if not os.path.exists(images_dir):
632	os.mkdir(images_dir)
633	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
634	for extension in ('.jpg','.JPG'):
635	fullname = "%(picture)s%(extension)s" % vars()
636	if os.path.exists(fullname):
637	copy2(fullname,image_name)
638	return "successfully copied passport picture"
639	return "passport picture not found: %s.jpg or .JPG" % picture
640	###)
641
642	def old____loadStudentFoto(self,student,filename,folder): ###(
643	"return a student passport picture"
644	app = student.application
645	app_doc = app.getContent()
646	#clear = student.clearance
647	#clear_doc = clear.getContent()
648	#matric_no = clear_doc.matric_no.upper()
649	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
650	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
651	#import pdb;pdb.set_trace()
652	if os.path.exists(picture1):
653	file = open(picture1)
654	elif os.path.exists(picture2):
655	file = open(picture2)
656	else:
657	return "passport picture not found %s" % picture1
658	reopened = False
659	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
660	self.portal_workflow.doActionFor(app,'open')
661	reopened = True
662	outfile = file.read()
663	app_doc.manage_addFile('passport',
664	file=outfile,
665	title="%s.jpg" % filename)
666	if reopened:
667	self.portal_workflow.doActionFor(app,'close')
668	return "successfully loaded passport picture"
669	###)
670
671	security.declareProtected(ModifyPortalContent,'createOne') ###(
672	def createOne(self,students_folder,student_brain,letter,commit=False):
673	sid = self.waeup_tool.generateStudentId(letter)
674	students_folder.invokeFactory('Student', sid)
675	student = getattr(students_folder,sid)
676	self.portal_workflow.doActionFor(student,'return')
677	student.manage_setLocalRoles(sid, ['Owner',])
678	matric_no = student_brain.matric_no
679	jamb_reg_no = student_brain.Entryregno
680	self.students_catalog.addRecord(id = sid,
681	matric_no = matric_no,
682	jamb_reg_no = jamb_reg_no,
683	sex = student_brain.Sex == "F",
684	name = "%s %s %s" % (student_brain.Firstname,
685	student_brain.Middlename,
686	student_brain.Lastname)
687	)
688	if commit:
689	transaction.commit()
690	return sid,jamb_reg_no
691	###)
692
693	security.declareProtected(ModifyPortalContent,'addStudent') ###(
694	def addStudent(self,dict):
695	students_folder = self.portal_url.getPortalObject().campus.students
696	sid = self.waeup_tool.generateStudentId('?')
697	students_folder.invokeFactory('Student', sid)
698	student_obj = getattr(students_folder,sid)
699	f2t = StudentImport.field2types_student
700	#from pdb import set_trace; set_trace()
701	d = {}
702	#d['jamb_sex'] = 'M'
703	#if dict.get('sex'):
704	# d['jamb_sex'] = 'F'
705
706	entry_session = dict.get('entry_session')
707	if entry_session == self.getSessionId()[0]:
708	wfaction = 'admit'
709	wft = 'wf_transition_admit'
710	password = None
711	else:
712	wfaction = 'return'
713	wft = 'wf_transition_return'
714	password = self.generatePassword()
715	self.makeStudentMember(sid,password)
716
717	for pt in f2t.keys():
718	student_obj.invokeFactory(pt,f2t[pt]['id'])
719	sub_obj = getattr(student_obj,f2t[pt]['id'])
720	sub_doc = sub_obj.getContent()
721	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
722	#d['Title'] = f2t[pt]['title']
723	for field in f2t[pt]['fields']:
724	d[field] = dict.get(field,'')
725	sub_doc.edit(mapping = d)
726	new_state = f2t[pt][wft]
727	if new_state != "remain":
728	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
729	self.portal_workflow.doActionFor(student_obj,wfaction)
730	student_obj.manage_setLocalRoles(sid, ['Owner',])
731	return sid,password
732	###)
733
734	security.declarePublic('getCertificateBrain') ###(
735	def getCertificateBrain(self,cert_id):
736	"do it"
737	res = ZCatalog.searchResults(self.portal_catalog_real,
738	{'portal_type':"Certificate",
739	'id': cert_id})
740	if res:
741	return res[0]
742	return None
743	###)
744
745	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
746	def get_csv_filenames(self):
747	"do it"
748	files = [file for file in os.listdir("%s/import/" % (i_home))
749	if file.endswith('.csv') and (file.find('imported') == -1 and
750	file.find('pending') == -1)]
751	return files
752	###)
753
754	security.declarePublic('findStudentByMatricelNo') ###(
755	def findStudentByMatricelNo(self,matric_no):
756	"do it"
757	res = ZCatalog.searchResults(self.portal_catalog_real,
758	{'portal_type':"StudentClearance",
759	'SearchableText': matric_no})
760	if res:
761	return res[0]
762	return None
763	###)
764
765	security.declarePublic('makeStudentMember') ###(
766	def makeStudentMember(self,sid,password='uNsEt'):
767	"""make the student a member"""
768	membership = self.portal_membership
769	membership.addMember(sid,
770	password ,
771	roles=('Member',
772	'Student',
773	),
774	domains='',
775	properties = {'memberareaCreationFlag': False,
776	'homeless': True},)
777	member = membership.getMemberById(sid)
778	self.portal_registration.afterAdd(member, sid, password, None)
779	#self.manage_setLocalRoles(sid, ['Owner',])
780	###)
781
782	security.declareProtected(View,'makeStudentData') ###(
783	def makeStudentData(self,student_id,email=None,phone_nr=None):
784	"create Datastructure for a returning Student"
785	#import pdb;pdb.set_trace()
786	logger = logging.getLogger('WAeUPTool.makeStudentData')
787	students_folder = self.portal_url.getPortalObject().campus.students
788	#res = self.students_catalog(id=student_id)
789	#if res:
790	# st = res[0]
791	#res = self.returning_import(matric_no = st.matric_no)
792	res = self.returning_import(id = student_id)
793	if res:
794	student = res[0]
795	else:
796	logger.info('Id %s not found in returning_import' % student_id)
797	return
798	logger.info('%s creates data structure' % student_id)
799	s_results = self.results_import(matric_no = student.matric_no)
800	if s_results:
801	lnr = self.getLevelFromResultsCosCode(s_results)
802	level = "%d00" % lnr
803	verdict,eligible = self.getVerdict(s_results[0].Verdict)
804	#if eligible:
805	# level = "%d00" % (lnr + 1)
806	else:
807	logger.info('matric_no %s not found in results_import' % student.matric_no)
808	level = ''
809	verdict = ''
810	#student should not be allowed to perform this transition
811	#wftool = self.portal_workflow
812	#wftool.doActionFor(student,'return')
813	certcode_org = student.Coursemajorcode
814	certcode = makeCertificateCode(certcode_org)
815	certificate_brain = self.getCertificateBrain(certcode)
816	if not certificate_brain:
817	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
818	logger.info(em)
819	matric_no = student.matric_no
820	sid = student_id
821	student_obj = getattr(students_folder,sid)
822	if not getattr(student_obj,'application'):
823	student_obj.invokeFactory('StudentApplication','application')
824	application = student_obj.application
825	self.portal_workflow.doActionFor(application,'open',dest_container=application)
826	da = {'Title': 'Application Data'}
827	student_obj.invokeFactory('StudentPersonal','personal')
828	da['jamb_reg_no'] = student.Entryregno
829	em = self.getEntryMode(student.Entryregno)
830	da['entry_mode'] = em
831	personal = student_obj.personal
832	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
833	dp = {'Title': 'Personal Data'}
834	student_obj.invokeFactory('StudentClearance','clearance')
835	clearance = student_obj.clearance
836	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
837	dc = {'Title': 'Clearance/Eligibility Record'}
838	dc['matric_no'] = matric_no
839	state = student.State
840	lga = student.LGA
841	if state and lga:
842	lga = state + ' / ' + lga
843	else:
844	lga = "None"
845	da['jamb_lga'] = dc['lga'] = lga
846	da['app_email'] = dp['email'] = email
847	da['app_mobile'] = dp['phone'] = phone_nr
848	dp['firstname'] = student.Firstname
849	dp['middlename'] = student.Middlename
850	dp['lastname'] = student.Lastname
851	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
852	da['jamb_sex'] = student.Sex
853	dp['sex'] = student.Sex == 'F'
854	dp['perm_address'] = student.Permanent_Address
855	application.getContent().edit(mapping=da)
856	self.portal_workflow.doActionFor(application,'close',dest_container=application)
857	personal.getContent().edit(mapping=dp)
858	clearance.getContent().edit(mapping=dc)
859	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
860	#
861	# Study Course
862	#
863	student_obj.invokeFactory('StudentStudyCourse','study_course')
864	studycourse = student_obj.study_course
865	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
866	dsc = {}
867	dsc['study_course'] = certcode
868	dsc['current_level'] = level
869	dsc['current_verdict'] = verdict
870	dsc['current_mode'] = em #no longer used
871	dsc['current_session'] = '05'
872	studycourse.getContent().edit(mapping=dsc)
873	#
874	# Level
875	#
876	# l = getattr(studycourse,level,None)
877	# if l is None:
878	# studycourse.invokeFactory('StudentStudyLevel', level)
879	# l = getattr(studycourse, level)
880	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
881	# l.getContent().edit(mapping={'Title': "Level %s" % level})
882	###)
883
884	def init_timing(self): ###(
885	if self.with_timing:
886	if not hasattr(self,'_v_step_times'):
887	self._v_timer_count = 0
888	self._v_total = 0
889	self._v_step_times = {}
890	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
891	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
892	self.timer_step = 0
893	self.total_time = 0
894	self.elapse = time.time()
895	self.i_elapse = time.time()
896	self.c_elapse = time.clock()
897	###)
898
899	def do_timing(self): ###(
900	if self.with_timing:
901	try:
902	raise 'dummy'
903	except:
904	frame = sys.exc_traceback.tb_frame.f_back
905	locals = frame.f_locals
906	globals = frame.f_globals
907	functionname = frame.f_code.co_name
908	filename = os.path.basename(frame.f_code.co_filename)
909	lineno = frame.f_lineno
910	mod_line = "%(functionname)s:%(lineno)s" % vars()
911	i_time = time.time() - self.i_elapse
912	td = {}
913	if self._v_step_times.has_key(mod_line):
914	a_time = self._v_step_times[mod_line]['a_time'] + i_time
915	td['a_time'] = a_time
916	else:
917	td['a_time'] = i_time
918	td['i_time'] = i_time
919	self._v_step_times[mod_line] = td
920	self.i_time = i_time
921	self.total_time += i_time
922	self.timer_step +=1
923	self.i_elapse = time.time()
924	###)
925
926	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
927	def print_timing(self):
928	if self.with_timing:
929	l = []
930	timer_count = self._v_timer_count + 1
931	mod_lines = self._v_step_times.keys()
932	mod_lines.sort(cmp,reverse=0)
933	for mod_line in mod_lines:
934	td = self._v_step_times[mod_line]
935	i_time = td['i_time']
936	a_time = td['a_time']/(self._v_timer_count + 1)
937	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
938	total_time = self.total_time
939	total_avarage = self._v_total / timer_count
940	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
941	print "\r\n".join(l)
942	out = open(self._v_timer_file,'a')
943	out.write("\r\n".join(l))
944	out.close()
945	###)
946
947	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
948	def get_timing_data(self):
949	if self.with_timing:
950	timer_count = self._v_timer_count + 1
951	results = {}
952	for k,d in self._v_step_times.items():
953	dd = {}
954	dd['a_time'] = d['a_time']/timer_count
955	dd['i_time'] = d['i_time']
956	dd['count'] = timer_count
957	results[k] = dd
958	dd = {}
959	dd['a_time'] = self._v_total / timer_count
960	dd['i_time'] = self.total_time
961	dd['count'] = timer_count
962	results["total"] = dd
963	return results
964	###)
965
966	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
967	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
968	"create Datastructure for an admitted Student"
969	#import pdb;pdb.set_trace()
970	logger = logging.getLogger('WAeUPTool.admitOneStudent')
971	self.with_timing = with_timing
972
973	if brain.screening_type in ('cest','sandwich',):
974	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
975	else:
976	reg_no = brain.reg_no
977
978	#ignore argument entry_session
979	if not brain.entry_session:
980	logger.info('no entry_session for %s provided' % (reg_no))
981	return
982
983	if not hasattr(self,"_v_certificates"):
984	self._v_certificates = self.getCertificatesDict()
985	students_folder = self.portal_url.getPortalObject().campus.students
986
987	res = self.students_catalog(jamb_reg_no = reg_no)
988	if res:
989	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
990	return
991	if brain.status != "admitted":
992	logger.info('status of %s is %s' % (reg_no,brain.status))
993	return
994	pin_parts = brain.pin.split('-')
995	if pin_parts and len(pin_parts) != 3:
996	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
997	return
998	if not brain.course_admitted:
999	logger.info('no course_admitted provided for %s' % (reg_no))
1000	return
1001	if brain.course_admitted not in self._v_certificates:
1002	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
1003	return
1004	if brain.sex not in (True,False):
1005	logger.info('sex of %s not available' % (reg_no))
1006	return
1007	self.init_timing()
1008	student_id = self.generateStudentId('?')
1009	students_folder.invokeFactory('Student', student_id)
1010	student_object = getattr(students_folder,student_id)
1011	self.do_timing()
1012	if pin_password:
1013	password = pin_parts[2]
1014	self.makeStudentMember(student_id,password = password)
1015	student_object.manage_setLocalRoles(student_id, ['Owner',])
1016	self.do_timing()
1017	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1018	#
1019	# application
1020	#
1021	student_object.invokeFactory('StudentApplication','application')
1022	application = student_object.application
1023	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1024	#self.do_timing()
1025	da = {'Title': 'Application Data'}
1026	da['jamb_reg_no'] = reg_no
1027
1028	sex = 'M'
1029	if brain.sex:
1030	sex = 'F'
1031	da['jamb_sex'] = sex
1032	da['jamb_age'] = brain.jamb_age
1033	da['app_reg_pin'] = brain.pin
1034	da['jamb_lga'] = brain.jamb_lga
1035	da['jamb_state'] = brain.jamb_state
1036	da['jamb_score'] = brain.aggregate
1037	da['app_email'] = brain.email
1038	da['app_mobile'] = brain.phone
1039
1040	# entry_mode cannot be retrieved from the certtificate
1041	# because ug_ft has two different entry modes
1042
1043	if brain.entry_mode: # does not happen because import_application
1044	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
1045	elif brain.screening_type == 'pume':
1046	da['entry_mode'] = 'ume_ft'
1047	elif brain.screening_type == 'pde':
1048	da['entry_mode'] = 'de_ft'
1049	else:
1050	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1051
1052	#elif brain.screening_type == 'pce':
1053	# da['entry_mode'] = 'pce'
1054	#elif brain.screening_type == 'prence':
1055	# da['entry_mode'] = 'prence'
1056	#else:
1057	# da['entry_mode'] = ''
1058
1059
1060	#da['entry_session'] = entry_session
1061	da['entry_session'] = brain.entry_session
1062	da['jamb_lastname'] = brain.lastname
1063	da['jamb_middlename'] = brain.middlenames # different field names!
1064	da['jamb_firstname'] = brain.firstname
1065	da['screening_application_date'] = brain.application_date
1066	da['date_of_birth'] = brain.date_of_birth
1067	da['jamb_first_cos'] = brain.course1
1068	da['jamb_second_cos'] = brain.course2
1069	da['course3'] = brain.course3
1070	da['screening_type'] = brain.screening_type
1071	da['screening_score'] = brain.screening_score
1072	da['screening_date'] = brain.screening_date
1073	da['hq_type'] = brain.hq_type
1074	da['hq_grade'] = brain.hq_grade
1075	da['aos'] = brain.aos
1076
1077	application.getContent().edit(mapping=da)
1078	self.do_timing()
1079	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1080	#
1081	# personal
1082	#
1083	student_object.invokeFactory('StudentPersonal','personal')
1084	personal = student_object.personal
1085	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1086	#self.do_timing()
1087	dp = {'Title': 'Personal Data'}
1088	dp['sex'] = brain.sex
1089	dp['email'] = brain.email
1090	dp['phone'] = brain.phone
1091	dp['lastname'] = brain.lastname
1092	dp['middlename'] = brain.middlenames # different field names!
1093	dp['firstname'] = brain.firstname
1094	personal.getContent().edit(mapping=dp)
1095	self.do_timing()
1096	#
1097	# clearance
1098	#
1099	student_object.invokeFactory('StudentClearance','clearance')
1100	clearance = student_object.clearance
1101	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1102	dc = {'Title': 'Clearance/Eligibility Record'}
1103	dc['lga'] = brain.lga
1104	dc['birthday'] = brain.date_of_birth
1105	clearance.getContent().edit(mapping=dc)
1106	self.do_timing()
1107	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1108	#
1109	# study Course
1110	#
1111	student_object.invokeFactory('StudentStudyCourse','study_course')
1112	studycourse = student_object.study_course
1113	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1114	#self.do_timing()
1115	dsc = {}
1116	dsc['study_course'] = brain.course_admitted
1117	dsc['current_verdict'] = ''
1118	dsc['current_mode'] = da['entry_mode'] # no longer used
1119	if da['entry_mode'].startswith('de'):
1120	dsc['current_level'] = '200'
1121	elif da['entry_mode'].startswith('pre'):
1122	dsc['current_level'] = '000'
1123	else:
1124	dsc['current_level'] = '100'
1125	dsc['current_session'] = brain.entry_session
1126	studycourse.getContent().edit(mapping=dsc)
1127	self.do_timing()
1128	#
1129	# payments folder
1130	student_object.invokeFactory('PaymentsFolder','payments')
1131	payments = getattr(student_object,'payments')
1132	#self.do_timing()
1133	dpay = {}
1134	dpay['Title'] = 'Payments'
1135	payments.getContent().edit(mapping=dpay)
1136	self.portal_workflow.doActionFor(payments,'open')
1137	self.do_timing()
1138	#
1139	# passport foto
1140	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1141	brain.screening_type,
1142	brain.reg_no)
1143	images_dir = getImagesDir(student_id)
1144	#images_dir = os.path.join("%s" % images_base,student_id)
1145	letter_dir,student_dir = os.path.split(images_dir)
1146	if not os.path.exists(letter_dir):
1147	os.mkdir(letter_dir)
1148	if not os.path.exists(images_dir):
1149	os.mkdir(images_dir)
1150	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1151	if os.path.exists(app_picture):
1152	copy2(app_picture,image_name)
1153	else:
1154	logger.info('passport of %s/%s not found: %s' % (student_id,
1155	brain.reg_no,
1156	app_picture))
1157
1158	self.do_timing()
1159	self.print_timing()
1160	if with_timing:
1161	self.timer_step = 0
1162	self._v_timer_count += 1
1163	self._v_total += self.total_time
1164	return student_id
1165	###)
1166
1167	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1168	def makeStudentLevel(self,student_id):
1169	"create the StudyLevel for a returning Student"
1170	#import pdb;pdb.set_trace()
1171	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1172	students_folder = self.portal_url.getPortalObject().campus.students
1173	res = self.students_catalog(id=student_id)
1174	if res:
1175	st = res[0]
1176	course = st.course
1177	matric_no = st.matric_no
1178	level = st.level
1179	res = self.results_import(matric_no = matric_no)
1180	if res:
1181	results = res
1182	logger.info('%s creating Level %s' % (student_id,level))
1183	#
1184	# Level
1185	#
1186	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1187	studycourse = getattr(student_obj,"study_course",None)
1188	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1189	l = getattr(studycourse,level,None)
1190	if l is None:
1191	studycourse.invokeFactory('StudentStudyLevel', level)
1192	l = getattr(studycourse, level)
1193	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1194	l.getContent().edit(mapping={'Title': "Level %s" % level})
1195	###)
1196
1197	security.declarePublic('getHallInfo') ###(
1198	def getHallInfo(self,bed):
1199	"""return Hall Info"""
1200	info = {}
1201	bedsplit = bed.split('_')
1202	if len(bedsplit) == 4:
1203	hall,block,room,letter = bed.split('_')
1204	else:
1205	info['maintenance_code'] = 'None'
1206	return info
1207	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1208	if res and len(res) == 1:
1209	hall_brain = res[0]
1210	hall_doc = hall_brain.getObject().getContent()
1211	else:
1212	return info
1213	info['hall_title'] = hall_brain.Title
1214	info['maintenance_code'] = hall_doc.maintenance_code
1215	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1216	batch_doc = None
1217	for brain in res:
1218	if brain.id.startswith(info['maintenance_code']):
1219	batch_doc = brain.getObject().getContent()
1220	break
1221	if batch_doc is None:
1222	info['maintenance_fee'] = ''
1223	else:
1224	info['maintenance_fee'] = batch_doc.cost
1225	return info
1226	###)
1227
1228	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1229	def removePictureFolder(self,student_id):
1230	"""remove picture_folder by renaming it"""
1231	picture_path = getImagesDir(student_id)
1232	dest_path = os.path.join("%s" % images_base,'removed',student_id)
1233	dest_path = dest_path + "_removed"
1234	if os.path.exists(dest_path) or not os.path.exists(picture_path):
1235	return False
1236	os.rename(picture_path,dest_path)
1237	return True
1238	###)
1239
1240	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1241	def restorePictureFolder(self,student_id):
1242	"""restore picture_folder by renaming it"""
1243	picture_path = getImagesDir(student_id)
1244	orig_path = os.path.join("%s" % images_base,'removed',student_id)
1245	orig_path = orig_path + "_removed"
1246	if os.path.exists(picture_path) or not os.path.exists(orig_path):
1247	return False
1248	os.rename(picture_path + "_removed",picture_path)
1249	return True
1250	###)
1251
1252	security.declarePublic('picturesExist') ###(
1253	def picturesExist(self, ids,student_id=None):
1254	"""check if pictures exist in the filesystem"""
1255	if student_id is None:
1256	student_id = self.getStudentId()
1257	if student_id is None:
1258	return False
1259	picture_path = getImagesDir(student_id)
1260	#picture_path = os.path.join(images_base,student_id)
1261	if not os.path.exists(picture_path):
1262	return False
1263	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1264	return set(ids).issubset(set(pictures))
1265	###)
1266
1267
1268	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1269	def removeUnusedImageFolders(self):
1270	"""check if an unused image folders exists in the filesystem"""
1271	mtool = self.portal_membership
1272	member = mtool.getAuthenticatedMember()
1273	member_id = str(member)
1274	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1275	abc = os.listdir(images_base)
1276	ifolders = []
1277	for i in abc:
1278	picture_path = os.path.join(images_base,i)
1279	ifolders.extend(os.listdir(picture_path))
1280	unused_ids = []
1281	for id in ifolders:
1282	res = self.students_catalog(id=id)
1283	if not res:
1284	unused_ids.append(id)
1285	#import pdb;pdb.set_trace()
1286	if not id.endswith('removed'):
1287	removed = self.waeup_tool.removePictureFolder(id)
1288	if removed:
1289	logger.info('%s: image folder %s successfully removed' % (member_id,id))
1290	else:
1291	logger.info('%s: image folder %s could not be removed' % (member_id,id))
1292	return
1293
1294	###)
1295
1296	security.declarePublic('picturesList') ###(
1297	def picturesList(self):
1298	"""check if pictures exist in the filesystem"""
1299	path = 'images'
1300	student_id = self.getStudentId()
1301	#picture_path = os.path.join(i_home,path,student_id)
1302	picture_path = getImagesDir(student_id)
1303	if not os.path.exists(picture_path):
1304	return []
1305	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1306	###)
1307
1308	security.declarePublic('showFsPicture') ###(
1309	def showFsPicture(self,path):
1310	"""return a picture from the filesystem"""
1311	#picture_path = os.path.join(i_home,path)
1312	picture_path = os.path.join(images_base,path)
1313	response = self.REQUEST.RESPONSE
1314	#import pdb;pdb.set_trace()
1315	registry = getToolByName(self, 'mimetypes_registry')
1316	mimetype = str(registry.lookupExtension(path.lower()) or
1317	registry.lookupExtension('file.bin'))
1318	if os.path.exists(picture_path):
1319	response.setHeader('Content-type',mimetype)
1320	return open(picture_path).read()
1321	picture_path = os.path.join(i_home,'import',path)
1322	if os.path.exists(picture_path):
1323	return open(picture_path).read()
1324	###)
1325
1326	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1327	def deleteAllCourses(self,department="All"):
1328	''' delete the courses'''
1329	pm = self.portal_membership
1330	member = pm.getAuthenticatedMember()
1331
1332	if str(member) not in ("henrik","joachim"):
1333	return "not possible"
1334	if department == "All":
1335	res = self.portal_catalog({'meta_type': 'Department'})
1336	if len(res) < 1:
1337	return "No Departments found"
1338
1339	deleted = []
1340	for dep in res:
1341	cf = dep.getObject().courses
1342	if cf:
1343	cf.manage_delObjects(ids=cf.objectIds())
1344	deleted.append("deleted Courses in %s" % dep.getId)
1345	return "\r".join(deleted)
1346	###)
1347
1348	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1349	def getLogfileLines(self,filename="event.log",numlines=20):
1350	"""Get last NUMLINES lines of logfile FILENAME.
1351
1352	Return last lines' of a file in the instances logfile directory as
1353	a list. The number of returned lines equals `numlines' or less. If
1354	less than `numlines' lines are available, the whole file ist
1355	returned. If the file can not be opened or some other error
1356	occurs, empty list is returend.
1357	"""
1358	result = []
1359	lines_hit = 0
1360
1361	# We only handle files in instances' log directory...
1362	logpath = os.path.join(i_home, "log")
1363	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1364	if not filename.startswith( logpath ):
1365	# Attempt to access file outside log-dir...
1366	return []
1367
1368	try:
1369	fd = file( filename, "rb" )
1370	except IOError:
1371	return []
1372	if not fd:
1373	return []
1374
1375	if os.linesep == None:
1376	linesep = '\n'
1377	else:
1378	linesep = os.linesep
1379
1380	# Try to find 'numlines' times a lineseparator, searching from end
1381	# and moving to the beginning of file...
1382	fd.seek( 0, 2) # Move to end of file...
1383	while lines_hit < numlines:
1384	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1385	# one step forward...
1386	lines_hit += 1
1387	try:
1388	fd.seek( -2, 1) # Go two bytes back from current pos...
1389	except IOError:
1390	# We cannot go back two bytes. Maybe the file is too small...
1391	break
1392	fd.seek(2,1)
1393
1394	# Read all lines from current position...
1395	result = fd.readlines()
1396	# Remove line endings...
1397	result = [x.strip() for x in result]
1398	fd.close()
1399	return result
1400	###)
1401
1402	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1403	def getCallbacksFromLog(self,filename):
1404	"""fix Online Payment Transactions from Z2.log entries"""
1405	import transaction
1406	import random
1407	from cgi import parse_qs
1408	from urlparse import urlparse
1409	#from pdb import set_trace
1410	wftool = self.portal_workflow
1411	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1412	students_folder = self.portal_url.getPortalObject().campus.students
1413	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1414	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1415	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1416	data = re.compile(s)
1417	start = True
1418	tr_count = 1
1419	total = 0
1420	#name = 'pume_results'
1421	#name = 'epaymentsuccessful_z2log2'
1422	name = filename
1423	no_import = []
1424	imported = []
1425	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1426	try:
1427	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1428	except:
1429	logger.error('Error reading %s' % name)
1430	return
1431	tas = []
1432	for line in transactions:
1433	dict = {}
1434	items = data.search(line)
1435	dict['idict'] = idict = items.groupdict()
1436	#print idict
1437	#from pdb import set_trace;set_trace()
1438	urlparsed = urlparse(idict['get'][4:])
1439	#print urlparsed
1440	path = urlparsed[2].split('/')
1441	dict['student_id'] = student_id = path[8]
1442	dict['payment_id'] = payment_id = path[10]
1443	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1444	tas.append(dict)
1445	tr_count += 1
1446	return tas
1447	###)
1448
1449	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1450	def importOnlinePaymentTransactions(self):
1451	"""load Online Payment Transactions from CSV values"""
1452	import transaction
1453	import random
1454	#from pdb import set_trace
1455	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1456	opt = self.online_payments_import
1457	students_folder = self.portal_url.getPortalObject().campus.students
1458	start = True
1459	tr_count = 1
1460	total = 0
1461	#name = 'pume_results'
1462	name = 'OnlineTransactions'
1463	no_import = []
1464	imported = []
1465	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1466	try:
1467	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1468	except:
1469	logger.error('Error reading %s.csv' % name)
1470	return
1471	for pay_transaction in transactions:
1472	if start:
1473	start = False
1474	logger.info('Start loading from %s.csv' % name)
1475	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1476	no_import.append('%s,"Error"' % s)
1477	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1478	format_error = format + ',"%(Error)s"'
1479	data = {}
1480
1481	# format of the first file sent by Tayo
1482	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1483	#data['student_id'] = student_id = pay_transaction['Payer ID']
1484	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1485	#data['response_code'] = response_code = pay_transaction['Resp Code']
1486	#data['amount'] = amount = pay_transaction['Amount']
1487
1488	# format of the second file sent by Tayo
1489	#data['datetime'] = date = 0
1490	#data['student_id'] = student_id = pay_transaction['Payer ID']
1491	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1492	#data['response_code'] = response_code = '00'
1493	#data['amount'] = amount = pay_transaction['Amount']
1494
1495	# format of the third file sent by Kehinde
1496	data['datetime'] = date = 0
1497	data['student_id'] = student_id = pay_transaction['customer_id']
1498	data['order_id'] = order_id = pay_transaction['merchant_reference']
1499	data['response_code'] = response_code = '00'
1500	data['amount'] = amount = pay_transaction['Amount']
1501
1502	dup = False
1503	if response_code == "12":
1504	continue
1505	try:
1506	opt.addRecord(**data)
1507	except ValueError:
1508	dup = True
1509	#from pdb import set_trace;set_trace()
1510	if dup:
1511	if response_code == "00":
1512	try:
1513	opt.modifyRecord(**data)
1514	except:
1515	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1516	continue
1517	else:
1518	pay_transaction['Error'] = "Duplicate order_id"
1519	no_import.append( format_error % pay_transaction)
1520	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1521	continue
1522	tr_count += 1
1523	if tr_count > 1000:
1524	if len(no_import) > 0:
1525	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1526	'\n'.join(no_import) + '\n')
1527	no_import = []
1528	em = '%d transactions committed\n' % (tr_count)
1529	transaction.commit()
1530	regs = []
1531	logger.info(em)
1532	total += tr_count
1533	tr_count = 0
1534	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1535	'\n'.join(no_import))
1536	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1537	###)
1538
1539	security.declareProtected(ModifyPortalContent,"importData")###(
1540	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1541	"""load data from CSV values"""
1542	import transaction
1543	import random
1544	students_folder = self.portal_url.getPortalObject().campus.students
1545	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1546	pending_only = False
1547	pend_str = '--'
1548	elapse = time.time()
1549	#
1550	# preparations
1551	#
1552	if filename == pend_str:
1553	pending_only = True
1554	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1555	importer = eval("%sImport" % importer_name)(self)
1556	logger = importer.logger
1557	if importer.init_errors:
1558	logger.info(importer.init_errors)
1559	return importer.init_errors
1560	member = importer.member
1561	#current = importer.current
1562	import_date = importer.import_date
1563	#
1564	# not_imported
1565	#
1566	info = importer.info
1567	data_keys = importer.data_keys
1568	csv_keys = importer.csv_keys
1569	#csv_keys.extend(info.keys())
1570	headline_mapping = dict((k,k) for k in csv_keys)
1571	#
1572	# pending
1573	#
1574	pending_path = importer.pending_path
1575	pending_tmp = importer.pending_tmp
1576	pending_backup = importer.pending_backup
1577	pending_fn = importer.pending_fn
1578	imported_path = importer.imported_path
1579	imported_fn = importer.imported_fn
1580	commit_after = importer.commit_after
1581	pending = []
1582	pending_digests = []
1583	#total_added_to_pending = 0
1584	if not pending_only:
1585	pending,pending_digests = importer.makeIdLists()
1586	pending_at_start = len(pending)
1587	datafile = open(pending_tmp,"w")
1588	pending_csv_writer = csv.DictWriter(datafile,
1589	csv_keys,
1590	extrasaction='ignore')
1591	pending_csv_writer.writerow(headline_mapping)
1592	datafile.close()
1593	#
1594	# imported
1595	#
1596	if not os.path.exists(imported_path):
1597	datafile = open(imported_path,"w")
1598	imported_csv_writer = csv.DictWriter(datafile,
1599	csv_keys,
1600	extrasaction='ignore')
1601	imported_csv_writer.writerow(headline_mapping)
1602	datafile.close()
1603	start = True
1604	tr_count = 0
1605	total = 0
1606	total_added_to_imported = 0
1607	total_pending = 0
1608	import_source_done = ""
1609	if pending_only:
1610	import_source_path = pending_path
1611	else:
1612	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1613	import_source_done = "%s/import/%s.done" % (i_home,filename)
1614	if not os.path.exists(import_source_path):
1615	fn = os.path.split(import_source_path)[1]
1616	em = 'no import file %(fn)s' % vars()
1617	return em
1618	import_source_fn = os.path.split(import_source_path)[1]
1619	if not pending_only:
1620	info['imported_from'] = import_source_fn
1621	headline = csv.reader(open(import_source_path,"rb")).next()
1622	if "import_mode" not in headline:
1623	msg = 'import_mode must be in heading'
1624	return msg
1625	invalid_keys = importer.checkHeadline(headline)
1626	if invalid_keys:
1627	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1628
1629	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1630	or k.strip() in info.keys())]
1631	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1632	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1633	# if diff2schema and diff2schema != set(['id',]):
1634	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1635	# return msg
1636	#
1637	# start importing
1638	#
1639	try:
1640	reader = csv.DictReader(open(import_source_path,"rb"))
1641	except:
1642	msg = 'Error reading %s.csv' % filename
1643	logger.error(msg)
1644	return msg
1645	items = [item for item in reader]
1646	total_to_import = len(items)
1647	tti_float = float(total_to_import)
1648	if pending_only:
1649	pending_at_start = total_to_import
1650	count = 0
1651	imported = []
1652	old_commit_count = 0
1653	error_count = imported_count = 0
1654	already_in = 0
1655	for record in items:
1656	item = {}
1657	empty_value_keys = []
1658	for k,v in record.items():
1659	if k is None:
1660	continue
1661	if v:
1662	if v == EMPTY:
1663	empty_value_keys += k,
1664	v = ''
1665	item[k.strip()] = v.strip()
1666	count += 1
1667	if start:
1668	start = False
1669	adapters = [MappingStorageAdapter(importer.schema, item)]
1670	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1671	dm = DataModel(item, adapters,context=self)
1672	ds = DataStructure(data=item,datamodel=dm)
1673	error_string = ""
1674	total += 1
1675	import_mode = item.get('import_mode','')
1676	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1677	if import_method is None:
1678	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1679	elif (import_mode in importer.required_modes and
1680	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1681	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1682	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1683	else:
1684	for k in import_keys:
1685	if k not in item.keys() or k not in importer.validators.keys():
1686	continue
1687	if not importer.validators[k](ds,mode=import_mode):
1688	error_string += ' ++ '
1689	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1690	ds.getErrorMapping(k)))
1691	if error_string:
1692	error = error_string
1693	id = ''
1694	mapping = item
1695	else:
1696	temp_item = item.copy()
1697	temp_item.update(dm)
1698	results = import_method(temp_item)
1699	id = results[0]
1700	error = results[1]
1701	mapping = results[2]
1702	#
1703	# restore some values
1704	#
1705	for k in empty_value_keys:
1706	mapping[k] = EMPTY
1707	if mapping.has_key('sex'):
1708	#import pdb;pdb.set_trace()
1709	if mapping['sex'] == True:
1710	mapping['sex'] = 'F'
1711	elif mapping['sex'] == False:
1712	mapping['sex'] = 'M'
1713	if pending_only:
1714	info['imported_from'] = item['imported_from']
1715	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1716	info['error'] = error
1717	info['import_record_no'] = count + 1
1718	mapping.update(info)
1719	log_list = []
1720	if error:
1721	error_count += 1
1722	digest = makeDigest(mapping,data_keys)
1723	if digest not in pending_digests:
1724	pending_digests += digest,
1725	pending.append(mapping)
1726	if not pending_only:
1727	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1728	else:
1729	already_in += 1
1730	pass
1731	else:
1732	imported_count += 1
1733	imported += mapping,
1734	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1735	if log_list:
1736	time_till_now = time.time() - elapse
1737	percent_finished = (error_count + imported_count)/tti_float*100
1738	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1739	logger.info(' '.join(log_list))
1740	finished = count > total_to_import - 1
1741	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1742	if must_commit:
1743	old_commit_count = imported_count
1744
1745	if must_commit or finished:
1746	if len(imported):
1747	transaction.commit()
1748	datafile = open(imported_path,"a")
1749	writer = csv.DictWriter(datafile,
1750	csv_keys,
1751	extrasaction='ignore')
1752	writer.writerows(imported)
1753	datafile.close()
1754	total_added_to_imported += len(imported)
1755	imported = []
1756	if len(pending) > 0:
1757	datafile = open(pending_tmp,"a")
1758	writer = csv.DictWriter(datafile,
1759	csv_keys,
1760	extrasaction='ignore')
1761	writer.writerows(pending)
1762	datafile.close()
1763	total_pending += len(pending)
1764	#total_added_to_pending += len(pending)
1765	pending = []
1766	if not finished:
1767	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1768	logger.info(msg)
1769	elapse = time.time() - elapse
1770	if os.path.exists(pending_path):
1771	copy2(pending_path,pending_backup)
1772	copy2(pending_tmp,pending_path)
1773	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1774	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1775	if pending_only:
1776	removed_pending = pending_at_start - total_pending
1777	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1778	else:
1779	added_pending = total_pending - pending_at_start
1780	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1781	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1782	logger.info(msg)
1783	if import_source_done:
1784	copy(import_source_path,import_source_done)
1785	os.remove(import_source_path)
1786	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1787	if upload is not None:
1788	upload_doc = upload.getContent()
1789	mapping = {}
1790	#mapping['import_date'] = DateTime.DateTime()
1791	mapping['import_date'] = import_date
1792	mapping['imported_by'] = importer.imported_by
1793	mapping['import_message'] = msg
1794	upload_doc.edit(mapping = mapping)
1795	os.remove(pending_tmp)
1796	return msg
1797	###)
1798
1799	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1800	def moveImagesToFS(self,student_id="O738726"):
1801	"move the images to the filesystem"
1802	images_dir = getImagesDir(student_id)
1803	#images_dir = os.path.join("%s" % images_base,student_id)
1804	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1805	stool = getToolByName(self, 'portal_schemas')
1806	schemas = ['student_application',
1807	'student_clearance',
1808	]
1809	created = False
1810	for schema_id in schemas:
1811	schema = stool._getOb(schema_id)
1812	object = getattr(student_folder,schema_id[len('student_'):],None)
1813	if object is None:
1814	continue
1815	doc = object.getContent()
1816	for key in schema.keys():
1817	if schema[key].meta_type != "CPS Image Field":
1818	continue
1819	#import pdb;pdb.set_trace()
1820	image = getattr(doc,key,None)
1821	if not image or not hasattr(image,"data"):
1822	continue
1823	if not created:
1824	if not os.path.exists(images_dir):
1825	os.mkdir(images_dir)
1826	created = True
1827	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1828	open(filename,"wb").write(str(image.data))
1829	###)
1830
1831	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1832	def movePassportToFS(self,student_id="O738726"):
1833	"move the passports to the filesystem"
1834	images_dir = os.path.join("%s" % i_home,'passports')
1835	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1836	stool = getToolByName(self, 'portal_schemas')
1837	schemas = ['student_application',
1838	#'student_clearance',
1839	]
1840	created = False
1841	for schema_id in schemas:
1842	schema = stool._getOb(schema_id)
1843	object = getattr(student_folder,schema_id[len('student_'):],None)
1844	if object is None:
1845	continue
1846	doc = object.getContent()
1847	for key in schema.keys():
1848	if schema[key].meta_type != "CPS Image Field":
1849	continue
1850	#import pdb;pdb.set_trace()
1851	image = getattr(doc,key)
1852	if not hasattr(image,"data"):
1853	continue
1854	if not created:
1855	if not os.path.exists(images_dir):
1856	os.mkdir(images_dir)
1857	created = True
1858	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1859	open(filename,"wb").write(str(image.data))
1860	###)
1861
1862	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: