Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 3827

Last change on this file since 3827 was 3824, checked in by Henrik Bettermann, 16 years ago

implement configuration object (step 1)

Configuration file must be created with create_configuration_file.py

Property svn:keywords set to Id

File size: 76.6 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3824 2008-12-19 15:44:45Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108	###)
109
110	security.declareProtected(View,'updateRoleMappingsFor') ###(
111	def updateRoleMappingsFor(self,wf_definition,ob):
112	"do so for public"
113	wf_def = getattr(self.portal_workflow,wf_definition)
114	wf_def.updateRoleMappingsFor(ob)
115	###)
116
117	security.declareProtected(View,'getStatesLgas') ###(
118	def getStatesLgas(self):
119	"""return lga info"""
120	voc = getattr(self.portal_vocabularies,'local_gov_areas')
121	states = []
122	lgas = []
123	d = {}
124	wd = {}
125	for k,v in voc.items():
126	parts = v.split(' / ')
127	if len(parts) == 1:
128	state = parts[0].lower()
129	lga = ""
130	elif len(parts) == 2:
131	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133	else:
134	continue
135	if state not in states:
136	states.append(state)
137	if lga not in lgas:
138	lgas.append(lga)
139	words = re.split('[^a-zA-Z0-9/]',k)
140	words.sort()
141	wd[k] = words
142	d[k] = v
143	mapping = {}
144	mapping['word_dict'] = wd
145	mapping['lga_dict'] = d
146	mapping['states'] = states
147	mapping['lgas'] = lgas
148	return mapping
149	###)
150
151	security.declareProtected(View,'findLga') ###(
152	def findLga(self,words,words_dict):
153	words = re.split('[^a-zA-Z0-9/]',words)
154	lga_words = []
155	for word in words:
156	if word:
157	lga_words += word.strip().lower(),
158	lga_words.sort()
159	state_lga = ''
160	while not state_lga:
161	for k,l in words_dict.items():
162	if lga_words == l:
163	state_lga = k
164	break
165	break
166	return state_lga
167	###)
168	security.declareProtected(View,'getAccessInfo') ###(
169	def getAccessInfo(self,context):
170	"return a dict with access_info"
171	logger = logging.getLogger('WAeUPTool.getAccessInfo')
172	mtool = self.portal_membership
173	member = mtool.getAuthenticatedMember()
174	member_id = str(member)
175	info = {}
176	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178	is_staff = info['is_staff'] = not is_anonymous and not is_student
179	roles = member.getRolesInContext(context)
180	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181	"SectionManager" in roles or
182	"Manager" in roles)
183	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184	is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if is_allowed:
188	if not is_student and requested_id:
189	student_id = requested_id
190	elif not is_allowed and (not is_staff or member_id != requested_id):
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	else:
193	student_id = member_id
194	info['student_id'] = student_id
195	return info
196	###)
197
198	security.declareProtected(ModifyPortalContent,'openLog') ###(
199	def openLog(self,name):
200	"""open a log file"""
201	version = 1
202	path = "%s/log/%s_%d.log" % (i_home,name,version)
203	while os.path.exists(path):
204	version += 1
205	path = "%s/log/%s_%d.log" % (i_home,name,version)
206	log = open(path,"w")
207	return log
208	###)
209
210	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211	def bypassQueueCatalog(self,enable=True):
212	"""bypass the QueueCatalog by setting all indexes to process imediate,
213	if enable is True (default) the old settings are restored
214	"""
215
216	###)
217
218	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220	"""measure Object access Time"""
221	import random
222	if hasattr(self,'portal_catalog_real'):
223	aq_portal = self.portal_catalog_real.evalAdvancedQuery
224	else:
225	aq_portal = self.portal_catalog.evalAdvancedQuery
226	nr_pts = int(nr_pts)
227	probe = int(probe)
228	intervall = probe/10
229	objects = ("application","clearance","personal")
230	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231	#i = random.randrange(num_objects)
232	count = 0
233	found = 0
234	not_found = 0
235	t_found = 0
236	t_not_found = 0
237	time_found = time_not_found = 0.0
238	t_time_found = t_time_not_found = 0.0
239	accessed = []
240	t_min = 1000
241	t_max = 0
242	#import pdb;pdb.set_trace()
243	students = self.portal_catalog(portal_type="Student")
244	num_students = len(students)
245	if method == "d":
246	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247	res = aq_portal(query)
248	brains = {}
249	for r in res:
250	sid = r.relative_path.split('/')[-2]
251	if brains.has_key(sid):
252	brains[sid][r.portal_type] = r
253	else:
254	brains[sid] = {r.portal_type : r}
255	brains_list = brains.keys()
256	num_objects = len(brains_list)
257	else:
258	num_objects = num_students
259	print "="*40
260	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261	probe,
262	nr_pts,
263	num_objects)
264	print "nr found/not time found/not min/max"
265	elapse = time.time()
266	i_elapse = time.time()
267	c_elapse = time.clock()
268	for c in range(1,probe + 1):
269	i = random.randrange(num_objects)
270	if method in ('a','b','c'):
271	student_brain = students[i]
272	elif method == "d":
273	#import pdb;pdb.set_trace()
274	student_brain = brains[brains_list[i]]
275	if method == "c":
276	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277	res = aq_portal(query)
278	this_portal_types = [r.portal_type for r in res]
279	for i in range(nr_pts):
280	oid = objects[i]
281	if method == "a":
282	try:
283	student_path = student_brain.getPath()
284	path = "%s/%s" % (student_path,oid)
285	doc = self.unrestrictedTraverse(path).getContent()
286	found += 1
287	i_time = time.time() - i_elapse
288	time_found += i_time
289	except:
290	not_found += 1
291	i_time = time.time() - i_elapse
292	time_not_found += i_time
293	pass
294	elif method == "b":
295	try:
296	student_object = student_brain.getObject()
297	doc = getattr(student_object,oid).getContent()
298	found += 1
299	i_time = time.time() - i_elapse
300	time_found += i_time
301	except:
302	i_time = time.time() - i_elapse
303	time_not_found += i_time
304	not_found += 1
305	pass
306	elif method == "c":
307	if portal_types[i] in this_portal_types:
308	found += 1
309	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310	i_time = time.time() - i_elapse
311	time_found += i_time
312	else:
313	not_found += 1
314	i_time = time.time() - i_elapse
315	time_not_found += i_time
316	elif method == "d":
317	if student_brain.has_key(portal_types[i]):
318	found += 1
319	doc = student_brain[portal_types[i]].getObject().getContent()
320	i_time = time.time() - i_elapse
321	time_found += i_time
322	else:
323	not_found += 1
324	i_time = time.time() - i_elapse
325	time_not_found += i_time
326	i_elapse = time.time()
327	if c and (c % intervall == 0):
328	#i_time = time.time() - i_elapse
329	t_per = 0.0
330	if found:
331	t_per = time_found/found
332	if t_per > t_max:
333	t_max = t_per
334	if t_per > 0.0 and t_per < t_min:
335	t_min = t_per
336	itf = 0.0
337	if found:
338	itf = time_found/found
339	itnf = 0.0
340	if not_found :
341	itnf = time_not_found / not_found
342	interval_time = time_found + time_not_found
343	s = "%(c)d: %(found)d/%(not_found)d " % vars()
344	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
346	print s
347	t_found += found
348	t_not_found += not_found
349	t_time_found += time_found
350	t_time_not_found += time_not_found
351	time_found = time_not_found = 0.0
352	found = not_found = 0
353	# t_found += found
354	# t_not_found += not_found
355	elapse = time.time() - elapse
356	itf = 0.0
357	if t_found:
358	itf = t_time_found/t_found
359	itnf = 0.0
360	if t_not_found:
361	itnf = t_time_not_found / t_not_found
362	#c_elapse = time.clock() - c_elapse
363	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
366	print "-"*40
367	print s
368	rel_found = float(t_found)/probe
369	rel_not_found = float(t_not_found)/probe
370	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
371	print estimated_total_time
372	###)
373
374	security.declareProtected(ModifyPortalContent,'writeLog') ###(
375	def writeLog(self,logfile,s):
376	"""write to the log file"""
377	logfile.write(s)
378	###)
379
380	def generateStudentId(self,letter): ###(
381	import random
382	logger = logging.getLogger('WAeUPTool.generateStudentId')
383	r = random
384	if letter == '?':
385	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
386	sid = "%c%d" % (letter,r.randint(99999,1000000))
387	students = self.portal_url.getPortalObject().campus.students
388	while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid):
389	if self.waeup_tool.picturePathExists(sid):
390	logger.info('picture path %s exists' % sid)
391	sid = "%c%d" % (letter,r.randint(99999,1000000))
392	return sid
393	###)
394
395	def generatePassword(self,s=None): ###(
396	import random
397	r = random
398	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
399	if s is None:
400	s = 'abcdefghklmnpqrstuvwxy23456789'
401	pw = ''
402	while len(pw) < 6:
403	pw += r.choice(s)
404	return pw
405	###)
406
407	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
408	def dumpSchoolfeePayments(self):
409	"dump paid schoolfees"
410	mtool = self.portal_membership
411	member = mtool.getAuthenticatedMember()
412	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
413	aq_student = self.students_catalog.evalAdvancedQuery
414	query = In('review_state',('schoolfee_paid',
415	'courses_registered',
416	'courses_validated',
417	))
418	res = aq_student(query)
419	#import pdb;pdb.set_trace()
420	l = []
421	logger.info("start for %d" % len(res))
422	count = 1
423	log_after = 100
424	for student in res:
425	if not count % log_after:
426	logger.info("processed %d total %d" % (log_after,count))
427	count += 1
428	fee_dict =self.getSchoolFee(student)
429	fulltime = student.mode.endswith('_ft')
430	d = {}
431	d['student_id'] = student.id
432	d['name'] = student.name
433	d['amount'] = fee_dict.get(new_returning)
434	l += d,
435	csv_name = self.dumpListToCSV(l,'payments')
436	logger.info('%s dumped payments to %s' % (member,export_file))
437	###)
438
439	security.declarePublic('dumpListToCSV') ###(
440	def dumpListToCSV(self,l,filename,fields=None):
441	"""dump a list of dicts to a CSV file"""
442	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
443	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
444	if fields is None:
445	fields = l[0].keys()
446	headline = ','.join(fields)
447	out = open(export_file,"wb")
448	out.write(headline +'\n')
449	out.close()
450	out = open(export_file,"a")
451	csv_writer = csv.DictWriter(out,fields,)
452	csv_writer.writerows(l)
453	return export_file
454	###)
455
456	security.declareProtected(ManagePortal, 'listMembers') ###(
457	def listMembers(self):
458	"list all members"
459	mtool = self.portal_membership
460	member = mtool.getAuthenticatedMember()
461	logger = logging.getLogger('WAeUPTool.listMembers')
462	if str(member) not in ('admin','joachim'):
463	logger.info('%s tried to list members' % (member))
464	return None
465	members = self.portal_directories.members
466	all = members.listEntryIdsAndTitles()
467	l = []
468	for user_id,name in all:
469	d = {}
470	d['user_id'] = user_id
471	d['name'] = name
472	d['pw'] = getattr(getattr(members,user_id),'password')
473	d['email'] = getattr(getattr(members,user_id),'email')
474	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
475	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
476	l += d,
477	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
478	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
479	logger.info('%s dumped member list to %s' % (member,export_file))
480	fields = l[0].keys()
481	headline = ','.join(fields)
482	out = open(export_file,"wb")
483	out.write(headline +'\n')
484	out.close()
485	out = open(export_file,"a")
486	csv_writer = csv.DictWriter(out,fields,)
487	csv_writer.writerows(l)
488	###)
489
490	security.declareProtected(ManagePortal, 'listStudents') ###(
491	def listStudents(self):
492	"list all students"
493	mtool = self.portal_membership
494	member = mtool.getAuthenticatedMember()
495	logger = logging.getLogger('WAeUPTool.listStudents')
496	if str(member) not in ('admin','joachim'):
497	logger.info('%s tried to list students' % (member))
498	return None
499	students = self.portal_directories.students
500	all = students.listEntryIdsAndTitles()
501	l = []
502	for user_id,name in all:
503	d = {}
504	d['user_id'] = user_id
505	d['name'] = name
506	d['pw'] = getattr(getattr(students,user_id),'password')
507	d['email'] = getattr(getattr(students,user_id),'email')
508	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
509	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
510	l += d,
511	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
512	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
513	logger.info('%s dumped student list to %s' % (member,export_file))
514	fields = l[0].keys()
515	headline = ','.join(fields)
516	out = open(export_file,"wb")
517	out.write(headline +'\n')
518	out.close()
519	out = open(export_file,"a")
520	csv_writer = csv.DictWriter(out,fields,)
521	csv_writer.writerows(l)
522	###)
523
524	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
525	def removeDeletedDocIds(self, max=1000):
526	"""
527	remove deleted docids from repository commit after max
528	"""
529	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
530	repository = getToolByName(self, 'portal_repository')
531	pxtool = getToolByName(self, 'portal_proxies')
532	logger.info('start')
533	pxtool_infos = pxtool.getRevisionsUsed()
534	logger.info('found %d used revisions ' % (len(pxtool_infos)))
535
536	nb_revs = 0
537	docids_d = {} # all docids
538	unused_docids_d = {} # all docids that are unused
539	ids_unused_revs_docids = [] # ids for revs of unused docids
540	unused_ids = [] # ids for unused revs
541	total = 0
542	idlist = repository.objectIds()
543	to_delete = 0
544	found = False
545	for id in idlist:
546	docid, rev = repository._splitId(id)
547	if docid is None:
548	logger.info("invalid doc_id %s" % docid)
549	continue
550	nb_revs += 1
551	if not pxtool_infos.has_key(docid):
552	found = True
553	to_delete += 1
554	unused_ids.append(id)
555	elif not pxtool_infos[docid].has_key(rev):
556	found = True
557	to_delete += 1
558	unused_ids.append(id)
559	if found and not to_delete % max:
560	found = False
561	#import pdb;pdb.set_trace()
562	repository.manage_delObjects(unused_ids)
563	transaction.commit()
564	logger.info('removed %d total %d unused docids ' % (max,to_delete))
565	else:
566	if unused_ids:
567	repository.manage_delObjects(unused_ids)
568	transaction.commit()
569	logger.info('finished removing %d unused docids ' % (to_delete))
570	###)
571
572	security.declareProtected(View,'getCredential') ###(
573	def getCredential(self,student_id):
574	student_entry = getattr(self.portal_directories.students,student_id,None)
575	if not self.isStaff():
576	mtool = self.portal_membership
577	member = mtool.getAuthenticatedMember()
578	logger = logging.getLogger('WAeUPTool.getCredential')
579	logger.info('%s tried to access password of %s' % (member,student_id))
580	return None
581	if student_entry is None:
582	return None
583	return getattr(student_entry,"password","not set")
584	###)
585
586	security.declarePublic('checkPassword') ###(
587	def checkPassword(self,student_id,password):
588	student_entry = getattr(self.portal_directories.students,student_id,None)
589	if student_entry is None:
590	return False
591	return getattr(student_entry,"password","not set") == password
592	###)
593
594	security.declarePublic('checkGenericPassword') ###(
595	def checkGenericPassword(self,member_id):
596	member_entry = getattr(self.portal_directories.members,member_id,None)
597	if member_entry is None:
598	return False
599	ltool = getToolByName(self, 'portal_layouts')
600	unsecure_words = ltool._getOb('members')['w__password'].check_words
601	password = getattr(member_entry,"password","not set")
602	is_unsecure = password in unsecure_words
603	if is_unsecure:
604	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
605	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
606	return is_unsecure
607	###)
608
609	security.declareProtected(ModifyPortalContent,'editPassword') ###(
610	def editPassword(self,student_id,password):
611	"edit a student password"
612	student_entry = getattr(self.portal_directories.students,student_id,None)
613	if student_entry is None:
614	return
615	setattr(student_entry,'password',password)
616	###)
617
618	security.declareProtected(ModifyPortalContent,'doCommit') ###(
619	def doCommit(self,logger=None):
620	"commit some transactions"
621	transaction.commit()
622	###)
623
624	security.declarePublic('loadStudentFoto') ###(
625	def loadStudentFoto(self,student,filename,folder):
626	"return a student passport picture"
627	#import pdb;pdb.set_trace()
628	picture ="%s/import/%s/%s" % (i_home,folder,filename)
629	student_id = student.getId()
630	images_dir = getImagesDir(student_id)
631	if not os.path.exists(images_dir):
632	os.mkdir(images_dir)
633	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
634	for extension in ('.jpg','.JPG'):
635	fullname = "%(picture)s%(extension)s" % vars()
636	if os.path.exists(fullname):
637	copy2(fullname,image_name)
638	return "successfully copied passport picture"
639	return "passport picture not found: %s.jpg or .JPG" % picture
640	###)
641
642	def old____loadStudentFoto(self,student,filename,folder): ###(
643	"return a student passport picture"
644	app = student.application
645	app_doc = app.getContent()
646	#clear = student.clearance
647	#clear_doc = clear.getContent()
648	#matric_no = clear_doc.matric_no.upper()
649	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
650	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
651	#import pdb;pdb.set_trace()
652	if os.path.exists(picture1):
653	file = open(picture1)
654	elif os.path.exists(picture2):
655	file = open(picture2)
656	else:
657	return "passport picture not found %s" % picture1
658	reopened = False
659	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
660	self.portal_workflow.doActionFor(app,'open')
661	reopened = True
662	outfile = file.read()
663	app_doc.manage_addFile('passport',
664	file=outfile,
665	title="%s.jpg" % filename)
666	if reopened:
667	self.portal_workflow.doActionFor(app,'close')
668	return "successfully loaded passport picture"
669	###)
670
671	security.declareProtected(ModifyPortalContent,'createOne') ###(
672	def createOne(self,students_folder,student_brain,letter,commit=False):
673	sid = self.waeup_tool.generateStudentId(letter)
674	students_folder.invokeFactory('Student', sid)
675	student = getattr(students_folder,sid)
676	self.portal_workflow.doActionFor(student,'return')
677	student.manage_setLocalRoles(sid, ['Owner',])
678	matric_no = student_brain.matric_no
679	jamb_reg_no = student_brain.Entryregno
680	self.students_catalog.addRecord(id = sid,
681	matric_no = matric_no,
682	jamb_reg_no = jamb_reg_no,
683	sex = student_brain.Sex == "F",
684	name = "%s %s %s" % (student_brain.Firstname,
685	student_brain.Middlename,
686	student_brain.Lastname)
687	)
688	if commit:
689	transaction.commit()
690	return sid,jamb_reg_no
691	###)
692
693	security.declareProtected(ModifyPortalContent,'addStudent') ###(
694	def addStudent(self,dict):
695	students_folder = self.portal_url.getPortalObject().campus.students
696	sid = self.waeup_tool.generateStudentId('?')
697	students_folder.invokeFactory('Student', sid)
698	student_obj = getattr(students_folder,sid)
699	f2t = StudentImport.field2types_student
700	#from pdb import set_trace; set_trace()
701	d = {}
702	#d['jamb_sex'] = 'M'
703	#if dict.get('sex'):
704	# d['jamb_sex'] = 'F'
705
706	entry_session = dict.get('entry_session')
707	if entry_session == self.getSessionId()[0]:
708	wfaction = 'admit'
709	wft = 'wf_transition_admit'
710	password = None
711	else:
712	wfaction = 'return'
713	wft = 'wf_transition_return'
714	password = self.generatePassword()
715	self.makeStudentMember(sid,password)
716
717	for pt in f2t.keys():
718	student_obj.invokeFactory(pt,f2t[pt]['id'])
719	sub_obj = getattr(student_obj,f2t[pt]['id'])
720	sub_doc = sub_obj.getContent()
721	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
722	#d['Title'] = f2t[pt]['title']
723	for field in f2t[pt]['fields']:
724	d[field] = dict.get(field,'')
725	sub_doc.edit(mapping = d)
726	new_state = f2t[pt][wft]
727	if new_state != "remain":
728	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
729	self.portal_workflow.doActionFor(student_obj,wfaction)
730	student_obj.manage_setLocalRoles(sid, ['Owner',])
731	return sid,password
732	###)
733
734	security.declarePublic('getCertificateBrain') ###(
735	def getCertificateBrain(self,cert_id):
736	"do it"
737	res = ZCatalog.searchResults(self.portal_catalog_real,
738	{'portal_type':"Certificate",
739	'id': cert_id})
740	if res:
741	return res[0]
742	return None
743	###)
744
745	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
746	def get_csv_filenames(self):
747	"do it"
748	files = [file for file in os.listdir("%s/import/" % (i_home))
749	if file.endswith('.csv') and (file.find('imported') == -1 and
750	file.find('pending') == -1)]
751	return files
752	###)
753
754	security.declarePublic('findStudentByMatricelNo') ###(
755	def findStudentByMatricelNo(self,matric_no):
756	"do it"
757	res = ZCatalog.searchResults(self.portal_catalog_real,
758	{'portal_type':"StudentClearance",
759	'SearchableText': matric_no})
760	if res:
761	return res[0]
762	return None
763	###)
764
765
766	security.declarePublic('getOfficerName') ###(
767	def getOfficerName(self,mid):
768	"""get the real name of a member"""
769	membership = self.portal_membership
770	member_record = membership.getMemberById(mid)
771	return member_record.getProperty('fullname',None)
772	###)
773
774
775	security.declarePublic('makeStudentMember') ###(
776	def makeStudentMember(self,sid,password='uNsEt'):
777	"""make the student a member"""
778	membership = self.portal_membership
779	membership.addMember(sid,
780	password ,
781	roles=('Member',
782	'Student',
783	),
784	domains='',
785	properties = {'memberareaCreationFlag': False,
786	'homeless': True},)
787	member = membership.getMemberById(sid)
788	self.portal_registration.afterAdd(member, sid, password, None)
789	#self.manage_setLocalRoles(sid, ['Owner',])
790	###)
791
792	security.declareProtected(View,'makeStudentData') ###(
793	def makeStudentData(self,student_id,email=None,phone_nr=None):
794	"create Datastructure for a returning Student"
795	#import pdb;pdb.set_trace()
796	logger = logging.getLogger('WAeUPTool.makeStudentData')
797	students_folder = self.portal_url.getPortalObject().campus.students
798	#res = self.students_catalog(id=student_id)
799	#if res:
800	# st = res[0]
801	#res = self.returning_import(matric_no = st.matric_no)
802	res = self.returning_import(id = student_id)
803	if res:
804	student = res[0]
805	else:
806	logger.info('Id %s not found in returning_import' % student_id)
807	return
808	logger.info('%s creates data structure' % student_id)
809	s_results = self.results_import(matric_no = student.matric_no)
810	if s_results:
811	lnr = self.getLevelFromResultsCosCode(s_results)
812	level = "%d00" % lnr
813	verdict,eligible = self.getVerdict(s_results[0].Verdict)
814	#if eligible:
815	# level = "%d00" % (lnr + 1)
816	else:
817	logger.info('matric_no %s not found in results_import' % student.matric_no)
818	level = ''
819	verdict = ''
820	#student should not be allowed to perform this transition
821	#wftool = self.portal_workflow
822	#wftool.doActionFor(student,'return')
823	certcode_org = student.Coursemajorcode
824	certcode = makeCertificateCode(certcode_org)
825	certificate_brain = self.getCertificateBrain(certcode)
826	if not certificate_brain:
827	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
828	logger.info(em)
829	matric_no = student.matric_no
830	sid = student_id
831	student_obj = getattr(students_folder,sid)
832	if not getattr(student_obj,'application'):
833	student_obj.invokeFactory('StudentApplication','application')
834	application = student_obj.application
835	self.portal_workflow.doActionFor(application,'open',dest_container=application)
836	da = {'Title': 'Application Data'}
837	student_obj.invokeFactory('StudentPersonal','personal')
838	da['jamb_reg_no'] = student.Entryregno
839	em = self.getEntryMode(student.Entryregno)
840	da['entry_mode'] = em
841	personal = student_obj.personal
842	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
843	dp = {'Title': 'Personal Data'}
844	student_obj.invokeFactory('StudentClearance','clearance')
845	clearance = student_obj.clearance
846	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
847	dc = {'Title': 'Clearance/Eligibility Record'}
848	dc['matric_no'] = matric_no
849	state = student.State
850	lga = student.LGA
851	if state and lga:
852	lga = state + ' / ' + lga
853	else:
854	lga = "None"
855	da['jamb_lga'] = dc['lga'] = lga
856	da['app_email'] = dp['email'] = email
857	da['app_mobile'] = dp['phone'] = phone_nr
858	dp['firstname'] = student.Firstname
859	dp['middlename'] = student.Middlename
860	dp['lastname'] = student.Lastname
861	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
862	da['jamb_sex'] = student.Sex
863	dp['sex'] = student.Sex == 'F'
864	dp['perm_address'] = student.Permanent_Address
865	application.getContent().edit(mapping=da)
866	self.portal_workflow.doActionFor(application,'close',dest_container=application)
867	personal.getContent().edit(mapping=dp)
868	clearance.getContent().edit(mapping=dc)
869	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
870	#
871	# Study Course
872	#
873	student_obj.invokeFactory('StudentStudyCourse','study_course')
874	studycourse = student_obj.study_course
875	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
876	dsc = {}
877	dsc['study_course'] = certcode
878	dsc['current_level'] = level
879	dsc['current_verdict'] = verdict
880	dsc['current_mode'] = em #no longer used
881	dsc['current_session'] = '05'
882	studycourse.getContent().edit(mapping=dsc)
883	#
884	# Level
885	#
886	# l = getattr(studycourse,level,None)
887	# if l is None:
888	# studycourse.invokeFactory('StudentStudyLevel', level)
889	# l = getattr(studycourse, level)
890	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
891	# l.getContent().edit(mapping={'Title': "Level %s" % level})
892	###)
893
894	def init_timing(self): ###(
895	if self.with_timing:
896	if not hasattr(self,'_v_step_times'):
897	self._v_timer_count = 0
898	self._v_total = 0
899	self._v_step_times = {}
900	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
901	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
902	self.timer_step = 0
903	self.total_time = 0
904	self.elapse = time.time()
905	self.i_elapse = time.time()
906	self.c_elapse = time.clock()
907	###)
908
909	def do_timing(self): ###(
910	if self.with_timing:
911	try:
912	raise 'dummy'
913	except:
914	frame = sys.exc_traceback.tb_frame.f_back
915	locals = frame.f_locals
916	globals = frame.f_globals
917	functionname = frame.f_code.co_name
918	filename = os.path.basename(frame.f_code.co_filename)
919	lineno = frame.f_lineno
920	mod_line = "%(functionname)s:%(lineno)s" % vars()
921	i_time = time.time() - self.i_elapse
922	td = {}
923	if self._v_step_times.has_key(mod_line):
924	a_time = self._v_step_times[mod_line]['a_time'] + i_time
925	td['a_time'] = a_time
926	else:
927	td['a_time'] = i_time
928	td['i_time'] = i_time
929	self._v_step_times[mod_line] = td
930	self.i_time = i_time
931	self.total_time += i_time
932	self.timer_step +=1
933	self.i_elapse = time.time()
934	###)
935
936	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
937	def print_timing(self):
938	if self.with_timing:
939	l = []
940	timer_count = self._v_timer_count + 1
941	mod_lines = self._v_step_times.keys()
942	mod_lines.sort(cmp,reverse=0)
943	for mod_line in mod_lines:
944	td = self._v_step_times[mod_line]
945	i_time = td['i_time']
946	a_time = td['a_time']/(self._v_timer_count + 1)
947	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
948	total_time = self.total_time
949	total_avarage = self._v_total / timer_count
950	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
951	print "\r\n".join(l)
952	out = open(self._v_timer_file,'a')
953	out.write("\r\n".join(l))
954	out.close()
955	###)
956
957	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
958	def get_timing_data(self):
959	if self.with_timing:
960	timer_count = self._v_timer_count + 1
961	results = {}
962	for k,d in self._v_step_times.items():
963	dd = {}
964	dd['a_time'] = d['a_time']/timer_count
965	dd['i_time'] = d['i_time']
966	dd['count'] = timer_count
967	results[k] = dd
968	dd = {}
969	dd['a_time'] = self._v_total / timer_count
970	dd['i_time'] = self.total_time
971	dd['count'] = timer_count
972	results["total"] = dd
973	return results
974	###)
975
976	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
977	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
978	"create Datastructure for an admitted Student"
979	#import pdb;pdb.set_trace()
980	logger = logging.getLogger('WAeUPTool.admitOneStudent')
981	self.with_timing = with_timing
982
983	if brain.screening_type in ('cest','sandwich',):
984	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
985	else:
986	reg_no = brain.reg_no
987
988	#ignore argument entry_session
989	if not brain.entry_session:
990	logger.info('no entry_session for %s provided' % (reg_no))
991	return
992
993	if not hasattr(self,"_v_certificates"):
994	self._v_certificates = self.getCertificatesDict()
995	students_folder = self.portal_url.getPortalObject().campus.students
996
997	res = self.students_catalog(jamb_reg_no = reg_no)
998	if res:
999	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
1000	return
1001	if brain.status != "admitted":
1002	logger.info('status of %s is %s' % (reg_no,brain.status))
1003	return
1004	pin_parts = brain.pin.split('-')
1005	if pin_parts and len(pin_parts) != 3:
1006	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
1007	return
1008	if not brain.course_admitted:
1009	logger.info('no course_admitted provided for %s' % (reg_no))
1010	return
1011	if brain.course_admitted not in self._v_certificates:
1012	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
1013	return
1014	if brain.sex not in (True,False):
1015	logger.info('sex of %s not available' % (reg_no))
1016	return
1017	self.init_timing()
1018	student_id = self.generateStudentId('?')
1019	students_folder.invokeFactory('Student', student_id)
1020	student_object = getattr(students_folder,student_id)
1021	self.do_timing()
1022	if pin_password:
1023	password = pin_parts[2]
1024	self.makeStudentMember(student_id,password = password)
1025	student_object.manage_setLocalRoles(student_id, ['Owner',])
1026	self.do_timing()
1027	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1028	#
1029	# application
1030	#
1031	student_object.invokeFactory('StudentApplication','application')
1032	application = student_object.application
1033	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1034	#self.do_timing()
1035	da = {'Title': 'Application Data'}
1036	da['jamb_reg_no'] = reg_no
1037
1038	sex = 'M'
1039	if brain.sex:
1040	sex = 'F'
1041	da['jamb_sex'] = sex
1042	da['jamb_age'] = brain.jamb_age
1043	da['app_reg_pin'] = brain.pin
1044	da['jamb_lga'] = brain.jamb_lga
1045	da['jamb_state'] = brain.jamb_state
1046	da['jamb_score'] = brain.aggregate
1047	da['app_email'] = brain.email
1048	da['app_mobile'] = brain.phone
1049
1050	# entry_mode cannot be retrieved from the certtificate
1051	# because ug_ft has two different entry modes
1052
1053	if brain.entry_mode: # does not happen because import_application
1054	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
1055	elif brain.screening_type == 'pume':
1056	da['entry_mode'] = 'ume_ft'
1057	elif brain.screening_type == 'pde':
1058	da['entry_mode'] = 'de_ft'
1059	else:
1060	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1061
1062	#elif brain.screening_type == 'pce':
1063	# da['entry_mode'] = 'pce'
1064	#elif brain.screening_type == 'prence':
1065	# da['entry_mode'] = 'prence'
1066	#else:
1067	# da['entry_mode'] = ''
1068
1069
1070	#da['entry_session'] = entry_session
1071	da['entry_session'] = brain.entry_session
1072	da['jamb_lastname'] = brain.lastname
1073	da['jamb_middlename'] = brain.middlenames # different field names!
1074	da['jamb_firstname'] = brain.firstname
1075	da['screening_application_date'] = brain.application_date
1076	da['date_of_birth'] = brain.date_of_birth
1077	da['jamb_first_cos'] = brain.course1
1078	da['jamb_second_cos'] = brain.course2
1079	da['course3'] = brain.course3
1080	da['screening_type'] = brain.screening_type
1081	da['screening_score'] = brain.screening_score
1082	da['screening_date'] = brain.screening_date
1083	da['hq_type'] = brain.hq_type
1084	da['hq_grade'] = brain.hq_grade
1085	da['aos'] = brain.aos
1086
1087	application.getContent().edit(mapping=da)
1088	self.do_timing()
1089	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1090	#
1091	# personal
1092	#
1093	student_object.invokeFactory('StudentPersonal','personal')
1094	personal = student_object.personal
1095	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1096	#self.do_timing()
1097	dp = {'Title': 'Personal Data'}
1098	dp['sex'] = brain.sex
1099	dp['email'] = brain.email
1100	dp['phone'] = brain.phone
1101	dp['lastname'] = brain.lastname
1102	dp['middlename'] = brain.middlenames # different field names!
1103	dp['firstname'] = brain.firstname
1104	personal.getContent().edit(mapping=dp)
1105	self.do_timing()
1106	#
1107	# clearance
1108	#
1109	student_object.invokeFactory('StudentClearance','clearance')
1110	clearance = student_object.clearance
1111	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1112	dc = {'Title': 'Clearance/Eligibility Record'}
1113	dc['lga'] = brain.lga
1114	dc['birthday'] = brain.date_of_birth
1115	clearance.getContent().edit(mapping=dc)
1116	self.do_timing()
1117	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1118	#
1119	# study Course
1120	#
1121	student_object.invokeFactory('StudentStudyCourse','study_course')
1122	studycourse = student_object.study_course
1123	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1124	#self.do_timing()
1125	dsc = {}
1126	dsc['study_course'] = brain.course_admitted
1127	dsc['current_verdict'] = ''
1128	dsc['current_mode'] = da['entry_mode'] # no longer used
1129	if da['entry_mode'].startswith('de'):
1130	dsc['current_level'] = '200'
1131	elif da['entry_mode'].startswith('pre'):
1132	dsc['current_level'] = '000'
1133	else:
1134	dsc['current_level'] = '100'
1135	dsc['current_session'] = brain.entry_session
1136	studycourse.getContent().edit(mapping=dsc)
1137	self.do_timing()
1138	#
1139	# payments folder
1140	student_object.invokeFactory('PaymentsFolder','payments')
1141	payments = getattr(student_object,'payments')
1142	#self.do_timing()
1143	dpay = {}
1144	dpay['Title'] = 'Payments'
1145	payments.getContent().edit(mapping=dpay)
1146	self.portal_workflow.doActionFor(payments,'open')
1147	self.do_timing()
1148	#
1149	# passport foto
1150	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1151	brain.screening_type,
1152	brain.reg_no)
1153	images_dir = getImagesDir(student_id)
1154	#images_dir = os.path.join("%s" % images_base,student_id)
1155	letter_dir,student_dir = os.path.split(images_dir)
1156	if not os.path.exists(letter_dir):
1157	os.mkdir(letter_dir)
1158	if not os.path.exists(images_dir):
1159	os.mkdir(images_dir)
1160	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1161	if os.path.exists(app_picture):
1162	copy2(app_picture,image_name)
1163	else:
1164	logger.info('passport of %s/%s not found: %s' % (student_id,
1165	brain.reg_no,
1166	app_picture))
1167
1168	self.do_timing()
1169	self.print_timing()
1170	if with_timing:
1171	self.timer_step = 0
1172	self._v_timer_count += 1
1173	self._v_total += self.total_time
1174	return student_id
1175	###)
1176
1177	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1178	def makeStudentLevel(self,student_id):
1179	"create the StudyLevel for a returning Student"
1180	#import pdb;pdb.set_trace()
1181	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1182	students_folder = self.portal_url.getPortalObject().campus.students
1183	res = self.students_catalog(id=student_id)
1184	if res:
1185	st = res[0]
1186	course = st.course
1187	matric_no = st.matric_no
1188	level = st.level
1189	res = self.results_import(matric_no = matric_no)
1190	if res:
1191	results = res
1192	logger.info('%s creating Level %s' % (student_id,level))
1193	#
1194	# Level
1195	#
1196	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1197	studycourse = getattr(student_obj,"study_course",None)
1198	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1199	l = getattr(studycourse,level,None)
1200	if l is None:
1201	studycourse.invokeFactory('StudentStudyLevel', level)
1202	l = getattr(studycourse, level)
1203	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1204	l.getContent().edit(mapping={'Title': "Level %s" % level})
1205	###)
1206
1207	security.declarePublic('getHallInfo') ###(
1208	def getHallInfo(self,bed):
1209	"""return Hall Info"""
1210	info = {}
1211	bedsplit = bed.split('_')
1212	if len(bedsplit) == 4:
1213	hall,block,room,letter = bed.split('_')
1214	else:
1215	info['maintenance_code'] = 'None'
1216	return info
1217	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1218	if res and len(res) == 1:
1219	hall_brain = res[0]
1220	hall_doc = hall_brain.getObject().getContent()
1221	else:
1222	return info
1223	info['hall_title'] = hall_brain.Title
1224	info['maintenance_code'] = hall_doc.maintenance_code
1225	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1226	batch_doc = None
1227	for brain in res:
1228	if brain.id.startswith(info['maintenance_code']):
1229	batch_doc = brain.getObject().getContent()
1230	break
1231	if batch_doc is None:
1232	info['maintenance_fee'] = ''
1233	else:
1234	info['maintenance_fee'] = batch_doc.cost
1235	return info
1236	###)
1237
1238	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1239	def removePictureFolder(self,student_id):
1240	"""remove picture_folder by renaming it"""
1241	picture_path = getImagesDir(student_id)
1242	dest_path = os.path.join("%s" % images_base,'removed',student_id)
1243	dest_path = dest_path + "_removed"
1244	if os.path.exists(dest_path) or not os.path.exists(picture_path):
1245	return False
1246	os.rename(picture_path,dest_path)
1247	return True
1248	###)
1249
1250	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1251	def restorePictureFolder(self,student_id):
1252	"""restore picture_folder by renaming it"""
1253	picture_path = getImagesDir(student_id)
1254	orig_path = os.path.join("%s" % images_base,'removed',student_id)
1255	orig_path = orig_path + "_removed"
1256	if os.path.exists(picture_path) or not os.path.exists(orig_path):
1257	return False
1258	os.rename(picture_path + "_removed",picture_path)
1259	return True
1260	###)
1261
1262	security.declarePublic('picturesExist') ###(
1263	def picturesExist(self, ids,student_id=None):
1264	"""check if pictures exist in the filesystem"""
1265	if student_id is None:
1266	student_id = self.getStudentId()
1267	if student_id is None:
1268	return False
1269	picture_path = getImagesDir(student_id)
1270	#picture_path = os.path.join(images_base,student_id)
1271	if not os.path.exists(picture_path):
1272	return False
1273	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1274	return set(ids).issubset(set(pictures))
1275	###)
1276
1277	security.declarePublic('picturePathExists') ###(
1278	def picturePathExists(self, student_id=None):
1279	"""check if picture path exists in the filesystem"""
1280	if student_id is None:
1281	student_id = self.getStudentId()
1282	if student_id is None:
1283	return False
1284	picture_path = getImagesDir(student_id)
1285	if os.path.exists(picture_path):
1286	return True
1287	return False
1288	###)
1289
1290
1291	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1292	def removeUnusedImageFolders(self):
1293	"""check if an unused image folders exists in the filesystem"""
1294	mtool = self.portal_membership
1295	member = mtool.getAuthenticatedMember()
1296	member_id = str(member)
1297	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1298	abc = os.listdir(images_base)
1299	ifolders = []
1300	for i in abc:
1301	picture_path = os.path.join(images_base,i)
1302	ifolders.extend(os.listdir(picture_path))
1303	unused_ids = []
1304	for id in ifolders:
1305	res = self.students_catalog(id=id)
1306	if not res:
1307	unused_ids.append(id)
1308	#import pdb;pdb.set_trace()
1309	if not id.endswith('removed'):
1310	removed = self.waeup_tool.removePictureFolder(id)
1311	if removed:
1312	logger.info('%s: image folder %s successfully removed' % (member_id,id))
1313	else:
1314	logger.info('%s: image folder %s could not be removed' % (member_id,id))
1315	return
1316
1317	###)
1318
1319	security.declarePublic('picturesList') ###(
1320	def picturesList(self):
1321	"""check if pictures exist in the filesystem"""
1322	path = 'images'
1323	student_id = self.getStudentId()
1324	#picture_path = os.path.join(i_home,path,student_id)
1325	picture_path = getImagesDir(student_id)
1326	if not os.path.exists(picture_path):
1327	return []
1328	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1329	###)
1330
1331	security.declarePublic('showFsPicture') ###(
1332	def showFsPicture(self,path):
1333	"""return a picture from the filesystem"""
1334	#picture_path = os.path.join(i_home,path)
1335	picture_path = os.path.join(images_base,path)
1336	response = self.REQUEST.RESPONSE
1337	#import pdb;pdb.set_trace()
1338	registry = getToolByName(self, 'mimetypes_registry')
1339	mimetype = str(registry.lookupExtension(path.lower()) or
1340	registry.lookupExtension('file.bin'))
1341	if os.path.exists(picture_path):
1342	response.setHeader('Content-type',mimetype)
1343	return open(picture_path).read()
1344	picture_path = os.path.join(i_home,'import',path)
1345	if os.path.exists(picture_path):
1346	return open(picture_path).read()
1347	###)
1348
1349	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1350	def deleteAllCourses(self,department="All"):
1351	''' delete the courses'''
1352	pm = self.portal_membership
1353	member = pm.getAuthenticatedMember()
1354
1355	if str(member) not in ("henrik","joachim"):
1356	return "not possible"
1357	if department == "All":
1358	res = self.portal_catalog({'meta_type': 'Department'})
1359	if len(res) < 1:
1360	return "No Departments found"
1361
1362	deleted = []
1363	for dep in res:
1364	cf = dep.getObject().courses
1365	if cf:
1366	cf.manage_delObjects(ids=cf.objectIds())
1367	deleted.append("deleted Courses in %s" % dep.getId)
1368	return "\r".join(deleted)
1369	###)
1370
1371	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1372	def getLogfileLines(self,filename="event.log",numlines=20):
1373	"""Get last NUMLINES lines of logfile FILENAME.
1374
1375	Return last lines' of a file in the instances logfile directory as
1376	a list. The number of returned lines equals `numlines' or less. If
1377	less than `numlines' lines are available, the whole file ist
1378	returned. If the file can not be opened or some other error
1379	occurs, empty list is returend.
1380	"""
1381	result = []
1382	lines_hit = 0
1383
1384	# We only handle files in instances' log directory...
1385	logpath = os.path.join(i_home, "log")
1386	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1387	if not filename.startswith( logpath ):
1388	# Attempt to access file outside log-dir...
1389	return []
1390
1391	try:
1392	fd = file( filename, "rb" )
1393	except IOError:
1394	return []
1395	if not fd:
1396	return []
1397
1398	if os.linesep == None:
1399	linesep = '\n'
1400	else:
1401	linesep = os.linesep
1402
1403	# Try to find 'numlines' times a lineseparator, searching from end
1404	# and moving to the beginning of file...
1405	fd.seek( 0, 2) # Move to end of file...
1406	while lines_hit < numlines:
1407	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1408	# one step forward...
1409	lines_hit += 1
1410	try:
1411	fd.seek( -2, 1) # Go two bytes back from current pos...
1412	except IOError:
1413	# We cannot go back two bytes. Maybe the file is too small...
1414	break
1415	fd.seek(2,1)
1416
1417	# Read all lines from current position...
1418	result = fd.readlines()
1419	# Remove line endings...
1420	result = [x.strip() for x in result]
1421	fd.close()
1422	return result
1423	###)
1424
1425	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1426	def getCallbacksFromLog(self,filename):
1427	"""fix Online Payment Transactions from Z2.log entries"""
1428	import transaction
1429	import random
1430	from cgi import parse_qs
1431	from urlparse import urlparse
1432	#from pdb import set_trace
1433	wftool = self.portal_workflow
1434	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1435	students_folder = self.portal_url.getPortalObject().campus.students
1436	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1437	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1438	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1439	data = re.compile(s)
1440	start = True
1441	tr_count = 1
1442	total = 0
1443	#name = 'pume_results'
1444	#name = 'epaymentsuccessful_z2log2'
1445	name = filename
1446	no_import = []
1447	imported = []
1448	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1449	try:
1450	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1451	except:
1452	logger.error('Error reading %s' % name)
1453	return
1454	tas = []
1455	for line in transactions:
1456	dict = {}
1457	items = data.search(line)
1458	dict['idict'] = idict = items.groupdict()
1459	#print idict
1460	#from pdb import set_trace;set_trace()
1461	urlparsed = urlparse(idict['get'][4:])
1462	#print urlparsed
1463	path = urlparsed[2].split('/')
1464	dict['student_id'] = student_id = path[8]
1465	dict['payment_id'] = payment_id = path[10]
1466	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1467	tas.append(dict)
1468	tr_count += 1
1469	return tas
1470	###)
1471
1472	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1473	def importOnlinePaymentTransactions(self):
1474	"""load Online Payment Transactions from CSV values"""
1475	import transaction
1476	import random
1477	#from pdb import set_trace
1478	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1479	opt = self.online_payments_import
1480	students_folder = self.portal_url.getPortalObject().campus.students
1481	start = True
1482	tr_count = 1
1483	total = 0
1484	#name = 'pume_results'
1485	name = 'OnlineTransactions'
1486	no_import = []
1487	imported = []
1488	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1489	try:
1490	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1491	except:
1492	logger.error('Error reading %s.csv' % name)
1493	return
1494	for pay_transaction in transactions:
1495	if start:
1496	start = False
1497	logger.info('Start loading from %s.csv' % name)
1498	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1499	no_import.append('%s,"Error"' % s)
1500	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1501	format_error = format + ',"%(Error)s"'
1502	data = {}
1503
1504	# format of the first file sent by Tayo
1505	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1506	#data['student_id'] = student_id = pay_transaction['Payer ID']
1507	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1508	#data['response_code'] = response_code = pay_transaction['Resp Code']
1509	#data['amount'] = amount = pay_transaction['Amount']
1510
1511	# format of the second file sent by Tayo
1512	#data['datetime'] = date = 0
1513	#data['student_id'] = student_id = pay_transaction['Payer ID']
1514	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1515	#data['response_code'] = response_code = '00'
1516	#data['amount'] = amount = pay_transaction['Amount']
1517
1518	# format of the third file sent by Kehinde
1519	data['datetime'] = date = 0
1520	data['student_id'] = student_id = pay_transaction['customer_id']
1521	data['order_id'] = order_id = pay_transaction['merchant_reference']
1522	data['response_code'] = response_code = '00'
1523	data['amount'] = amount = pay_transaction['Amount']
1524
1525	dup = False
1526	if response_code == "12":
1527	continue
1528	try:
1529	opt.addRecord(**data)
1530	except ValueError:
1531	dup = True
1532	#from pdb import set_trace;set_trace()
1533	if dup:
1534	if response_code == "00":
1535	try:
1536	opt.modifyRecord(**data)
1537	except:
1538	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1539	continue
1540	else:
1541	pay_transaction['Error'] = "Duplicate order_id"
1542	no_import.append( format_error % pay_transaction)
1543	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1544	continue
1545	tr_count += 1
1546	if tr_count > 1000:
1547	if len(no_import) > 0:
1548	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1549	'\n'.join(no_import) + '\n')
1550	no_import = []
1551	em = '%d transactions committed\n' % (tr_count)
1552	transaction.commit()
1553	regs = []
1554	logger.info(em)
1555	total += tr_count
1556	tr_count = 0
1557	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1558	'\n'.join(no_import))
1559	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1560	###)
1561
1562	security.declareProtected(ModifyPortalContent,"importData")###(
1563	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1564	"""load data from CSV values"""
1565	import transaction
1566	import random
1567	students_folder = self.portal_url.getPortalObject().campus.students
1568	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1569	pending_only = False
1570	pend_str = '--'
1571	elapse = time.time()
1572	#
1573	# preparations
1574	#
1575	if filename == pend_str:
1576	pending_only = True
1577	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1578	importer = eval("%sImport" % importer_name)(self)
1579	logger = importer.logger
1580	if importer.init_errors:
1581	logger.info(importer.init_errors)
1582	return importer.init_errors
1583	member = importer.member
1584	#current = importer.current
1585	import_date = importer.import_date
1586	#
1587	# not_imported
1588	#
1589	info = importer.info
1590	data_keys = importer.data_keys
1591	csv_keys = importer.csv_keys
1592	#csv_keys.extend(info.keys())
1593	headline_mapping = dict((k,k) for k in csv_keys)
1594	#
1595	# pending
1596	#
1597	pending_path = importer.pending_path
1598	pending_tmp = importer.pending_tmp
1599	pending_backup = importer.pending_backup
1600	pending_fn = importer.pending_fn
1601	imported_path = importer.imported_path
1602	imported_fn = importer.imported_fn
1603	commit_after = importer.commit_after
1604	pending = []
1605	pending_digests = []
1606	#total_added_to_pending = 0
1607	if not pending_only:
1608	pending,pending_digests = importer.makeIdLists()
1609	pending_at_start = len(pending)
1610	datafile = open(pending_tmp,"w")
1611	pending_csv_writer = csv.DictWriter(datafile,
1612	csv_keys,
1613	extrasaction='ignore')
1614	pending_csv_writer.writerow(headline_mapping)
1615	datafile.close()
1616	#
1617	# imported
1618	#
1619	if not os.path.exists(imported_path):
1620	datafile = open(imported_path,"w")
1621	imported_csv_writer = csv.DictWriter(datafile,
1622	csv_keys,
1623	extrasaction='ignore')
1624	imported_csv_writer.writerow(headline_mapping)
1625	datafile.close()
1626	start = True
1627	tr_count = 0
1628	total = 0
1629	total_added_to_imported = 0
1630	total_pending = 0
1631	import_source_done = ""
1632	if pending_only:
1633	import_source_path = pending_path
1634	else:
1635	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1636	import_source_done = "%s/import/%s.done" % (i_home,filename)
1637	if not os.path.exists(import_source_path):
1638	fn = os.path.split(import_source_path)[1]
1639	em = 'no import file %(fn)s' % vars()
1640	return em
1641	import_source_fn = os.path.split(import_source_path)[1]
1642	if not pending_only:
1643	info['imported_from'] = import_source_fn
1644	headline = csv.reader(open(import_source_path,"rb")).next()
1645	if "import_mode" not in headline:
1646	msg = 'import_mode must be in heading'
1647	return msg
1648	invalid_keys = importer.checkHeadline(headline)
1649	if invalid_keys:
1650	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1651
1652	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1653	or k.strip() in info.keys())]
1654	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1655	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1656	# if diff2schema and diff2schema != set(['id',]):
1657	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1658	# return msg
1659	#
1660	# start importing
1661	#
1662	try:
1663	reader = csv.DictReader(open(import_source_path,"rb"))
1664	except:
1665	msg = 'Error reading %s.csv' % filename
1666	logger.error(msg)
1667	return msg
1668	items = [item for item in reader]
1669	total_to_import = len(items)
1670	tti_float = float(total_to_import)
1671	if pending_only:
1672	pending_at_start = total_to_import
1673	count = 0
1674	imported = []
1675	old_commit_count = 0
1676	error_count = imported_count = 0
1677	already_in = 0
1678	for record in items:
1679	item = {}
1680	empty_value_keys = []
1681	for k,v in record.items():
1682	if k is None:
1683	continue
1684	if v:
1685	if v == EMPTY:
1686	empty_value_keys += k,
1687	v = ''
1688	item[k.strip()] = v.strip()
1689	count += 1
1690	if start:
1691	start = False
1692	adapters = [MappingStorageAdapter(importer.schema, item)]
1693	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1694	dm = DataModel(item, adapters,context=self)
1695	ds = DataStructure(data=item,datamodel=dm)
1696	error_string = ""
1697	total += 1
1698	import_mode = item.get('import_mode','')
1699	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1700	if import_method is None:
1701	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1702	elif (import_mode in importer.required_modes and
1703	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1704	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1705	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1706	else:
1707	for k in import_keys:
1708	if k not in item.keys() or k not in importer.validators.keys():
1709	continue
1710	if not importer.validators[k](ds,mode=import_mode):
1711	error_string += ' ++ '
1712	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1713	ds.getErrorMapping(k)))
1714	if error_string:
1715	error = error_string
1716	id = ''
1717	mapping = item
1718	else:
1719	temp_item = item.copy()
1720	temp_item.update(dm)
1721	results = import_method(temp_item)
1722	id = results[0]
1723	error = results[1]
1724	mapping = results[2]
1725	#
1726	# restore some values
1727	#
1728	for k in empty_value_keys:
1729	mapping[k] = EMPTY
1730	if mapping.has_key('sex'):
1731	#import pdb;pdb.set_trace()
1732	if mapping['sex'] == True:
1733	mapping['sex'] = 'F'
1734	elif mapping['sex'] == False:
1735	mapping['sex'] = 'M'
1736	if pending_only:
1737	info['imported_from'] = item['imported_from']
1738	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1739	info['error'] = error
1740	info['import_record_no'] = count + 1
1741	mapping.update(info)
1742	log_list = []
1743	if error:
1744	error_count += 1
1745	digest = makeDigest(mapping,data_keys)
1746	if digest not in pending_digests:
1747	pending_digests += digest,
1748	pending.append(mapping)
1749	if not pending_only:
1750	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1751	else:
1752	already_in += 1
1753	pass
1754	else:
1755	imported_count += 1
1756	imported += mapping,
1757	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1758	if log_list:
1759	time_till_now = time.time() - elapse
1760	percent_finished = (error_count + imported_count)/tti_float*100
1761	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1762	logger.info(' '.join(log_list))
1763	finished = count > total_to_import - 1
1764	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1765	if must_commit:
1766	old_commit_count = imported_count
1767
1768	if must_commit or finished:
1769	if len(imported):
1770	transaction.commit()
1771	datafile = open(imported_path,"a")
1772	writer = csv.DictWriter(datafile,
1773	csv_keys,
1774	extrasaction='ignore')
1775	writer.writerows(imported)
1776	datafile.close()
1777	total_added_to_imported += len(imported)
1778	imported = []
1779	if len(pending) > 0:
1780	datafile = open(pending_tmp,"a")
1781	writer = csv.DictWriter(datafile,
1782	csv_keys,
1783	extrasaction='ignore')
1784	writer.writerows(pending)
1785	datafile.close()
1786	total_pending += len(pending)
1787	#total_added_to_pending += len(pending)
1788	pending = []
1789	if not finished:
1790	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1791	logger.info(msg)
1792	elapse = time.time() - elapse
1793	if os.path.exists(pending_path):
1794	copy2(pending_path,pending_backup)
1795	copy2(pending_tmp,pending_path)
1796	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1797	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1798	if pending_only:
1799	removed_pending = pending_at_start - total_pending
1800	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1801	else:
1802	added_pending = total_pending - pending_at_start
1803	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1804	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1805	logger.info(msg)
1806	if import_source_done:
1807	copy(import_source_path,import_source_done)
1808	os.remove(import_source_path)
1809	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1810	if upload is not None:
1811	upload_doc = upload.getContent()
1812	mapping = {}
1813	#mapping['import_date'] = DateTime.DateTime()
1814	mapping['import_date'] = import_date
1815	mapping['imported_by'] = importer.imported_by
1816	mapping['import_message'] = msg
1817	upload_doc.edit(mapping = mapping)
1818	os.remove(pending_tmp)
1819	return msg
1820	###)
1821
1822	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1823	def moveImagesToFS(self,student_id="O738726"):
1824	"move the images to the filesystem"
1825	images_dir = getImagesDir(student_id)
1826	#images_dir = os.path.join("%s" % images_base,student_id)
1827	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1828	stool = getToolByName(self, 'portal_schemas')
1829	schemas = ['student_application',
1830	'student_clearance',
1831	]
1832	created = False
1833	for schema_id in schemas:
1834	schema = stool._getOb(schema_id)
1835	object = getattr(student_folder,schema_id[len('student_'):],None)
1836	if object is None:
1837	continue
1838	doc = object.getContent()
1839	for key in schema.keys():
1840	if schema[key].meta_type != "CPS Image Field":
1841	continue
1842	#import pdb;pdb.set_trace()
1843	image = getattr(doc,key,None)
1844	if not image or not hasattr(image,"data"):
1845	continue
1846	if not created:
1847	if not os.path.exists(images_dir):
1848	os.mkdir(images_dir)
1849	created = True
1850	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1851	open(filename,"wb").write(str(image.data))
1852	###)
1853
1854	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1855	def movePassportToFS(self,student_id="O738726"):
1856	"move the passports to the filesystem"
1857	images_dir = os.path.join("%s" % i_home,'passports')
1858	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1859	stool = getToolByName(self, 'portal_schemas')
1860	schemas = ['student_application',
1861	#'student_clearance',
1862	]
1863	created = False
1864	for schema_id in schemas:
1865	schema = stool._getOb(schema_id)
1866	object = getattr(student_folder,schema_id[len('student_'):],None)
1867	if object is None:
1868	continue
1869	doc = object.getContent()
1870	for key in schema.keys():
1871	if schema[key].meta_type != "CPS Image Field":
1872	continue
1873	#import pdb;pdb.set_trace()
1874	image = getattr(doc,key)
1875	if not hasattr(image,"data"):
1876	continue
1877	if not created:
1878	if not os.path.exists(images_dir):
1879	os.mkdir(images_dir)
1880	created = True
1881	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1882	open(filename,"wb").write(str(image.data))
1883	###)
1884
1885
1886	security.declarePublic('getConfigParams')
1887	def getConfigParams(self,conf_id="configuration"):
1888	conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
1889	conf_obj = conf.getContent()
1890	stool = getToolByName(self, 'portal_schemas')
1891	schema = stool._getOb('configuration')
1892	d = {}
1893	for key in schema.keys():
1894	d[key] = getattr(conf_obj,key,None)
1895	return d
1896
1897
1898	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: