Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3362

Last change on this file since 3362 was 3350, checked in by joachim, 17 years ago
some fixes example with_timing enabled
Property svn:keywords set to `Id`
File size: 72.6 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3350 2008-03-18 18:24:53Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108
109	###)
110
111	security.declareProtected(View,'updateRoleMappingsFor') ###(
112	def updateRoleMappingsFor(self,wf_definition,ob):
113	"do so for public"
114	wf_def = getattr(self.portal_workflow,wf_definition)
115	wf_def.updateRoleMappingsFor(ob)
116	###)
117
118	security.declareProtected(View,'getStatesLgas') ###(
119	def getStatesLgas(self):
120	"""return lga info"""
121	voc = getattr(self.portal_vocabularies,'local_gov_areas')
122	states = []
123	lgas = []
124	d = {}
125	wd = {}
126	for k,v in voc.items():
127	parts = v.split(' / ')
128	if len(parts) == 1:
129	state = parts[0].lower()
130	lga = ""
131	elif len(parts) == 2:
132	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134	else:
135	continue
136	if state not in states:
137	states.append(state)
138	if lga not in lgas:
139	lgas.append(lga)
140	words = re.split('[^a-zA-Z0-9/]',k)
141	words.sort()
142	wd[k] = words
143	d[k] = v
144	mapping = {}
145	mapping['word_dict'] = wd
146	mapping['lga_dict'] = d
147	mapping['states'] = states
148	mapping['lgas'] = lgas
149	return mapping
150	###)
151
152	security.declareProtected(View,'findLga') ###(
153	def findLga(self,words,words_dict):
154	words = re.split('[^a-zA-Z0-9/]',words)
155	lga_words = []
156	for word in words:
157	if word:
158	lga_words += word.strip().lower(),
159	lga_words.sort()
160	state_lga = ''
161	while not state_lga:
162	for k,l in words_dict.items():
163	if lga_words == l:
164	state_lga = k
165	break
166	break
167	return state_lga
168	###)
169	security.declareProtected(View,'getAccessInfo') ###(
170	def getAccessInfo(self,context):
171	"return a dict with access_info"
172	logger = logging.getLogger('WAeUPTool.getAccessInfo')
173	mtool = self.portal_membership
174	member = mtool.getAuthenticatedMember()
175	member_id = str(member)
176	info = {}
177	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179	is_staff = info['is_staff'] = not is_anonymous and not is_student
180	roles = member.getRolesInContext(context)
181	is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182	"SectionManager" in roles or
183	"Manager" in roles)
184	is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if is_allowed:
188	if not is_student and requested_id:
189	student_id = requested_id
190	elif is_anonymous or member_id != requested_id:
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	else:
193	student_id = member_id
194	info['student_id'] = student_id
195	return info
196	###)
197
198	security.declareProtected(ModifyPortalContent,'openLog') ###(
199	def openLog(self,name):
200	"""open a log file"""
201	version = 1
202	path = "%s/log/%s_%d.log" % (i_home,name,version)
203	while os.path.exists(path):
204	version += 1
205	path = "%s/log/%s_%d.log" % (i_home,name,version)
206	log = open(path,"w")
207	return log
208	###)
209
210	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211	def bypassQueueCatalog(self,enable=True):
212	"""bypass the QueueCatalog by setting all indexes to process imediate,
213	if enable is True (default) the old settings are restored
214	"""
215
216	###)
217
218	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220	"""measure Object access Time"""
221	import random
222	if hasattr(self,'portal_catalog_real'):
223	aq_portal = self.portal_catalog_real.evalAdvancedQuery
224	else:
225	aq_portal = self.portal_catalog.evalAdvancedQuery
226	nr_pts = int(nr_pts)
227	probe = int(probe)
228	intervall = probe/10
229	objects = ("application","clearance","personal")
230	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231	#i = random.randrange(num_objects)
232	count = 0
233	found = 0
234	not_found = 0
235	t_found = 0
236	t_not_found = 0
237	time_found = time_not_found = 0.0
238	t_time_found = t_time_not_found = 0.0
239	accessed = []
240	t_min = 1000
241	t_max = 0
242	#import pdb;pdb.set_trace()
243	students = self.portal_catalog(portal_type="Student")
244	num_students = len(students)
245	if method == "d":
246	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247	res = aq_portal(query)
248	brains = {}
249	for r in res:
250	sid = r.relative_path.split('/')[-2]
251	if brains.has_key(sid):
252	brains[sid][r.portal_type] = r
253	else:
254	brains[sid] = {r.portal_type : r}
255	brains_list = brains.keys()
256	num_objects = len(brains_list)
257	else:
258	num_objects = num_students
259	print "="*40
260	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261	probe,
262	nr_pts,
263	num_objects)
264	print "nr found/not time found/not min/max"
265	elapse = time.time()
266	i_elapse = time.time()
267	c_elapse = time.clock()
268	for c in range(1,probe + 1):
269	i = random.randrange(num_objects)
270	if method in ('a','b','c'):
271	student_brain = students[i]
272	elif method == "d":
273	#import pdb;pdb.set_trace()
274	student_brain = brains[brains_list[i]]
275	if method == "c":
276	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277	res = aq_portal(query)
278	this_portal_types = [r.portal_type for r in res]
279	for i in range(nr_pts):
280	oid = objects[i]
281	if method == "a":
282	try:
283	student_path = student_brain.getPath()
284	path = "%s/%s" % (student_path,oid)
285	doc = self.unrestrictedTraverse(path).getContent()
286	found += 1
287	i_time = time.time() - i_elapse
288	time_found += i_time
289	except:
290	not_found += 1
291	i_time = time.time() - i_elapse
292	time_not_found += i_time
293	pass
294	elif method == "b":
295	try:
296	student_object = student_brain.getObject()
297	doc = getattr(student_object,oid).getContent()
298	found += 1
299	i_time = time.time() - i_elapse
300	time_found += i_time
301	except:
302	i_time = time.time() - i_elapse
303	time_not_found += i_time
304	not_found += 1
305	pass
306	elif method == "c":
307	if portal_types[i] in this_portal_types:
308	found += 1
309	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310	i_time = time.time() - i_elapse
311	time_found += i_time
312	else:
313	not_found += 1
314	i_time = time.time() - i_elapse
315	time_not_found += i_time
316	elif method == "d":
317	if student_brain.has_key(portal_types[i]):
318	found += 1
319	doc = student_brain[portal_types[i]].getObject().getContent()
320	i_time = time.time() - i_elapse
321	time_found += i_time
322	else:
323	not_found += 1
324	i_time = time.time() - i_elapse
325	time_not_found += i_time
326	i_elapse = time.time()
327	if c and (c % intervall == 0):
328	#i_time = time.time() - i_elapse
329	t_per = 0.0
330	if found:
331	t_per = time_found/found
332	if t_per > t_max:
333	t_max = t_per
334	if t_per > 0.0 and t_per < t_min:
335	t_min = t_per
336	itf = 0.0
337	if found:
338	itf = time_found/found
339	itnf = 0.0
340	if not_found :
341	itnf = time_not_found / not_found
342	interval_time = time_found + time_not_found
343	s = "%(c)d: %(found)d/%(not_found)d " % vars()
344	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
346	print s
347	t_found += found
348	t_not_found += not_found
349	t_time_found += time_found
350	t_time_not_found += time_not_found
351	time_found = time_not_found = 0.0
352	found = not_found = 0
353	# t_found += found
354	# t_not_found += not_found
355	elapse = time.time() - elapse
356	itf = 0.0
357	if t_found:
358	itf = t_time_found/t_found
359	itnf = 0.0
360	if t_not_found:
361	itnf = t_time_not_found / t_not_found
362	#c_elapse = time.clock() - c_elapse
363	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
366	print "-"*40
367	print s
368	rel_found = float(t_found)/probe
369	rel_not_found = float(t_not_found)/probe
370	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
371	print estimated_total_time
372	###)
373
374	security.declareProtected(ModifyPortalContent,'writeLog') ###(
375	def writeLog(self,logfile,s):
376	"""write to the log file"""
377	logfile.write(s)
378
379	###)
380
381	def generateStudentId(self,letter): ###(
382	import random
383	r = random
384	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385	if letter == '?':
386	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387	sid = "%c%d" % (letter,r.randint(99999,1000000))
388	students = self.portal_url.getPortalObject().campus.students
389	## while hasattr(students, sid):
390	## sid = "%c%d" % (letter,r.randint(99999,1000000))
391	while self.students_catalog(id = sid):
392	sid = "%c%d" % (letter,r.randint(99999,1000000))
393	return sid
394	###)
395
396	def generatePassword(self,s=None): ###(
397	import random
398	r = random
399	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400	if s is None:
401	s = 'abcdefghklmnpqrstuvwxy23456789'
402	pw = ''
403	while len(pw) < 6:
404	pw += r.choice(s)
405	return pw
406	###)
407
408	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409	def dumpSchoolfeePayments(self):
410	"dump paid schoolfees"
411	mtool = self.portal_membership
412	member = mtool.getAuthenticatedMember()
413	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414	aq_student = self.students_catalog.evalAdvancedQuery
415	query = In('review_state',('schoolfee_paid',
416	'courses_registered',
417	'courses_validated',
418	))
419	res = aq_student(query)
420	#import pdb;pdb.set_trace()
421	l = []
422	logger.info("start for %d" % len(res))
423	count = 1
424	log_after = 100
425	for student in res:
426	if not count % log_after:
427	logger.info("processed %d total %d" % (log_after,count))
428	count += 1
429	fee_dict =self.getSchoolFee(student)
430	fulltime = student.mode.endswith('_ft')
431	d = {}
432	d['student_id'] = student.id
433	d['name'] = student.name
434	d['amount'] = fee_dict.get(new_returning)
435	l += d,
436	csv_name = self.dumpListToCSV(l,'payments')
437	logger.info('%s dumped payments to %s' % (member,export_file))
438	###)
439
440	security.declarePublic('dumpListToCSV') ###(
441	def dumpListToCSV(self,l,filename,fields=None):
442	"""dump a list of dicts to a CSV file"""
443	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445	if fields is None:
446	fields = l[0].keys()
447	headline = ','.join(fields)
448	out = open(export_file,"wb")
449	out.write(headline +'\n')
450	out.close()
451	out = open(export_file,"a")
452	csv_writer = csv.DictWriter(out,fields,)
453	csv_writer.writerows(l)
454	return export_file
455	###)
456
457	security.declareProtected(ManagePortal, 'listMembers') ###(
458	def listMembers(self):
459	"list all members"
460	mtool = self.portal_membership
461	member = mtool.getAuthenticatedMember()
462	logger = logging.getLogger('WAeUPTool.listMembers')
463	if str(member) not in ('admin','joachim'):
464	logger.info('%s tried to list members' % (member))
465	return None
466	members = self.portal_directories.members
467	all = members.listEntryIdsAndTitles()
468	l = []
469	for user_id,name in all:
470	d = {}
471	d['user_id'] = user_id
472	d['name'] = name
473	d['pw'] = getattr(getattr(members,user_id),'password')
474	d['email'] = getattr(getattr(members,user_id),'email')
475	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477	l += d,
478	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480	logger.info('%s dumped member list to %s' % (member,export_file))
481	fields = l[0].keys()
482	headline = ','.join(fields)
483	out = open(export_file,"wb")
484	out.write(headline +'\n')
485	out.close()
486	out = open(export_file,"a")
487	csv_writer = csv.DictWriter(out,fields,)
488	csv_writer.writerows(l)
489	###)
490
491	security.declareProtected(ManagePortal, 'listStudents') ###(
492	def listStudents(self):
493	"list all students"
494	mtool = self.portal_membership
495	member = mtool.getAuthenticatedMember()
496	logger = logging.getLogger('WAeUPTool.listStudents')
497	if str(member) not in ('admin','joachim'):
498	logger.info('%s tried to list students' % (member))
499	return None
500	students = self.portal_directories.students
501	all = students.listEntryIdsAndTitles()
502	l = []
503	for user_id,name in all:
504	d = {}
505	d['user_id'] = user_id
506	d['name'] = name
507	d['pw'] = getattr(getattr(students,user_id),'password')
508	d['email'] = getattr(getattr(students,user_id),'email')
509	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511	l += d,
512	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514	logger.info('%s dumped student list to %s' % (member,export_file))
515	fields = l[0].keys()
516	headline = ','.join(fields)
517	out = open(export_file,"wb")
518	out.write(headline +'\n')
519	out.close()
520	out = open(export_file,"a")
521	csv_writer = csv.DictWriter(out,fields,)
522	csv_writer.writerows(l)
523	###)
524
525	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526	def removeDeletedDocIds(self, max=1000):
527	"""
528	remove deleted docids from repository commit after max
529	"""
530	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531	repository = getToolByName(self, 'portal_repository')
532	pxtool = getToolByName(self, 'portal_proxies')
533	logger.info('start')
534	pxtool_infos = pxtool.getRevisionsUsed()
535	logger.info('found %d used revisions ' % (len(pxtool_infos)))
536
537	nb_revs = 0
538	docids_d = {} # all docids
539	unused_docids_d = {} # all docids that are unused
540	ids_unused_revs_docids = [] # ids for revs of unused docids
541	unused_ids = [] # ids for unused revs
542	total = 0
543	idlist = repository.objectIds()
544	to_delete = 0
545	found = False
546	for id in idlist:
547	docid, rev = repository._splitId(id)
548	if docid is None:
549	logger.info("invalid doc_id %s" % docid)
550	continue
551	nb_revs += 1
552	if not pxtool_infos.has_key(docid):
553	found = True
554	to_delete += 1
555	unused_ids.append(id)
556	elif not pxtool_infos[docid].has_key(rev):
557	found = True
558	to_delete += 1
559	unused_ids.append(id)
560	if found and not to_delete % max:
561	found = False
562	#import pdb;pdb.set_trace()
563	repository.manage_delObjects(unused_ids)
564	transaction.commit()
565	logger.info('removed %d total %d unused docids ' % (max,to_delete))
566	else:
567	if unused_ids:
568	repository.manage_delObjects(unused_ids)
569	transaction.commit()
570	logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573	###)
574
575	security.declareProtected(View,'getCredential') ###(
576	def getCredential(self,student_id):
577	student_entry = getattr(self.portal_directories.students,student_id,None)
578	if not self.isStaff():
579	mtool = self.portal_membership
580	member = mtool.getAuthenticatedMember()
581	logger = logging.getLogger('WAeUPTool.getCredential')
582	logger.info('%s tried to access password of %s' % (member,student_id))
583	return None
584	if student_entry is None:
585	return None
586	return getattr(student_entry,"password","not set")
587	###)
588
589	security.declarePublic('checkPassword') ###(
590	def checkPassword(self,student_id,password):
591	student_entry = getattr(self.portal_directories.students,student_id,None)
592	if student_entry is None:
593	return False
594	return getattr(student_entry,"password","not set") == password
595	###)
596
597	security.declarePublic('checkGenericPassword') ###(
598	def checkGenericPassword(self,member_id):
599	member_entry = getattr(self.portal_directories.members,member_id,None)
600	if member_entry is None:
601	return False
602	ltool = getToolByName(self, 'portal_layouts')
603	unsecure_words = ltool._getOb('members')['w__password'].check_words
604	password = getattr(member_entry,"password","not set")
605	is_unsecure = password in unsecure_words
606	if is_unsecure:
607	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609	return is_unsecure
610	###)
611
612	security.declareProtected(ModifyPortalContent,'editPassword') ###(
613	def editPassword(self,student_id,password):
614	"edit a student password"
615	student_entry = getattr(self.portal_directories.students,student_id,None)
616	if student_entry is None:
617	return
618	setattr(student_entry,'password',password)
619	###)
620
621	security.declareProtected(ModifyPortalContent,'doCommit') ###(
622	def doCommit(self,logger=None):
623	"commit some transactions"
624	transaction.commit()
625	###)
626
627	security.declarePublic('loadStudentFoto') ###(
628	def loadStudentFoto(self,student,filename,folder):
629	"return a student passport picture"
630	#import pdb;pdb.set_trace()
631	picture ="%s/import/%s/%s" % (i_home,folder,filename)
632	student_id = student.getId()
633	images_dir = getImagesDir(student_id)
634	if not os.path.exists(images_dir):
635	os.mkdir(images_dir)
636	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637	for extension in ('.jpg','.JPG'):
638	fullname = "%(picture)s%(extension)s" % vars()
639	if os.path.exists(fullname):
640	copy2(fullname,image_name)
641	return "successfully copied passport picture"
642	return "passport picture not found: %s.jpg or .JPG" % picture
643	###)
644
645	def old____loadStudentFoto(self,student,filename,folder): ###(
646	"return a student passport picture"
647	app = student.application
648	app_doc = app.getContent()
649	#clear = student.clearance
650	#clear_doc = clear.getContent()
651	#matric_no = clear_doc.matric_no.upper()
652	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654	#import pdb;pdb.set_trace()
655	if os.path.exists(picture1):
656	file = open(picture1)
657	elif os.path.exists(picture2):
658	file = open(picture2)
659	else:
660	return "passport picture not found %s" % picture1
661	reopened = False
662	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663	self.portal_workflow.doActionFor(app,'open')
664	reopened = True
665	outfile = file.read()
666	app_doc.manage_addFile('passport',
667	file=outfile,
668	title="%s.jpg" % filename)
669	if reopened:
670	self.portal_workflow.doActionFor(app,'close')
671	return "successfully loaded passport picture"
672	###)
673
674	security.declareProtected(ModifyPortalContent,'createOne') ###(
675	def createOne(self,students_folder,student_brain,letter,commit=False):
676	sid = self.waeup_tool.generateStudentId(letter)
677	students_folder.invokeFactory('Student', sid)
678	student = getattr(students_folder,sid)
679	self.portal_workflow.doActionFor(student,'return')
680	student.manage_setLocalRoles(sid, ['Owner',])
681	matric_no = student_brain.matric_no
682	jamb_reg_no = student_brain.Entryregno
683	self.students_catalog.addRecord(id = sid,
684	matric_no = matric_no,
685	jamb_reg_no = jamb_reg_no,
686	sex = student_brain.Sex == "F",
687	name = "%s %s %s" % (student_brain.Firstname,
688	student_brain.Middlename,
689	student_brain.Lastname)
690	)
691	if commit:
692	transaction.commit()
693	return sid,jamb_reg_no
694	###)
695
696	security.declareProtected(ModifyPortalContent,'addStudent') ###(
697	def addStudent(self,dict):
698	students_folder = self.portal_url.getPortalObject().campus.students
699	sid = self.waeup_tool.generateStudentId('?')
700	students_folder.invokeFactory('Student', sid)
701	student_obj = getattr(students_folder,sid)
702	f2t = self.field2types_student
703	#from pdb import set_trace; set_trace()
704	d = {}
705	#d['jamb_sex'] = 'M'
706	#if dict.get('sex'):
707	# d['jamb_sex'] = 'F'
708
709	entry_session = dict.get('entry_session')
710	if entry_session == self.getSessionId()[0]:
711	wfaction = 'admit'
712	wft = 'wf_transition_admit'
713	password = None
714	else:
715	wfaction = 'return'
716	wft = 'wf_transition_return'
717	password = self.generatePassword()
718	self.makeStudentMember(sid,password)
719
720	for pt in f2t.keys():
721	student_obj.invokeFactory(pt,f2t[pt]['id'])
722	sub_obj = getattr(student_obj,f2t[pt]['id'])
723	sub_doc = sub_obj.getContent()
724	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725	d['Title'] = f2t[pt]['title']
726	for field in f2t[pt]['fields']:
727	d[field] = dict.get(field,'')
728	sub_doc.edit(mapping = d)
729	new_state = f2t[pt][wft]
730	if new_state != "remain":
731	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732	self.portal_workflow.doActionFor(student_obj,wfaction)
733	student_obj.manage_setLocalRoles(sid, ['Owner',])
734	return sid,password
735	###)
736
737	security.declarePublic('getCertificateBrain') ###(
738	def getCertificateBrain(self,cert_id):
739	"do it"
740	res = ZCatalog.searchResults(self.portal_catalog_real,
741	{'portal_type':"Certificate",
742	'id': cert_id})
743	if res:
744	return res[0]
745	return None
746	###)
747
748	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749	def get_csv_filenames(self):
750	"do it"
751	files = [file for file in os.listdir("%s/import/" % (i_home))
752	if file.endswith('.csv') and (file.find('imported') == -1 and
753	file.find('pending') == -1)]
754	return files
755	###)
756
757	security.declarePublic('findStudentByMatricelNo') ###(
758	def findStudentByMatricelNo(self,matric_no):
759	"do it"
760	res = ZCatalog.searchResults(self.portal_catalog_real,
761	{'portal_type':"StudentClearance",
762	'SearchableText': matric_no})
763	if res:
764	return res[0]
765	return None
766	###)
767
768	security.declarePublic('makeStudentMember') ###(
769	def makeStudentMember(self,sid,password='uNsEt'):
770	"""make the student a member"""
771	membership = self.portal_membership
772	membership.addMember(sid,
773	password ,
774	roles=('Member',
775	'Student',
776	),
777	domains='',
778	properties = {'memberareaCreationFlag': False,
779	'homeless': True},)
780	member = membership.getMemberById(sid)
781	self.portal_registration.afterAdd(member, sid, password, None)
782	#self.manage_setLocalRoles(sid, ['Owner',])
783	###)
784
785	security.declareProtected(View,'makeStudentData') ###(
786	def makeStudentData(self,student_id,email=None,phone_nr=None):
787	"create Datastructure for a returning Student"
788	#import pdb;pdb.set_trace()
789	logger = logging.getLogger('WAeUPTool.makeStudentData')
790	students_folder = self.portal_url.getPortalObject().campus.students
791	#res = self.students_catalog(id=student_id)
792	#if res:
793	# st = res[0]
794	#res = self.returning_import(matric_no = st.matric_no)
795	res = self.returning_import(id = student_id)
796	if res:
797	student = res[0]
798	else:
799	logger.info('Id %s not found in returning_import' % student_id)
800	return
801	logger.info('%s creates data structure' % student_id)
802	s_results = self.results_import(matric_no = student.matric_no)
803	if s_results:
804	lnr = self.getLevelFromResultsCosCode(s_results)
805	level = "%d00" % lnr
806	verdict,eligible = self.getVerdict(s_results[0].Verdict)
807	#if eligible:
808	# level = "%d00" % (lnr + 1)
809	else:
810	logger.info('matric_no %s not found in results_import' % student.matric_no)
811	level = ''
812	verdict = ''
813	#student should not be allowed to perform this transition
814	#wftool = self.portal_workflow
815	#wftool.doActionFor(student,'return')
816	certcode_org = student.Coursemajorcode
817	certcode = makeCertificateCode(certcode_org)
818	certificate_brain = self.getCertificateBrain(certcode)
819	if not certificate_brain:
820	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821	logger.info(em)
822	matric_no = student.matric_no
823	sid = student_id
824	student_obj = getattr(students_folder,sid)
825	if not getattr(student_obj,'application'):
826	student_obj.invokeFactory('StudentApplication','application')
827	application = student_obj.application
828	self.portal_workflow.doActionFor(application,'open',dest_container=application)
829	da = {'Title': 'Application Data'}
830	student_obj.invokeFactory('StudentPersonal','personal')
831	da['jamb_reg_no'] = student.Entryregno
832	em = self.getEntryMode(student.Entryregno)
833	da['entry_mode'] = em
834	personal = student_obj.personal
835	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836	dp = {'Title': 'Personal Data'}
837	student_obj.invokeFactory('StudentClearance','clearance')
838	clearance = student_obj.clearance
839	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840	dc = {'Title': 'Clearance/Eligibility Record'}
841	dc['matric_no'] = matric_no
842	state = student.State
843	lga = student.LGA
844	if state and lga:
845	lga = state + ' / ' + lga
846	else:
847	lga = "None"
848	da['jamb_lga'] = dc['lga'] = lga
849	da['app_email'] = dp['email'] = email
850	da['app_mobile'] = dp['phone'] = phone_nr
851	dp['firstname'] = student.Firstname
852	dp['middlename'] = student.Middlename
853	dp['lastname'] = student.Lastname
854	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855	da['jamb_sex'] = student.Sex
856	dp['sex'] = student.Sex == 'F'
857	dp['perm_address'] = student.Permanent_Address
858	application.getContent().edit(mapping=da)
859	self.portal_workflow.doActionFor(application,'close',dest_container=application)
860	personal.getContent().edit(mapping=dp)
861	clearance.getContent().edit(mapping=dc)
862	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863	#
864	# Study Course
865	#
866	student_obj.invokeFactory('StudentStudyCourse','study_course')
867	studycourse = student_obj.study_course
868	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869	dsc = {}
870	dsc['study_course'] = certcode
871	dsc['current_level'] = level
872	dsc['current_verdict'] = verdict
873	dsc['current_mode'] = em
874	dsc['current_session'] = '05'
875	studycourse.getContent().edit(mapping=dsc)
876	#
877	# Level
878	#
879	## l = getattr(studycourse,level,None)
880	## if l is None:
881	## studycourse.invokeFactory('StudentStudyLevel', level)
882	## l = getattr(studycourse, level)
883	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
884	## l.getContent().edit(mapping={'Title': "Level %s" % level})
885	###)
886
887	def init_timing(self): ###(
888	if self.with_timing:
889	if not hasattr(self,'_v_step_times'):
890	self._v_timer_count = 0
891	self._v_total = 0
892	self._v_step_times = {}
893	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
894	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
895	self.timer_step = 0
896	self.total_time = 0
897	self.elapse = time.time()
898	self.i_elapse = time.time()
899	self.c_elapse = time.clock()
900
901	###)
902
903	def do_timing(self): ###(
904	if self.with_timing:
905	try:
906	raise 'dummy'
907	except:
908	frame = sys.exc_traceback.tb_frame.f_back
909	locals = frame.f_locals
910	globals = frame.f_globals
911	functionname = frame.f_code.co_name
912	filename = os.path.basename(frame.f_code.co_filename)
913	lineno = frame.f_lineno
914	mod_line = "%(functionname)s:%(lineno)s" % vars()
915	i_time = time.time() - self.i_elapse
916	td = {}
917	if self._v_step_times.has_key(mod_line):
918	a_time = self._v_step_times[mod_line]['a_time'] + i_time
919	td['a_time'] = a_time
920	else:
921	td['a_time'] = i_time
922	td['i_time'] = i_time
923	self._v_step_times[mod_line] = td
924	self.i_time = i_time
925	self.total_time += i_time
926	self.timer_step +=1
927	self.i_elapse = time.time()
928
929	###)
930
931	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
932	def print_timing(self):
933	if self.with_timing:
934	l = []
935	timer_count = self._v_timer_count + 1
936	mod_lines = self._v_step_times.keys()
937	mod_lines.sort(cmp,reverse=0)
938	for mod_line in mod_lines:
939	td = self._v_step_times[mod_line]
940	i_time = td['i_time']
941	a_time = td['a_time']/(self._v_timer_count + 1)
942	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
943	total_time = self.total_time
944	total_avarage = self._v_total / timer_count
945	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
946	print "\r\n".join(l)
947	out = open(self._v_timer_file,'a')
948	out.write("\r\n".join(l))
949	out.close()
950
951	###)
952
953	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
954	def get_timing_data(self):
955	if self.with_timing:
956	timer_count = self._v_timer_count + 1
957	results = {}
958	for k,d in self._v_step_times.items():
959	dd = {}
960	dd['a_time'] = d['a_time']/timer_count
961	dd['i_time'] = d['i_time']
962	dd['count'] = timer_count
963	results[k] = dd
964	dd = {}
965	dd['a_time'] = self._v_total / timer_count
966	dd['i_time'] = self.total_time
967	dd['count'] = timer_count
968	results["total"] = dd
969	return results
970	###)
971
972	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
973	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
974	"create Datastructure for an admitted Student"
975	#import pdb;pdb.set_trace()
976	logger = logging.getLogger('WAeUPTool.admitOneStudent')
977	self.with_timing = with_timing
978	if not hasattr(self,"_v_certificates"):
979	self._v_certificates = self.getCertificatesDict()
980	students_folder = self.portal_url.getPortalObject().campus.students
981	if brain.screening_type in ('cest','sandwich',):
982	reg_no = "%s%s/07" % (brain.course1,brain.serial)
983	else:
984	reg_no = brain.reg_no
985	res = self.students_catalog(jamb_reg_no = reg_no)
986	if res:
987	logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
988	return
989	if brain.status != "admitted":
990	logger.info('status of %s is %s' % (reg_no,brain.status))
991	return
992	pin_parts = brain.pin.split('-')
993	if pin_parts and len(pin_parts) != 3:
994	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
995	return
996	if brain.course_admitted not in self._v_certificates:
997	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
998	return
999	if brain.sex not in (True,False):
1000	logger.info('sex of %s not available' % (reg_no))
1001	return
1002	self.init_timing()
1003	student_id = self.generateStudentId('?')
1004	students_folder.invokeFactory('Student', student_id)
1005	student_object = getattr(students_folder,student_id)
1006	self.do_timing()
1007	if pin_password:
1008	password = pin_parts[2]
1009	self.makeStudentMember(student_id,password = password)
1010	student_object.manage_setLocalRoles(student_id, ['Owner',])
1011	self.do_timing()
1012	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1013	#
1014	# application
1015	#
1016	student_object.invokeFactory('StudentApplication','application')
1017	application = student_object.application
1018	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1019	#self.do_timing()
1020	da = {'Title': 'Application Data'}
1021	da['jamb_reg_no'] = reg_no
1022
1023	sex = 'M'
1024	if brain.sex:
1025	sex = 'F'
1026	da['jamb_sex'] = sex
1027	da['jamb_age'] = brain.jamb_age
1028	da['app_reg_pin'] = brain.pin
1029	da['jamb_lga'] = brain.jamb_lga
1030	da['jamb_state'] = brain.jamb_state
1031	da['jamb_score'] = brain.aggregate
1032	da['app_email'] = brain.email
1033	da['app_mobile'] = brain.phone
1034
1035	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1036
1037	da['entry_session'] = entry_session
1038	da['jamb_lastname'] = brain.lastname
1039	da['jamb_middlename'] = brain.middlenames # different field names!
1040	da['jamb_firstname'] = brain.firstname
1041	da['screening_application_date'] = brain.application_date
1042	da['date_of_birth'] = brain.date_of_birth
1043	da['jamb_first_cos'] = brain.course1
1044	da['jamb_second_cos'] = brain.course2
1045	da['course3'] = brain.course3
1046	da['screening_type'] = brain.screening_type
1047	da['screening_score'] = brain.screening_score
1048	da['screening_date'] = brain.screening_date
1049	da['hq_type'] = brain.hq_type
1050	da['hq_grade'] = brain.hq_grade
1051	da['aos'] = brain.aos
1052
1053	application.getContent().edit(mapping=da)
1054	self.do_timing()
1055	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1056	#
1057	# personal
1058	#
1059	student_object.invokeFactory('StudentPersonal','personal')
1060	personal = student_object.personal
1061	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1062	#self.do_timing()
1063	dp = {'Title': 'Personal Data'}
1064	dp['sex'] = brain.sex
1065	dp['email'] = brain.email
1066	dp['phone'] = brain.phone
1067	dp['lastname'] = brain.lastname
1068	dp['middlename'] = brain.middlenames # different field names!
1069	dp['firstname'] = brain.firstname
1070	personal.getContent().edit(mapping=dp)
1071	self.do_timing()
1072	#
1073	# clearance
1074	#
1075	student_object.invokeFactory('StudentClearance','clearance')
1076	clearance = student_object.clearance
1077	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1078	dc = {'Title': 'Clearance/Eligibility Record'}
1079	dc['lga'] = brain.lga
1080	dc['birthday'] = brain.date_of_birth
1081	clearance.getContent().edit(mapping=dc)
1082	self.do_timing()
1083	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1084	#
1085	# study Course
1086	#
1087	student_object.invokeFactory('StudentStudyCourse','study_course')
1088	studycourse = student_object.study_course
1089	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1090	#self.do_timing()
1091	dsc = {}
1092	dsc['study_course'] = brain.course_admitted
1093	dsc['current_verdict'] = ''
1094	dsc['current_mode'] = da['entry_mode']
1095	if da['entry_mode'].startswith('de'):
1096	dsc['current_level'] = '200'
1097	elif da['entry_mode'].startswith('pre'):
1098	dsc['current_level'] = '000'
1099	else:
1100	dsc['current_level'] = '100'
1101	dsc['current_session'] = entry_session
1102	studycourse.getContent().edit(mapping=dsc)
1103	self.do_timing()
1104	#
1105	# payments folder
1106	student_object.invokeFactory('PaymentsFolder','payments')
1107	payments = getattr(student_object,'payments')
1108	#self.do_timing()
1109	dpay = {}
1110	dpay['Title'] = 'Payments'
1111	payments.getContent().edit(mapping=dpay)
1112	self.portal_workflow.doActionFor(payments,'open')
1113	self.do_timing()
1114	#
1115	# passport foto
1116	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1117	brain.screening_type,
1118	brain.reg_no)
1119	images_dir = getImagesDir(student_id)
1120	#images_dir = os.path.join("%s" % images_base,student_id)
1121	letter_dir,student_dir = os.path.split(images_dir)
1122	if not os.path.exists(letter_dir):
1123	os.mkdir(letter_dir)
1124	if not os.path.exists(images_dir):
1125	os.mkdir(images_dir)
1126	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1127	if os.path.exists(app_picture):
1128	copy2(app_picture,image_name)
1129	else:
1130	logger.info('passport of %s/%s not found: %s' % (student_id,
1131	brain.reg_no,
1132	app_picture))
1133
1134	self.do_timing()
1135	self.print_timing()
1136	if with_timing:
1137	self.timer_step = 0
1138	self._v_timer_count += 1
1139	self._v_total += self.total_time
1140	return student_id
1141	###)
1142
1143	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1144	def makeStudentLevel(self,student_id):
1145	"create the StudyLevel for a returning Student"
1146	#import pdb;pdb.set_trace()
1147	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1148	students_folder = self.portal_url.getPortalObject().campus.students
1149	res = self.students_catalog(id=student_id)
1150	if res:
1151	st = res[0]
1152	course = st.course
1153	matric_no = st.matric_no
1154	level = st.level
1155	res = self.results_import(matric_no = matric_no)
1156	if res:
1157	results = res
1158	logger.info('%s creating Level %s' % (student_id,level))
1159	#
1160	# Level
1161	#
1162	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1163	studycourse = getattr(student_obj,"study_course",None)
1164	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1165	l = getattr(studycourse,level,None)
1166	if l is None:
1167	studycourse.invokeFactory('StudentStudyLevel', level)
1168	l = getattr(studycourse, level)
1169	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1170	l.getContent().edit(mapping={'Title': "Level %s" % level})
1171	###)
1172
1173	security.declarePublic('getHallInfo') ###(
1174	def getHallInfo(self,bed):
1175	"""return Hall Info"""
1176	info = {}
1177	hall,block,room,letter = bed.split('_')
1178	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1179	if res and len(res) == 1:
1180	hall_brain = res[0]
1181	hall_doc = hall_brain.getObject().getContent()
1182	else:
1183	return info
1184	info['hall_title'] = hall_brain.Title
1185	info['maintenance_code'] = hall_doc.maintenance_code
1186	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1187	batch_doc = None
1188	for brain in res:
1189	if brain.id.startswith(info['maintenance_code']):
1190	batch_doc = brain.getObject().getContent()
1191	break
1192	if batch_doc is None:
1193	info['maintenance_fee'] = ''
1194	else:
1195	info['maintenance_fee'] = batch_doc.cost
1196	return info
1197	###)
1198
1199	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1200	def removePictureFolder(self,student_id):
1201	"""remove picture_folder by renaming it"""
1202	path = 'images'
1203	picture_path = os.path.join(i_home,path,student_id)
1204	if not os.path.exists(picture_path):
1205	return False
1206	os.rename(picture_path,picture_path + "_removed")
1207	return True
1208	###)
1209
1210	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1211	def restorePictureFolder(self,student_id):
1212	"""restore picture_folder by renaming it"""
1213	path = 'images'
1214	picture_path = os.path.join(i_home,path,student_id)
1215	if not os.path.exists(picture_path + "_removed"):
1216	return False
1217	os.rename(picture_path + "_removed",picture_path)
1218	return True
1219	###)
1220
1221	security.declarePublic('picturesExist') ###(
1222	def picturesExist(self, ids,student_id=None):
1223	"""check if pictures exist in the filesystem"""
1224	if student_id is None:
1225	student_id = self.getStudentId()
1226	if student_id is None:
1227	return False
1228	picture_path = getImagesDir(student_id)
1229	#picture_path = os.path.join(images_base,student_id)
1230	if not os.path.exists(picture_path):
1231	return False
1232	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1233	return set(ids).issubset(set(pictures))
1234	###)
1235
1236	security.declarePublic('picturesList') ###(
1237	def picturesList(self):
1238	"""check if pictures exist in the filesystem"""
1239	path = 'images'
1240	student_id = self.getStudentId()
1241	#picture_path = os.path.join(i_home,path,student_id)
1242	picture_path = getImagesDir(student_id)
1243	if not os.path.exists(picture_path):
1244	return []
1245	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1246	###)
1247
1248	security.declarePublic('showFsPicture') ###(
1249	def showFsPicture(self,path):
1250	"""return a picture from the filesystem"""
1251	#picture_path = os.path.join(i_home,path)
1252	picture_path = os.path.join(images_base,path)
1253	response = self.REQUEST.RESPONSE
1254	#import pdb;pdb.set_trace()
1255	registry = getToolByName(self, 'mimetypes_registry')
1256	mimetype = str(registry.lookupExtension(path.lower()) or
1257	registry.lookupExtension('file.bin'))
1258	if os.path.exists(picture_path):
1259	response.setHeader('Content-type',mimetype)
1260	return open(picture_path).read()
1261	picture_path = os.path.join(i_home,'import',path)
1262	if os.path.exists(picture_path):
1263	return open(picture_path).read()
1264	###)
1265
1266	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1267	def deleteAllCourses(self,department="All"):
1268	''' delete the courses'''
1269	pm = self.portal_membership
1270	member = pm.getAuthenticatedMember()
1271
1272	if str(member) not in ("henrik","joachim"):
1273	return "not possible"
1274	if department == "All":
1275	res = self.portal_catalog({'meta_type': 'Department'})
1276	if len(res) < 1:
1277	return "No Departments found"
1278
1279	deleted = []
1280	for dep in res:
1281	cf = dep.getObject().courses
1282	if cf:
1283	cf.manage_delObjects(ids=cf.objectIds())
1284	deleted.append("deleted Courses in %s" % dep.getId)
1285	return "\r".join(deleted)
1286	###)
1287
1288	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1289	def getLogfileLines(self,filename="event.log",numlines=20):
1290	"""Get last NUMLINES lines of logfile FILENAME.
1291
1292	Return last lines' of a file in the instances logfile directory as
1293	a list. The number of returned lines equals `numlines' or less. If
1294	less than `numlines' lines are available, the whole file ist
1295	returned. If the file can not be opened or some other error
1296	occurs, empty list is returend.
1297	"""
1298	result = []
1299	lines_hit = 0
1300
1301	# We only handle files in instances' log directory...
1302	logpath = os.path.join(i_home, "log")
1303	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1304	if not filename.startswith( logpath ):
1305	# Attempt to access file outside log-dir...
1306	return []
1307
1308	try:
1309	fd = file( filename, "rb" )
1310	except IOError:
1311	return []
1312	if not fd:
1313	return []
1314
1315	if os.linesep == None:
1316	linesep = '\n'
1317	else:
1318	linesep = os.linesep
1319
1320	# Try to find 'numlines' times a lineseparator, searching from end
1321	# and moving to the beginning of file...
1322	fd.seek( 0, 2) # Move to end of file...
1323	while lines_hit < numlines:
1324	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1325	# one step forward...
1326	lines_hit += 1
1327	try:
1328	fd.seek( -2, 1) # Go two bytes back from current pos...
1329	except IOError:
1330	# We cannot go back two bytes. Maybe the file is too small...
1331	break
1332	fd.seek(2,1)
1333
1334	# Read all lines from current position...
1335	result = fd.readlines()
1336	# Remove line endings...
1337	result = [x.strip() for x in result]
1338	fd.close()
1339	return result
1340	###)
1341
1342	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1343	def getCallbacksFromLog(self,filename):
1344	"""fix Online Payment Transactions from Z2.log entries"""
1345	import transaction
1346	import random
1347	from cgi import parse_qs
1348	from urlparse import urlparse
1349	#from pdb import set_trace
1350	wftool = self.portal_workflow
1351	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1352	students_folder = self.portal_url.getPortalObject().campus.students
1353	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1354	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1355	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1356	data = re.compile(s)
1357	start = True
1358	tr_count = 1
1359	total = 0
1360	#name = 'pume_results'
1361	#name = 'epaymentsuccessful_z2log2'
1362	name = filename
1363	no_import = []
1364	imported = []
1365	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1366	try:
1367	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1368	except:
1369	logger.error('Error reading %s' % name)
1370	return
1371	tas = []
1372	for line in transactions:
1373	dict = {}
1374	items = data.search(line)
1375	dict['idict'] = idict = items.groupdict()
1376	#print idict
1377	#from pdb import set_trace;set_trace()
1378	urlparsed = urlparse(idict['get'][4:])
1379	#print urlparsed
1380	path = urlparsed[2].split('/')
1381	dict['student_id'] = student_id = path[8]
1382	dict['payment_id'] = payment_id = path[10]
1383	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1384	tas.append(dict)
1385	tr_count += 1
1386	return tas
1387	###)
1388
1389	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1390	def importOnlinePaymentTransactions(self):
1391	"""load Online Payment Transactions from CSV values"""
1392	import transaction
1393	import random
1394	#from pdb import set_trace
1395	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1396	opt = self.online_payments_import
1397	students_folder = self.portal_url.getPortalObject().campus.students
1398	start = True
1399	tr_count = 1
1400	total = 0
1401	#name = 'pume_results'
1402	name = 'OnlineTransactions'
1403	no_import = []
1404	imported = []
1405	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1406	try:
1407	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1408	except:
1409	logger.error('Error reading %s.csv' % name)
1410	return
1411	for pay_transaction in transactions:
1412	if start:
1413	start = False
1414	logger.info('Start loading from %s.csv' % name)
1415	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1416	no_import.append('%s,"Error"' % s)
1417	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1418	format_error = format + ',"%(Error)s"'
1419	data = {}
1420
1421	# format of the first file sent by Tayo
1422	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1423	#data['student_id'] = student_id = pay_transaction['Payer ID']
1424	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1425	#data['response_code'] = response_code = pay_transaction['Resp Code']
1426	#data['amount'] = amount = pay_transaction['Amount']
1427
1428	# format of the second file sent by Tayo
1429	#data['datetime'] = date = 0
1430	#data['student_id'] = student_id = pay_transaction['Payer ID']
1431	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1432	#data['response_code'] = response_code = '00'
1433	#data['amount'] = amount = pay_transaction['Amount']
1434
1435	# format of the third file sent by Kehinde
1436	data['datetime'] = date = 0
1437	data['student_id'] = student_id = pay_transaction['customer_id']
1438	data['order_id'] = order_id = pay_transaction['merchant_reference']
1439	data['response_code'] = response_code = '00'
1440	data['amount'] = amount = pay_transaction['Amount']
1441
1442	dup = False
1443	if response_code == "12":
1444	continue
1445	try:
1446	opt.addRecord(**data)
1447	except ValueError:
1448	dup = True
1449	#from pdb import set_trace;set_trace()
1450	if dup:
1451	if response_code == "00":
1452	try:
1453	opt.modifyRecord(**data)
1454	except:
1455	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1456	continue
1457	else:
1458	pay_transaction['Error'] = "Duplicate order_id"
1459	no_import.append( format_error % pay_transaction)
1460	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1461	continue
1462	tr_count += 1
1463	if tr_count > 1000:
1464	if len(no_import) > 0:
1465	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1466	'\n'.join(no_import) + '\n')
1467	no_import = []
1468	em = '%d transactions committed\n' % (tr_count)
1469	transaction.commit()
1470	regs = []
1471	logger.info(em)
1472	total += tr_count
1473	tr_count = 0
1474	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1475	'\n'.join(no_import))
1476	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1477	###)
1478
1479	security.declareProtected(ModifyPortalContent,"importData")###(
1480	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1481	"""load data from CSV values"""
1482	import transaction
1483	import random
1484	students_folder = self.portal_url.getPortalObject().campus.students
1485	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1486	pending_only = False
1487	pend_str = '--'
1488	elapse = time.time()
1489	#
1490	# preparations
1491	#
1492	if filename == pend_str:
1493	pending_only = True
1494	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1495	importer = eval("%sImport" % importer_name)(self)
1496	logger = importer.logger
1497	if importer.init_errors:
1498	logger.info(importer.init_errors)
1499	return importer.init_errors
1500	member = importer.member
1501	#current = importer.current
1502	import_date = importer.import_date
1503	#
1504	# not_imported
1505	#
1506	info = importer.info
1507	data_keys = importer.data_keys
1508	csv_keys = importer.csv_keys
1509	#csv_keys.extend(info.keys())
1510	headline_mapping = dict((k,k) for k in csv_keys)
1511	#
1512	# pending
1513	#
1514	pending_path = importer.pending_path
1515	pending_tmp = importer.pending_tmp
1516	pending_backup = importer.pending_backup
1517	pending_fn = importer.pending_fn
1518	imported_path = importer.imported_path
1519	imported_fn = importer.imported_fn
1520	commit_after = importer.commit_after
1521	pending = []
1522	pending_digests = []
1523	#total_added_to_pending = 0
1524	if not pending_only:
1525	pending,pending_digests = importer.makeIdLists()
1526	pending_at_start = len(pending)
1527	datafile = open(pending_tmp,"w")
1528	pending_csv_writer = csv.DictWriter(datafile,
1529	csv_keys,
1530	extrasaction='ignore')
1531	pending_csv_writer.writerow(headline_mapping)
1532	datafile.close()
1533	#
1534	# imported
1535	#
1536	if not os.path.exists(imported_path):
1537	datafile = open(imported_path,"w")
1538	imported_csv_writer = csv.DictWriter(datafile,
1539	csv_keys,
1540	extrasaction='ignore')
1541	imported_csv_writer.writerow(headline_mapping)
1542	datafile.close()
1543	start = True
1544	tr_count = 0
1545	total = 0
1546	total_added_to_imported = 0
1547	total_pending = 0
1548	import_source_done = ""
1549	if pending_only:
1550	import_source_path = pending_path
1551	else:
1552	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1553	import_source_done = "%s/import/%s.done" % (i_home,filename)
1554	if not os.path.exists(import_source_path):
1555	fn = os.path.split(import_source_path)[1]
1556	em = 'no import file %(fn)s' % vars()
1557	return em
1558	import_source_fn = os.path.split(import_source_path)[1]
1559	if not pending_only:
1560	info['imported_from'] = import_source_fn
1561	headline = csv.reader(open(import_source_path,"rb")).next()
1562	if "import_mode" not in headline:
1563	msg = 'import_mode must be in heading'
1564	return msg
1565	invalid_keys = importer.checkHeadline(headline)
1566	if invalid_keys:
1567	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1568
1569	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1570	or k.strip() in info.keys())]
1571	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1572	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1573	# if diff2schema and diff2schema != set(['id',]):
1574	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1575	# return msg
1576	#
1577	# start importing
1578	#
1579	try:
1580	reader = csv.DictReader(open(import_source_path,"rb"))
1581	except:
1582	msg = 'Error reading %s.csv' % filename
1583	logger.error(msg)
1584	return msg
1585	items = [item for item in reader]
1586	total_to_import = len(items)
1587	tti_float = float(total_to_import)
1588	if pending_only:
1589	pending_at_start = total_to_import
1590	count = 0
1591	imported = []
1592	old_commit_count = -1
1593	error_count = imported_count = 0
1594	already_in = 0
1595	for record in items:
1596	item = {}
1597	empty_value_keys = []
1598	for k,v in record.items():
1599	if k is None:
1600	continue
1601	if v:
1602	if v == EMPTY:
1603	empty_value_keys += k,
1604	v = ''
1605	item[k.strip()] = v.strip()
1606	count += 1
1607	if start:
1608	start = False
1609	adapters = [MappingStorageAdapter(importer.schema, item)]
1610	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1611	dm = DataModel(item, adapters,context=self)
1612	ds = DataStructure(data=item,datamodel=dm)
1613	error_string = ""
1614	total += 1
1615	import_mode = item.get('import_mode','')
1616	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1617	if import_method is None:
1618	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1619	elif (import_mode in importer.required_modes and
1620	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1621	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1622	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1623	else:
1624	for k in import_keys:
1625	if k not in item.keys() or k not in importer.validators.keys():
1626	continue
1627	if not importer.validators[k](ds,mode=import_mode):
1628	error_string += ' ++ '
1629	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1630	ds.getErrorMapping(k)))
1631	if error_string:
1632	error = error_string
1633	id = ''
1634	mapping = item
1635	else:
1636	temp_item = item.copy()
1637	temp_item.update(dm)
1638	results = import_method(temp_item)
1639	id = results[0]
1640	error = results[1]
1641	mapping = results[2]
1642	#
1643	# restore some values
1644	#
1645	for k in empty_value_keys:
1646	mapping[k] = EMPTY
1647	if mapping.has_key('sex'):
1648	#import pdb;pdb.set_trace()
1649	if mapping['sex'] == True:
1650	mapping['sex'] = 'F'
1651	elif mapping['sex'] == False:
1652	mapping['sex'] = 'M'
1653	if pending_only:
1654	info['imported_from'] = item['imported_from']
1655	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1656	info['error'] = error
1657	info['import_record_no'] = count + 1
1658	mapping.update(info)
1659	log_list = []
1660	if error:
1661	error_count += 1
1662	digest = makeDigest(mapping,data_keys)
1663	if digest not in pending_digests:
1664	pending_digests += digest,
1665	pending.append(mapping)
1666	if not pending_only:
1667	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1668	else:
1669	already_in += 1
1670	pass
1671	else:
1672	imported_count += 1
1673	imported += mapping,
1674	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1675	if log_list:
1676	time_till_now = time.time() - elapse
1677	percent_finished = (error_count + imported_count)/tti_float*100
1678	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1679	logger.info(' '.join(log_list))
1680	finished = count > total_to_import - 1
1681	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1682	if must_commit:
1683	old_commit_count = imported_count
1684
1685	if must_commit or finished:
1686	if len(imported):
1687	transaction.commit()
1688	datafile = open(imported_path,"a")
1689	writer = csv.DictWriter(datafile,
1690	csv_keys,
1691	extrasaction='ignore')
1692	writer.writerows(imported)
1693	datafile.close()
1694	total_added_to_imported += len(imported)
1695	imported = []
1696	if len(pending) > 0:
1697	datafile = open(pending_tmp,"a")
1698	writer = csv.DictWriter(datafile,
1699	csv_keys,
1700	extrasaction='ignore')
1701	writer.writerows(pending)
1702	datafile.close()
1703	total_pending += len(pending)
1704	#total_added_to_pending += len(pending)
1705	pending = []
1706	if not finished:
1707	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1708	logger.info(msg)
1709	elapse = time.time() - elapse
1710	if os.path.exists(pending_path):
1711	copy2(pending_path,pending_backup)
1712	copy2(pending_tmp,pending_path)
1713	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1714	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1715	if pending_only:
1716	removed_pending = pending_at_start - total_pending
1717	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1718	else:
1719	added_pending = total_pending - pending_at_start
1720	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1721	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1722	logger.info(msg)
1723	if import_source_done:
1724	copy(import_source_path,import_source_done)
1725	os.remove(import_source_path)
1726	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1727	if upload is not None:
1728	upload_doc = upload.getContent()
1729	mapping = {}
1730	#mapping['import_date'] = DateTime.DateTime()
1731	mapping['import_date'] = import_date
1732	mapping['imported_by'] = importer.imported_by
1733	mapping['import_message'] = msg
1734	upload_doc.edit(mapping = mapping)
1735	os.remove(pending_tmp)
1736	return msg
1737	###)
1738
1739	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1740	def moveImagesToFS(self,student_id="O738726"):
1741	"move the images to the filesystem"
1742	images_dir = getImagesDir(student_id)
1743	#images_dir = os.path.join("%s" % images_base,student_id)
1744	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1745	stool = getToolByName(self, 'portal_schemas')
1746	schemas = ['student_application',
1747	'student_clearance',
1748	]
1749	created = False
1750	for schema_id in schemas:
1751	schema = stool._getOb(schema_id)
1752	object = getattr(student_folder,schema_id[len('student_'):],None)
1753	if object is None:
1754	continue
1755	doc = object.getContent()
1756	for key in schema.keys():
1757	if schema[key].meta_type != "CPS Image Field":
1758	continue
1759	#import pdb;pdb.set_trace()
1760	image = getattr(doc,key,None)
1761	if not image or not hasattr(image,"data"):
1762	continue
1763	if not created:
1764	if not os.path.exists(images_dir):
1765	os.mkdir(images_dir)
1766	created = True
1767	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1768	open(filename,"wb").write(str(image.data))
1769	###)
1770
1771	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1772	def movePassportToFS(self,student_id="O738726"):
1773	"move the passports to the filesystem"
1774	images_dir = os.path.join("%s" % i_home,'passports')
1775	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1776	stool = getToolByName(self, 'portal_schemas')
1777	schemas = ['student_application',
1778	#'student_clearance',
1779	]
1780	created = False
1781	for schema_id in schemas:
1782	schema = stool._getOb(schema_id)
1783	object = getattr(student_folder,schema_id[len('student_'):],None)
1784	if object is None:
1785	continue
1786	doc = object.getContent()
1787	for key in schema.keys():
1788	if schema[key].meta_type != "CPS Image Field":
1789	continue
1790	#import pdb;pdb.set_trace()
1791	image = getattr(doc,key)
1792	if not hasattr(image,"data"):
1793	continue
1794	if not created:
1795	if not os.path.exists(images_dir):
1796	os.mkdir(images_dir)
1797	created = True
1798	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1799	open(filename,"wb").write(str(image.data))
1800	###)
1801
1802	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: