Context navigation

WAeUPTool.py @ 3421

Last change on this file since 3421 was 3406, checked in by Henrik Bettermann, 17 years ago

WAeUPTool.py: fix traceback

Accommodation.py: enable import of reserved beds

release_bed.py: remove maintenance fee

reserve_accommodation.py: enable bed change for reserved beds. To be done: check for existing acco object before creating it in line 128

change_bed.py: enable bed change for reserved beds

Property svn:keywords set to Id

File size: 72.7 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3406 2008-04-02 07:16:08Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108
109	###)
110
111	security.declareProtected(View,'updateRoleMappingsFor') ###(
112	def updateRoleMappingsFor(self,wf_definition,ob):
113	"do so for public"
114	wf_def = getattr(self.portal_workflow,wf_definition)
115	wf_def.updateRoleMappingsFor(ob)
116	###)
117
118	security.declareProtected(View,'getStatesLgas') ###(
119	def getStatesLgas(self):
120	"""return lga info"""
121	voc = getattr(self.portal_vocabularies,'local_gov_areas')
122	states = []
123	lgas = []
124	d = {}
125	wd = {}
126	for k,v in voc.items():
127	parts = v.split(' / ')
128	if len(parts) == 1:
129	state = parts[0].lower()
130	lga = ""
131	elif len(parts) == 2:
132	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134	else:
135	continue
136	if state not in states:
137	states.append(state)
138	if lga not in lgas:
139	lgas.append(lga)
140	words = re.split('[^a-zA-Z0-9/]',k)
141	words.sort()
142	wd[k] = words
143	d[k] = v
144	mapping = {}
145	mapping['word_dict'] = wd
146	mapping['lga_dict'] = d
147	mapping['states'] = states
148	mapping['lgas'] = lgas
149	return mapping
150	###)
151
152	security.declareProtected(View,'findLga') ###(
153	def findLga(self,words,words_dict):
154	words = re.split('[^a-zA-Z0-9/]',words)
155	lga_words = []
156	for word in words:
157	if word:
158	lga_words += word.strip().lower(),
159	lga_words.sort()
160	state_lga = ''
161	while not state_lga:
162	for k,l in words_dict.items():
163	if lga_words == l:
164	state_lga = k
165	break
166	break
167	return state_lga
168	###)
169	security.declareProtected(View,'getAccessInfo') ###(
170	def getAccessInfo(self,context):
171	"return a dict with access_info"
172	logger = logging.getLogger('WAeUPTool.getAccessInfo')
173	mtool = self.portal_membership
174	member = mtool.getAuthenticatedMember()
175	member_id = str(member)
176	info = {}
177	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179	is_staff = info['is_staff'] = not is_anonymous and not is_student
180	roles = member.getRolesInContext(context)
181	is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182	"SectionManager" in roles or
183	"Manager" in roles)
184	is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if is_allowed:
188	if not is_student and requested_id:
189	student_id = requested_id
190	elif is_anonymous or member_id != requested_id:
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	else:
193	student_id = member_id
194	info['student_id'] = student_id
195	return info
196	###)
197
198	security.declareProtected(ModifyPortalContent,'openLog') ###(
199	def openLog(self,name):
200	"""open a log file"""
201	version = 1
202	path = "%s/log/%s_%d.log" % (i_home,name,version)
203	while os.path.exists(path):
204	version += 1
205	path = "%s/log/%s_%d.log" % (i_home,name,version)
206	log = open(path,"w")
207	return log
208	###)
209
210	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211	def bypassQueueCatalog(self,enable=True):
212	"""bypass the QueueCatalog by setting all indexes to process imediate,
213	if enable is True (default) the old settings are restored
214	"""
215
216	###)
217
218	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220	"""measure Object access Time"""
221	import random
222	if hasattr(self,'portal_catalog_real'):
223	aq_portal = self.portal_catalog_real.evalAdvancedQuery
224	else:
225	aq_portal = self.portal_catalog.evalAdvancedQuery
226	nr_pts = int(nr_pts)
227	probe = int(probe)
228	intervall = probe/10
229	objects = ("application","clearance","personal")
230	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231	#i = random.randrange(num_objects)
232	count = 0
233	found = 0
234	not_found = 0
235	t_found = 0
236	t_not_found = 0
237	time_found = time_not_found = 0.0
238	t_time_found = t_time_not_found = 0.0
239	accessed = []
240	t_min = 1000
241	t_max = 0
242	#import pdb;pdb.set_trace()
243	students = self.portal_catalog(portal_type="Student")
244	num_students = len(students)
245	if method == "d":
246	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247	res = aq_portal(query)
248	brains = {}
249	for r in res:
250	sid = r.relative_path.split('/')[-2]
251	if brains.has_key(sid):
252	brains[sid][r.portal_type] = r
253	else:
254	brains[sid] = {r.portal_type : r}
255	brains_list = brains.keys()
256	num_objects = len(brains_list)
257	else:
258	num_objects = num_students
259	print "="*40
260	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261	probe,
262	nr_pts,
263	num_objects)
264	print "nr found/not time found/not min/max"
265	elapse = time.time()
266	i_elapse = time.time()
267	c_elapse = time.clock()
268	for c in range(1,probe + 1):
269	i = random.randrange(num_objects)
270	if method in ('a','b','c'):
271	student_brain = students[i]
272	elif method == "d":
273	#import pdb;pdb.set_trace()
274	student_brain = brains[brains_list[i]]
275	if method == "c":
276	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277	res = aq_portal(query)
278	this_portal_types = [r.portal_type for r in res]
279	for i in range(nr_pts):
280	oid = objects[i]
281	if method == "a":
282	try:
283	student_path = student_brain.getPath()
284	path = "%s/%s" % (student_path,oid)
285	doc = self.unrestrictedTraverse(path).getContent()
286	found += 1
287	i_time = time.time() - i_elapse
288	time_found += i_time
289	except:
290	not_found += 1
291	i_time = time.time() - i_elapse
292	time_not_found += i_time
293	pass
294	elif method == "b":
295	try:
296	student_object = student_brain.getObject()
297	doc = getattr(student_object,oid).getContent()
298	found += 1
299	i_time = time.time() - i_elapse
300	time_found += i_time
301	except:
302	i_time = time.time() - i_elapse
303	time_not_found += i_time
304	not_found += 1
305	pass
306	elif method == "c":
307	if portal_types[i] in this_portal_types:
308	found += 1
309	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310	i_time = time.time() - i_elapse
311	time_found += i_time
312	else:
313	not_found += 1
314	i_time = time.time() - i_elapse
315	time_not_found += i_time
316	elif method == "d":
317	if student_brain.has_key(portal_types[i]):
318	found += 1
319	doc = student_brain[portal_types[i]].getObject().getContent()
320	i_time = time.time() - i_elapse
321	time_found += i_time
322	else:
323	not_found += 1
324	i_time = time.time() - i_elapse
325	time_not_found += i_time
326	i_elapse = time.time()
327	if c and (c % intervall == 0):
328	#i_time = time.time() - i_elapse
329	t_per = 0.0
330	if found:
331	t_per = time_found/found
332	if t_per > t_max:
333	t_max = t_per
334	if t_per > 0.0 and t_per < t_min:
335	t_min = t_per
336	itf = 0.0
337	if found:
338	itf = time_found/found
339	itnf = 0.0
340	if not_found :
341	itnf = time_not_found / not_found
342	interval_time = time_found + time_not_found
343	s = "%(c)d: %(found)d/%(not_found)d " % vars()
344	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
346	print s
347	t_found += found
348	t_not_found += not_found
349	t_time_found += time_found
350	t_time_not_found += time_not_found
351	time_found = time_not_found = 0.0
352	found = not_found = 0
353	# t_found += found
354	# t_not_found += not_found
355	elapse = time.time() - elapse
356	itf = 0.0
357	if t_found:
358	itf = t_time_found/t_found
359	itnf = 0.0
360	if t_not_found:
361	itnf = t_time_not_found / t_not_found
362	#c_elapse = time.clock() - c_elapse
363	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
366	print "-"*40
367	print s
368	rel_found = float(t_found)/probe
369	rel_not_found = float(t_not_found)/probe
370	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
371	print estimated_total_time
372	###)
373
374	security.declareProtected(ModifyPortalContent,'writeLog') ###(
375	def writeLog(self,logfile,s):
376	"""write to the log file"""
377	logfile.write(s)
378
379	###)
380
381	def generateStudentId(self,letter): ###(
382	import random
383	r = random
384	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385	if letter == '?':
386	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387	sid = "%c%d" % (letter,r.randint(99999,1000000))
388	students = self.portal_url.getPortalObject().campus.students
389	## while hasattr(students, sid):
390	## sid = "%c%d" % (letter,r.randint(99999,1000000))
391	while self.students_catalog(id = sid):
392	sid = "%c%d" % (letter,r.randint(99999,1000000))
393	return sid
394	###)
395
396	def generatePassword(self,s=None): ###(
397	import random
398	r = random
399	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400	if s is None:
401	s = 'abcdefghklmnpqrstuvwxy23456789'
402	pw = ''
403	while len(pw) < 6:
404	pw += r.choice(s)
405	return pw
406	###)
407
408	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409	def dumpSchoolfeePayments(self):
410	"dump paid schoolfees"
411	mtool = self.portal_membership
412	member = mtool.getAuthenticatedMember()
413	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414	aq_student = self.students_catalog.evalAdvancedQuery
415	query = In('review_state',('schoolfee_paid',
416	'courses_registered',
417	'courses_validated',
418	))
419	res = aq_student(query)
420	#import pdb;pdb.set_trace()
421	l = []
422	logger.info("start for %d" % len(res))
423	count = 1
424	log_after = 100
425	for student in res:
426	if not count % log_after:
427	logger.info("processed %d total %d" % (log_after,count))
428	count += 1
429	fee_dict =self.getSchoolFee(student)
430	fulltime = student.mode.endswith('_ft')
431	d = {}
432	d['student_id'] = student.id
433	d['name'] = student.name
434	d['amount'] = fee_dict.get(new_returning)
435	l += d,
436	csv_name = self.dumpListToCSV(l,'payments')
437	logger.info('%s dumped payments to %s' % (member,export_file))
438	###)
439
440	security.declarePublic('dumpListToCSV') ###(
441	def dumpListToCSV(self,l,filename,fields=None):
442	"""dump a list of dicts to a CSV file"""
443	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445	if fields is None:
446	fields = l[0].keys()
447	headline = ','.join(fields)
448	out = open(export_file,"wb")
449	out.write(headline +'\n')
450	out.close()
451	out = open(export_file,"a")
452	csv_writer = csv.DictWriter(out,fields,)
453	csv_writer.writerows(l)
454	return export_file
455	###)
456
457	security.declareProtected(ManagePortal, 'listMembers') ###(
458	def listMembers(self):
459	"list all members"
460	mtool = self.portal_membership
461	member = mtool.getAuthenticatedMember()
462	logger = logging.getLogger('WAeUPTool.listMembers')
463	if str(member) not in ('admin','joachim'):
464	logger.info('%s tried to list members' % (member))
465	return None
466	members = self.portal_directories.members
467	all = members.listEntryIdsAndTitles()
468	l = []
469	for user_id,name in all:
470	d = {}
471	d['user_id'] = user_id
472	d['name'] = name
473	d['pw'] = getattr(getattr(members,user_id),'password')
474	d['email'] = getattr(getattr(members,user_id),'email')
475	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477	l += d,
478	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480	logger.info('%s dumped member list to %s' % (member,export_file))
481	fields = l[0].keys()
482	headline = ','.join(fields)
483	out = open(export_file,"wb")
484	out.write(headline +'\n')
485	out.close()
486	out = open(export_file,"a")
487	csv_writer = csv.DictWriter(out,fields,)
488	csv_writer.writerows(l)
489	###)
490
491	security.declareProtected(ManagePortal, 'listStudents') ###(
492	def listStudents(self):
493	"list all students"
494	mtool = self.portal_membership
495	member = mtool.getAuthenticatedMember()
496	logger = logging.getLogger('WAeUPTool.listStudents')
497	if str(member) not in ('admin','joachim'):
498	logger.info('%s tried to list students' % (member))
499	return None
500	students = self.portal_directories.students
501	all = students.listEntryIdsAndTitles()
502	l = []
503	for user_id,name in all:
504	d = {}
505	d['user_id'] = user_id
506	d['name'] = name
507	d['pw'] = getattr(getattr(students,user_id),'password')
508	d['email'] = getattr(getattr(students,user_id),'email')
509	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511	l += d,
512	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514	logger.info('%s dumped student list to %s' % (member,export_file))
515	fields = l[0].keys()
516	headline = ','.join(fields)
517	out = open(export_file,"wb")
518	out.write(headline +'\n')
519	out.close()
520	out = open(export_file,"a")
521	csv_writer = csv.DictWriter(out,fields,)
522	csv_writer.writerows(l)
523	###)
524
525	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526	def removeDeletedDocIds(self, max=1000):
527	"""
528	remove deleted docids from repository commit after max
529	"""
530	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531	repository = getToolByName(self, 'portal_repository')
532	pxtool = getToolByName(self, 'portal_proxies')
533	logger.info('start')
534	pxtool_infos = pxtool.getRevisionsUsed()
535	logger.info('found %d used revisions ' % (len(pxtool_infos)))
536
537	nb_revs = 0
538	docids_d = {} # all docids
539	unused_docids_d = {} # all docids that are unused
540	ids_unused_revs_docids = [] # ids for revs of unused docids
541	unused_ids = [] # ids for unused revs
542	total = 0
543	idlist = repository.objectIds()
544	to_delete = 0
545	found = False
546	for id in idlist:
547	docid, rev = repository._splitId(id)
548	if docid is None:
549	logger.info("invalid doc_id %s" % docid)
550	continue
551	nb_revs += 1
552	if not pxtool_infos.has_key(docid):
553	found = True
554	to_delete += 1
555	unused_ids.append(id)
556	elif not pxtool_infos[docid].has_key(rev):
557	found = True
558	to_delete += 1
559	unused_ids.append(id)
560	if found and not to_delete % max:
561	found = False
562	#import pdb;pdb.set_trace()
563	repository.manage_delObjects(unused_ids)
564	transaction.commit()
565	logger.info('removed %d total %d unused docids ' % (max,to_delete))
566	else:
567	if unused_ids:
568	repository.manage_delObjects(unused_ids)
569	transaction.commit()
570	logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573	###)
574
575	security.declareProtected(View,'getCredential') ###(
576	def getCredential(self,student_id):
577	student_entry = getattr(self.portal_directories.students,student_id,None)
578	if not self.isStaff():
579	mtool = self.portal_membership
580	member = mtool.getAuthenticatedMember()
581	logger = logging.getLogger('WAeUPTool.getCredential')
582	logger.info('%s tried to access password of %s' % (member,student_id))
583	return None
584	if student_entry is None:
585	return None
586	return getattr(student_entry,"password","not set")
587	###)
588
589	security.declarePublic('checkPassword') ###(
590	def checkPassword(self,student_id,password):
591	student_entry = getattr(self.portal_directories.students,student_id,None)
592	if student_entry is None:
593	return False
594	return getattr(student_entry,"password","not set") == password
595	###)
596
597	security.declarePublic('checkGenericPassword') ###(
598	def checkGenericPassword(self,member_id):
599	member_entry = getattr(self.portal_directories.members,member_id,None)
600	if member_entry is None:
601	return False
602	ltool = getToolByName(self, 'portal_layouts')
603	unsecure_words = ltool._getOb('members')['w__password'].check_words
604	password = getattr(member_entry,"password","not set")
605	is_unsecure = password in unsecure_words
606	if is_unsecure:
607	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609	return is_unsecure
610	###)
611
612	security.declareProtected(ModifyPortalContent,'editPassword') ###(
613	def editPassword(self,student_id,password):
614	"edit a student password"
615	student_entry = getattr(self.portal_directories.students,student_id,None)
616	if student_entry is None:
617	return
618	setattr(student_entry,'password',password)
619	###)
620
621	security.declareProtected(ModifyPortalContent,'doCommit') ###(
622	def doCommit(self,logger=None):
623	"commit some transactions"
624	transaction.commit()
625	###)
626
627	security.declarePublic('loadStudentFoto') ###(
628	def loadStudentFoto(self,student,filename,folder):
629	"return a student passport picture"
630	#import pdb;pdb.set_trace()
631	picture ="%s/import/%s/%s" % (i_home,folder,filename)
632	student_id = student.getId()
633	images_dir = getImagesDir(student_id)
634	if not os.path.exists(images_dir):
635	os.mkdir(images_dir)
636	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637	for extension in ('.jpg','.JPG'):
638	fullname = "%(picture)s%(extension)s" % vars()
639	if os.path.exists(fullname):
640	copy2(fullname,image_name)
641	return "successfully copied passport picture"
642	return "passport picture not found: %s.jpg or .JPG" % picture
643	###)
644
645	def old____loadStudentFoto(self,student,filename,folder): ###(
646	"return a student passport picture"
647	app = student.application
648	app_doc = app.getContent()
649	#clear = student.clearance
650	#clear_doc = clear.getContent()
651	#matric_no = clear_doc.matric_no.upper()
652	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654	#import pdb;pdb.set_trace()
655	if os.path.exists(picture1):
656	file = open(picture1)
657	elif os.path.exists(picture2):
658	file = open(picture2)
659	else:
660	return "passport picture not found %s" % picture1
661	reopened = False
662	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663	self.portal_workflow.doActionFor(app,'open')
664	reopened = True
665	outfile = file.read()
666	app_doc.manage_addFile('passport',
667	file=outfile,
668	title="%s.jpg" % filename)
669	if reopened:
670	self.portal_workflow.doActionFor(app,'close')
671	return "successfully loaded passport picture"
672	###)
673
674	security.declareProtected(ModifyPortalContent,'createOne') ###(
675	def createOne(self,students_folder,student_brain,letter,commit=False):
676	sid = self.waeup_tool.generateStudentId(letter)
677	students_folder.invokeFactory('Student', sid)
678	student = getattr(students_folder,sid)
679	self.portal_workflow.doActionFor(student,'return')
680	student.manage_setLocalRoles(sid, ['Owner',])
681	matric_no = student_brain.matric_no
682	jamb_reg_no = student_brain.Entryregno
683	self.students_catalog.addRecord(id = sid,
684	matric_no = matric_no,
685	jamb_reg_no = jamb_reg_no,
686	sex = student_brain.Sex == "F",
687	name = "%s %s %s" % (student_brain.Firstname,
688	student_brain.Middlename,
689	student_brain.Lastname)
690	)
691	if commit:
692	transaction.commit()
693	return sid,jamb_reg_no
694	###)
695
696	security.declareProtected(ModifyPortalContent,'addStudent') ###(
697	def addStudent(self,dict):
698	students_folder = self.portal_url.getPortalObject().campus.students
699	sid = self.waeup_tool.generateStudentId('?')
700	students_folder.invokeFactory('Student', sid)
701	student_obj = getattr(students_folder,sid)
702	f2t = self.field2types_student
703	#from pdb import set_trace; set_trace()
704	d = {}
705	#d['jamb_sex'] = 'M'
706	#if dict.get('sex'):
707	# d['jamb_sex'] = 'F'
708
709	entry_session = dict.get('entry_session')
710	if entry_session == self.getSessionId()[0]:
711	wfaction = 'admit'
712	wft = 'wf_transition_admit'
713	password = None
714	else:
715	wfaction = 'return'
716	wft = 'wf_transition_return'
717	password = self.generatePassword()
718	self.makeStudentMember(sid,password)
719
720	for pt in f2t.keys():
721	student_obj.invokeFactory(pt,f2t[pt]['id'])
722	sub_obj = getattr(student_obj,f2t[pt]['id'])
723	sub_doc = sub_obj.getContent()
724	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725	d['Title'] = f2t[pt]['title']
726	for field in f2t[pt]['fields']:
727	d[field] = dict.get(field,'')
728	sub_doc.edit(mapping = d)
729	new_state = f2t[pt][wft]
730	if new_state != "remain":
731	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732	self.portal_workflow.doActionFor(student_obj,wfaction)
733	student_obj.manage_setLocalRoles(sid, ['Owner',])
734	return sid,password
735	###)
736
737	security.declarePublic('getCertificateBrain') ###(
738	def getCertificateBrain(self,cert_id):
739	"do it"
740	res = ZCatalog.searchResults(self.portal_catalog_real,
741	{'portal_type':"Certificate",
742	'id': cert_id})
743	if res:
744	return res[0]
745	return None
746	###)
747
748	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749	def get_csv_filenames(self):
750	"do it"
751	files = [file for file in os.listdir("%s/import/" % (i_home))
752	if file.endswith('.csv') and (file.find('imported') == -1 and
753	file.find('pending') == -1)]
754	return files
755	###)
756
757	security.declarePublic('findStudentByMatricelNo') ###(
758	def findStudentByMatricelNo(self,matric_no):
759	"do it"
760	res = ZCatalog.searchResults(self.portal_catalog_real,
761	{'portal_type':"StudentClearance",
762	'SearchableText': matric_no})
763	if res:
764	return res[0]
765	return None
766	###)
767
768	security.declarePublic('makeStudentMember') ###(
769	def makeStudentMember(self,sid,password='uNsEt'):
770	"""make the student a member"""
771	membership = self.portal_membership
772	membership.addMember(sid,
773	password ,
774	roles=('Member',
775	'Student',
776	),
777	domains='',
778	properties = {'memberareaCreationFlag': False,
779	'homeless': True},)
780	member = membership.getMemberById(sid)
781	self.portal_registration.afterAdd(member, sid, password, None)
782	#self.manage_setLocalRoles(sid, ['Owner',])
783	###)
784
785	security.declareProtected(View,'makeStudentData') ###(
786	def makeStudentData(self,student_id,email=None,phone_nr=None):
787	"create Datastructure for a returning Student"
788	#import pdb;pdb.set_trace()
789	logger = logging.getLogger('WAeUPTool.makeStudentData')
790	students_folder = self.portal_url.getPortalObject().campus.students
791	#res = self.students_catalog(id=student_id)
792	#if res:
793	# st = res[0]
794	#res = self.returning_import(matric_no = st.matric_no)
795	res = self.returning_import(id = student_id)
796	if res:
797	student = res[0]
798	else:
799	logger.info('Id %s not found in returning_import' % student_id)
800	return
801	logger.info('%s creates data structure' % student_id)
802	s_results = self.results_import(matric_no = student.matric_no)
803	if s_results:
804	lnr = self.getLevelFromResultsCosCode(s_results)
805	level = "%d00" % lnr
806	verdict,eligible = self.getVerdict(s_results[0].Verdict)
807	#if eligible:
808	# level = "%d00" % (lnr + 1)
809	else:
810	logger.info('matric_no %s not found in results_import' % student.matric_no)
811	level = ''
812	verdict = ''
813	#student should not be allowed to perform this transition
814	#wftool = self.portal_workflow
815	#wftool.doActionFor(student,'return')
816	certcode_org = student.Coursemajorcode
817	certcode = makeCertificateCode(certcode_org)
818	certificate_brain = self.getCertificateBrain(certcode)
819	if not certificate_brain:
820	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821	logger.info(em)
822	matric_no = student.matric_no
823	sid = student_id
824	student_obj = getattr(students_folder,sid)
825	if not getattr(student_obj,'application'):
826	student_obj.invokeFactory('StudentApplication','application')
827	application = student_obj.application
828	self.portal_workflow.doActionFor(application,'open',dest_container=application)
829	da = {'Title': 'Application Data'}
830	student_obj.invokeFactory('StudentPersonal','personal')
831	da['jamb_reg_no'] = student.Entryregno
832	em = self.getEntryMode(student.Entryregno)
833	da['entry_mode'] = em
834	personal = student_obj.personal
835	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836	dp = {'Title': 'Personal Data'}
837	student_obj.invokeFactory('StudentClearance','clearance')
838	clearance = student_obj.clearance
839	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840	dc = {'Title': 'Clearance/Eligibility Record'}
841	dc['matric_no'] = matric_no
842	state = student.State
843	lga = student.LGA
844	if state and lga:
845	lga = state + ' / ' + lga
846	else:
847	lga = "None"
848	da['jamb_lga'] = dc['lga'] = lga
849	da['app_email'] = dp['email'] = email
850	da['app_mobile'] = dp['phone'] = phone_nr
851	dp['firstname'] = student.Firstname
852	dp['middlename'] = student.Middlename
853	dp['lastname'] = student.Lastname
854	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855	da['jamb_sex'] = student.Sex
856	dp['sex'] = student.Sex == 'F'
857	dp['perm_address'] = student.Permanent_Address
858	application.getContent().edit(mapping=da)
859	self.portal_workflow.doActionFor(application,'close',dest_container=application)
860	personal.getContent().edit(mapping=dp)
861	clearance.getContent().edit(mapping=dc)
862	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863	#
864	# Study Course
865	#
866	student_obj.invokeFactory('StudentStudyCourse','study_course')
867	studycourse = student_obj.study_course
868	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869	dsc = {}
870	dsc['study_course'] = certcode
871	dsc['current_level'] = level
872	dsc['current_verdict'] = verdict
873	dsc['current_mode'] = em
874	dsc['current_session'] = '05'
875	studycourse.getContent().edit(mapping=dsc)
876	#
877	# Level
878	#
879	## l = getattr(studycourse,level,None)
880	## if l is None:
881	## studycourse.invokeFactory('StudentStudyLevel', level)
882	## l = getattr(studycourse, level)
883	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
884	## l.getContent().edit(mapping={'Title': "Level %s" % level})
885	###)
886
887	def init_timing(self): ###(
888	if self.with_timing:
889	if not hasattr(self,'_v_step_times'):
890	self._v_timer_count = 0
891	self._v_total = 0
892	self._v_step_times = {}
893	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
894	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
895	self.timer_step = 0
896	self.total_time = 0
897	self.elapse = time.time()
898	self.i_elapse = time.time()
899	self.c_elapse = time.clock()
900
901	###)
902
903	def do_timing(self): ###(
904	if self.with_timing:
905	try:
906	raise 'dummy'
907	except:
908	frame = sys.exc_traceback.tb_frame.f_back
909	locals = frame.f_locals
910	globals = frame.f_globals
911	functionname = frame.f_code.co_name
912	filename = os.path.basename(frame.f_code.co_filename)
913	lineno = frame.f_lineno
914	mod_line = "%(functionname)s:%(lineno)s" % vars()
915	i_time = time.time() - self.i_elapse
916	td = {}
917	if self._v_step_times.has_key(mod_line):
918	a_time = self._v_step_times[mod_line]['a_time'] + i_time
919	td['a_time'] = a_time
920	else:
921	td['a_time'] = i_time
922	td['i_time'] = i_time
923	self._v_step_times[mod_line] = td
924	self.i_time = i_time
925	self.total_time += i_time
926	self.timer_step +=1
927	self.i_elapse = time.time()
928
929	###)
930
931	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
932	def print_timing(self):
933	if self.with_timing:
934	l = []
935	timer_count = self._v_timer_count + 1
936	mod_lines = self._v_step_times.keys()
937	mod_lines.sort(cmp,reverse=0)
938	for mod_line in mod_lines:
939	td = self._v_step_times[mod_line]
940	i_time = td['i_time']
941	a_time = td['a_time']/(self._v_timer_count + 1)
942	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
943	total_time = self.total_time
944	total_avarage = self._v_total / timer_count
945	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
946	print "\r\n".join(l)
947	out = open(self._v_timer_file,'a')
948	out.write("\r\n".join(l))
949	out.close()
950
951	###)
952
953	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
954	def get_timing_data(self):
955	if self.with_timing:
956	timer_count = self._v_timer_count + 1
957	results = {}
958	for k,d in self._v_step_times.items():
959	dd = {}
960	dd['a_time'] = d['a_time']/timer_count
961	dd['i_time'] = d['i_time']
962	dd['count'] = timer_count
963	results[k] = dd
964	dd = {}
965	dd['a_time'] = self._v_total / timer_count
966	dd['i_time'] = self.total_time
967	dd['count'] = timer_count
968	results["total"] = dd
969	return results
970	###)
971
972	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
973	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
974	"create Datastructure for an admitted Student"
975	#import pdb;pdb.set_trace()
976	logger = logging.getLogger('WAeUPTool.admitOneStudent')
977	self.with_timing = with_timing
978	if not hasattr(self,"_v_certificates"):
979	self._v_certificates = self.getCertificatesDict()
980	students_folder = self.portal_url.getPortalObject().campus.students
981	if brain.screening_type in ('cest','sandwich',):
982	reg_no = "%s%s/07" % (brain.course1[:3],brain.serial)
983	else:
984	reg_no = brain.reg_no
985	res = self.students_catalog(jamb_reg_no = reg_no)
986	if res:
987	logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
988	return
989	if brain.status != "admitted":
990	logger.info('status of %s is %s' % (reg_no,brain.status))
991	return
992	pin_parts = brain.pin.split('-')
993	if pin_parts and len(pin_parts) != 3:
994	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
995	return
996	if brain.course_admitted not in self._v_certificates:
997	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
998	return
999	if brain.sex not in (True,False):
1000	logger.info('sex of %s not available' % (reg_no))
1001	return
1002	self.init_timing()
1003	student_id = self.generateStudentId('?')
1004	students_folder.invokeFactory('Student', student_id)
1005	student_object = getattr(students_folder,student_id)
1006	self.do_timing()
1007	if pin_password:
1008	password = pin_parts[2]
1009	self.makeStudentMember(student_id,password = password)
1010	student_object.manage_setLocalRoles(student_id, ['Owner',])
1011	self.do_timing()
1012	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1013	#
1014	# application
1015	#
1016	student_object.invokeFactory('StudentApplication','application')
1017	application = student_object.application
1018	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1019	#self.do_timing()
1020	da = {'Title': 'Application Data'}
1021	da['jamb_reg_no'] = reg_no
1022
1023	sex = 'M'
1024	if brain.sex:
1025	sex = 'F'
1026	da['jamb_sex'] = sex
1027	da['jamb_age'] = brain.jamb_age
1028	da['app_reg_pin'] = brain.pin
1029	da['jamb_lga'] = brain.jamb_lga
1030	da['jamb_state'] = brain.jamb_state
1031	da['jamb_score'] = brain.aggregate
1032	da['app_email'] = brain.email
1033	da['app_mobile'] = brain.phone
1034
1035	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1036
1037	da['entry_session'] = entry_session
1038	da['jamb_lastname'] = brain.lastname
1039	da['jamb_middlename'] = brain.middlenames # different field names!
1040	da['jamb_firstname'] = brain.firstname
1041	da['screening_application_date'] = brain.application_date
1042	da['date_of_birth'] = brain.date_of_birth
1043	da['jamb_first_cos'] = brain.course1
1044	da['jamb_second_cos'] = brain.course2
1045	da['course3'] = brain.course3
1046	da['screening_type'] = brain.screening_type
1047	da['screening_score'] = brain.screening_score
1048	da['screening_date'] = brain.screening_date
1049	da['hq_type'] = brain.hq_type
1050	da['hq_grade'] = brain.hq_grade
1051	da['aos'] = brain.aos
1052
1053	application.getContent().edit(mapping=da)
1054	self.do_timing()
1055	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1056	#
1057	# personal
1058	#
1059	student_object.invokeFactory('StudentPersonal','personal')
1060	personal = student_object.personal
1061	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1062	#self.do_timing()
1063	dp = {'Title': 'Personal Data'}
1064	dp['sex'] = brain.sex
1065	dp['email'] = brain.email
1066	dp['phone'] = brain.phone
1067	dp['lastname'] = brain.lastname
1068	dp['middlename'] = brain.middlenames # different field names!
1069	dp['firstname'] = brain.firstname
1070	personal.getContent().edit(mapping=dp)
1071	self.do_timing()
1072	#
1073	# clearance
1074	#
1075	student_object.invokeFactory('StudentClearance','clearance')
1076	clearance = student_object.clearance
1077	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1078	dc = {'Title': 'Clearance/Eligibility Record'}
1079	dc['lga'] = brain.lga
1080	dc['birthday'] = brain.date_of_birth
1081	clearance.getContent().edit(mapping=dc)
1082	self.do_timing()
1083	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1084	#
1085	# study Course
1086	#
1087	student_object.invokeFactory('StudentStudyCourse','study_course')
1088	studycourse = student_object.study_course
1089	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1090	#self.do_timing()
1091	dsc = {}
1092	dsc['study_course'] = brain.course_admitted
1093	dsc['current_verdict'] = ''
1094	dsc['current_mode'] = da['entry_mode']
1095	if da['entry_mode'].startswith('de'):
1096	dsc['current_level'] = '200'
1097	elif da['entry_mode'].startswith('pre'):
1098	dsc['current_level'] = '000'
1099	else:
1100	dsc['current_level'] = '100'
1101	dsc['current_session'] = entry_session
1102	studycourse.getContent().edit(mapping=dsc)
1103	self.do_timing()
1104	#
1105	# payments folder
1106	student_object.invokeFactory('PaymentsFolder','payments')
1107	payments = getattr(student_object,'payments')
1108	#self.do_timing()
1109	dpay = {}
1110	dpay['Title'] = 'Payments'
1111	payments.getContent().edit(mapping=dpay)
1112	self.portal_workflow.doActionFor(payments,'open')
1113	self.do_timing()
1114	#
1115	# passport foto
1116	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1117	brain.screening_type,
1118	brain.reg_no)
1119	images_dir = getImagesDir(student_id)
1120	#images_dir = os.path.join("%s" % images_base,student_id)
1121	letter_dir,student_dir = os.path.split(images_dir)
1122	if not os.path.exists(letter_dir):
1123	os.mkdir(letter_dir)
1124	if not os.path.exists(images_dir):
1125	os.mkdir(images_dir)
1126	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1127	if os.path.exists(app_picture):
1128	copy2(app_picture,image_name)
1129	else:
1130	logger.info('passport of %s/%s not found: %s' % (student_id,
1131	brain.reg_no,
1132	app_picture))
1133
1134	self.do_timing()
1135	self.print_timing()
1136	if with_timing:
1137	self.timer_step = 0
1138	self._v_timer_count += 1
1139	self._v_total += self.total_time
1140	return student_id
1141	###)
1142
1143	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1144	def makeStudentLevel(self,student_id):
1145	"create the StudyLevel for a returning Student"
1146	#import pdb;pdb.set_trace()
1147	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1148	students_folder = self.portal_url.getPortalObject().campus.students
1149	res = self.students_catalog(id=student_id)
1150	if res:
1151	st = res[0]
1152	course = st.course
1153	matric_no = st.matric_no
1154	level = st.level
1155	res = self.results_import(matric_no = matric_no)
1156	if res:
1157	results = res
1158	logger.info('%s creating Level %s' % (student_id,level))
1159	#
1160	# Level
1161	#
1162	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1163	studycourse = getattr(student_obj,"study_course",None)
1164	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1165	l = getattr(studycourse,level,None)
1166	if l is None:
1167	studycourse.invokeFactory('StudentStudyLevel', level)
1168	l = getattr(studycourse, level)
1169	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1170	l.getContent().edit(mapping={'Title': "Level %s" % level})
1171	###)
1172
1173	security.declarePublic('getHallInfo') ###(
1174	def getHallInfo(self,bed):
1175	"""return Hall Info"""
1176	info = {}
1177	bedsplit = bed.split('_')
1178	if len(bedsplit) == 4:
1179	hall,block,room,letter = bed.split('_')
1180	else:
1181	info['maintenance_code'] = 'None'
1182	return info
1183	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1184	if res and len(res) == 1:
1185	hall_brain = res[0]
1186	hall_doc = hall_brain.getObject().getContent()
1187	else:
1188	return info
1189	info['hall_title'] = hall_brain.Title
1190	info['maintenance_code'] = hall_doc.maintenance_code
1191	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1192	batch_doc = None
1193	for brain in res:
1194	if brain.id.startswith(info['maintenance_code']):
1195	batch_doc = brain.getObject().getContent()
1196	break
1197	if batch_doc is None:
1198	info['maintenance_fee'] = ''
1199	else:
1200	info['maintenance_fee'] = batch_doc.cost
1201	return info
1202	###)
1203
1204	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1205	def removePictureFolder(self,student_id):
1206	"""remove picture_folder by renaming it"""
1207	path = 'images'
1208	picture_path = os.path.join(i_home,path,student_id)
1209	if not os.path.exists(picture_path):
1210	return False
1211	os.rename(picture_path,picture_path + "_removed")
1212	return True
1213	###)
1214
1215	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1216	def restorePictureFolder(self,student_id):
1217	"""restore picture_folder by renaming it"""
1218	path = 'images'
1219	picture_path = os.path.join(i_home,path,student_id)
1220	if not os.path.exists(picture_path + "_removed"):
1221	return False
1222	os.rename(picture_path + "_removed",picture_path)
1223	return True
1224	###)
1225
1226	security.declarePublic('picturesExist') ###(
1227	def picturesExist(self, ids,student_id=None):
1228	"""check if pictures exist in the filesystem"""
1229	if student_id is None:
1230	student_id = self.getStudentId()
1231	if student_id is None:
1232	return False
1233	picture_path = getImagesDir(student_id)
1234	#picture_path = os.path.join(images_base,student_id)
1235	if not os.path.exists(picture_path):
1236	return False
1237	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1238	return set(ids).issubset(set(pictures))
1239	###)
1240
1241	security.declarePublic('picturesList') ###(
1242	def picturesList(self):
1243	"""check if pictures exist in the filesystem"""
1244	path = 'images'
1245	student_id = self.getStudentId()
1246	#picture_path = os.path.join(i_home,path,student_id)
1247	picture_path = getImagesDir(student_id)
1248	if not os.path.exists(picture_path):
1249	return []
1250	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1251	###)
1252
1253	security.declarePublic('showFsPicture') ###(
1254	def showFsPicture(self,path):
1255	"""return a picture from the filesystem"""
1256	#picture_path = os.path.join(i_home,path)
1257	picture_path = os.path.join(images_base,path)
1258	response = self.REQUEST.RESPONSE
1259	#import pdb;pdb.set_trace()
1260	registry = getToolByName(self, 'mimetypes_registry')
1261	mimetype = str(registry.lookupExtension(path.lower()) or
1262	registry.lookupExtension('file.bin'))
1263	if os.path.exists(picture_path):
1264	response.setHeader('Content-type',mimetype)
1265	return open(picture_path).read()
1266	picture_path = os.path.join(i_home,'import',path)
1267	if os.path.exists(picture_path):
1268	return open(picture_path).read()
1269	###)
1270
1271	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1272	def deleteAllCourses(self,department="All"):
1273	''' delete the courses'''
1274	pm = self.portal_membership
1275	member = pm.getAuthenticatedMember()
1276
1277	if str(member) not in ("henrik","joachim"):
1278	return "not possible"
1279	if department == "All":
1280	res = self.portal_catalog({'meta_type': 'Department'})
1281	if len(res) < 1:
1282	return "No Departments found"
1283
1284	deleted = []
1285	for dep in res:
1286	cf = dep.getObject().courses
1287	if cf:
1288	cf.manage_delObjects(ids=cf.objectIds())
1289	deleted.append("deleted Courses in %s" % dep.getId)
1290	return "\r".join(deleted)
1291	###)
1292
1293	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1294	def getLogfileLines(self,filename="event.log",numlines=20):
1295	"""Get last NUMLINES lines of logfile FILENAME.
1296
1297	Return last lines' of a file in the instances logfile directory as
1298	a list. The number of returned lines equals `numlines' or less. If
1299	less than `numlines' lines are available, the whole file ist
1300	returned. If the file can not be opened or some other error
1301	occurs, empty list is returend.
1302	"""
1303	result = []
1304	lines_hit = 0
1305
1306	# We only handle files in instances' log directory...
1307	logpath = os.path.join(i_home, "log")
1308	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1309	if not filename.startswith( logpath ):
1310	# Attempt to access file outside log-dir...
1311	return []
1312
1313	try:
1314	fd = file( filename, "rb" )
1315	except IOError:
1316	return []
1317	if not fd:
1318	return []
1319
1320	if os.linesep == None:
1321	linesep = '\n'
1322	else:
1323	linesep = os.linesep
1324
1325	# Try to find 'numlines' times a lineseparator, searching from end
1326	# and moving to the beginning of file...
1327	fd.seek( 0, 2) # Move to end of file...
1328	while lines_hit < numlines:
1329	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1330	# one step forward...
1331	lines_hit += 1
1332	try:
1333	fd.seek( -2, 1) # Go two bytes back from current pos...
1334	except IOError:
1335	# We cannot go back two bytes. Maybe the file is too small...
1336	break
1337	fd.seek(2,1)
1338
1339	# Read all lines from current position...
1340	result = fd.readlines()
1341	# Remove line endings...
1342	result = [x.strip() for x in result]
1343	fd.close()
1344	return result
1345	###)
1346
1347	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1348	def getCallbacksFromLog(self,filename):
1349	"""fix Online Payment Transactions from Z2.log entries"""
1350	import transaction
1351	import random
1352	from cgi import parse_qs
1353	from urlparse import urlparse
1354	#from pdb import set_trace
1355	wftool = self.portal_workflow
1356	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1357	students_folder = self.portal_url.getPortalObject().campus.students
1358	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1359	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1360	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1361	data = re.compile(s)
1362	start = True
1363	tr_count = 1
1364	total = 0
1365	#name = 'pume_results'
1366	#name = 'epaymentsuccessful_z2log2'
1367	name = filename
1368	no_import = []
1369	imported = []
1370	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1371	try:
1372	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1373	except:
1374	logger.error('Error reading %s' % name)
1375	return
1376	tas = []
1377	for line in transactions:
1378	dict = {}
1379	items = data.search(line)
1380	dict['idict'] = idict = items.groupdict()
1381	#print idict
1382	#from pdb import set_trace;set_trace()
1383	urlparsed = urlparse(idict['get'][4:])
1384	#print urlparsed
1385	path = urlparsed[2].split('/')
1386	dict['student_id'] = student_id = path[8]
1387	dict['payment_id'] = payment_id = path[10]
1388	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1389	tas.append(dict)
1390	tr_count += 1
1391	return tas
1392	###)
1393
1394	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1395	def importOnlinePaymentTransactions(self):
1396	"""load Online Payment Transactions from CSV values"""
1397	import transaction
1398	import random
1399	#from pdb import set_trace
1400	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1401	opt = self.online_payments_import
1402	students_folder = self.portal_url.getPortalObject().campus.students
1403	start = True
1404	tr_count = 1
1405	total = 0
1406	#name = 'pume_results'
1407	name = 'OnlineTransactions'
1408	no_import = []
1409	imported = []
1410	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1411	try:
1412	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1413	except:
1414	logger.error('Error reading %s.csv' % name)
1415	return
1416	for pay_transaction in transactions:
1417	if start:
1418	start = False
1419	logger.info('Start loading from %s.csv' % name)
1420	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1421	no_import.append('%s,"Error"' % s)
1422	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1423	format_error = format + ',"%(Error)s"'
1424	data = {}
1425
1426	# format of the first file sent by Tayo
1427	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1428	#data['student_id'] = student_id = pay_transaction['Payer ID']
1429	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1430	#data['response_code'] = response_code = pay_transaction['Resp Code']
1431	#data['amount'] = amount = pay_transaction['Amount']
1432
1433	# format of the second file sent by Tayo
1434	#data['datetime'] = date = 0
1435	#data['student_id'] = student_id = pay_transaction['Payer ID']
1436	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1437	#data['response_code'] = response_code = '00'
1438	#data['amount'] = amount = pay_transaction['Amount']
1439
1440	# format of the third file sent by Kehinde
1441	data['datetime'] = date = 0
1442	data['student_id'] = student_id = pay_transaction['customer_id']
1443	data['order_id'] = order_id = pay_transaction['merchant_reference']
1444	data['response_code'] = response_code = '00'
1445	data['amount'] = amount = pay_transaction['Amount']
1446
1447	dup = False
1448	if response_code == "12":
1449	continue
1450	try:
1451	opt.addRecord(**data)
1452	except ValueError:
1453	dup = True
1454	#from pdb import set_trace;set_trace()
1455	if dup:
1456	if response_code == "00":
1457	try:
1458	opt.modifyRecord(**data)
1459	except:
1460	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1461	continue
1462	else:
1463	pay_transaction['Error'] = "Duplicate order_id"
1464	no_import.append( format_error % pay_transaction)
1465	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1466	continue
1467	tr_count += 1
1468	if tr_count > 1000:
1469	if len(no_import) > 0:
1470	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1471	'\n'.join(no_import) + '\n')
1472	no_import = []
1473	em = '%d transactions committed\n' % (tr_count)
1474	transaction.commit()
1475	regs = []
1476	logger.info(em)
1477	total += tr_count
1478	tr_count = 0
1479	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1480	'\n'.join(no_import))
1481	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1482	###)
1483
1484	security.declareProtected(ModifyPortalContent,"importData")###(
1485	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1486	"""load data from CSV values"""
1487	import transaction
1488	import random
1489	students_folder = self.portal_url.getPortalObject().campus.students
1490	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1491	pending_only = False
1492	pend_str = '--'
1493	elapse = time.time()
1494	#
1495	# preparations
1496	#
1497	if filename == pend_str:
1498	pending_only = True
1499	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1500	importer = eval("%sImport" % importer_name)(self)
1501	logger = importer.logger
1502	if importer.init_errors:
1503	logger.info(importer.init_errors)
1504	return importer.init_errors
1505	member = importer.member
1506	#current = importer.current
1507	import_date = importer.import_date
1508	#
1509	# not_imported
1510	#
1511	info = importer.info
1512	data_keys = importer.data_keys
1513	csv_keys = importer.csv_keys
1514	#csv_keys.extend(info.keys())
1515	headline_mapping = dict((k,k) for k in csv_keys)
1516	#
1517	# pending
1518	#
1519	pending_path = importer.pending_path
1520	pending_tmp = importer.pending_tmp
1521	pending_backup = importer.pending_backup
1522	pending_fn = importer.pending_fn
1523	imported_path = importer.imported_path
1524	imported_fn = importer.imported_fn
1525	commit_after = importer.commit_after
1526	pending = []
1527	pending_digests = []
1528	#total_added_to_pending = 0
1529	if not pending_only:
1530	pending,pending_digests = importer.makeIdLists()
1531	pending_at_start = len(pending)
1532	datafile = open(pending_tmp,"w")
1533	pending_csv_writer = csv.DictWriter(datafile,
1534	csv_keys,
1535	extrasaction='ignore')
1536	pending_csv_writer.writerow(headline_mapping)
1537	datafile.close()
1538	#
1539	# imported
1540	#
1541	if not os.path.exists(imported_path):
1542	datafile = open(imported_path,"w")
1543	imported_csv_writer = csv.DictWriter(datafile,
1544	csv_keys,
1545	extrasaction='ignore')
1546	imported_csv_writer.writerow(headline_mapping)
1547	datafile.close()
1548	start = True
1549	tr_count = 0
1550	total = 0
1551	total_added_to_imported = 0
1552	total_pending = 0
1553	import_source_done = ""
1554	if pending_only:
1555	import_source_path = pending_path
1556	else:
1557	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1558	import_source_done = "%s/import/%s.done" % (i_home,filename)
1559	if not os.path.exists(import_source_path):
1560	fn = os.path.split(import_source_path)[1]
1561	em = 'no import file %(fn)s' % vars()
1562	return em
1563	import_source_fn = os.path.split(import_source_path)[1]
1564	if not pending_only:
1565	info['imported_from'] = import_source_fn
1566	headline = csv.reader(open(import_source_path,"rb")).next()
1567	if "import_mode" not in headline:
1568	msg = 'import_mode must be in heading'
1569	return msg
1570	invalid_keys = importer.checkHeadline(headline)
1571	if invalid_keys:
1572	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1573
1574	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1575	or k.strip() in info.keys())]
1576	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1577	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1578	# if diff2schema and diff2schema != set(['id',]):
1579	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1580	# return msg
1581	#
1582	# start importing
1583	#
1584	try:
1585	reader = csv.DictReader(open(import_source_path,"rb"))
1586	except:
1587	msg = 'Error reading %s.csv' % filename
1588	logger.error(msg)
1589	return msg
1590	items = [item for item in reader]
1591	total_to_import = len(items)
1592	tti_float = float(total_to_import)
1593	if pending_only:
1594	pending_at_start = total_to_import
1595	count = 0
1596	imported = []
1597	old_commit_count = 0
1598	error_count = imported_count = 0
1599	already_in = 0
1600	for record in items:
1601	item = {}
1602	empty_value_keys = []
1603	for k,v in record.items():
1604	if k is None:
1605	continue
1606	if v:
1607	if v == EMPTY:
1608	empty_value_keys += k,
1609	v = ''
1610	item[k.strip()] = v.strip()
1611	count += 1
1612	if start:
1613	start = False
1614	adapters = [MappingStorageAdapter(importer.schema, item)]
1615	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1616	dm = DataModel(item, adapters,context=self)
1617	ds = DataStructure(data=item,datamodel=dm)
1618	error_string = ""
1619	total += 1
1620	import_mode = item.get('import_mode','')
1621	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1622	if import_method is None:
1623	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1624	elif (import_mode in importer.required_modes and
1625	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1626	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1627	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1628	else:
1629	for k in import_keys:
1630	if k not in item.keys() or k not in importer.validators.keys():
1631	continue
1632	if not importer.validators[k](ds,mode=import_mode):
1633	error_string += ' ++ '
1634	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1635	ds.getErrorMapping(k)))
1636	if error_string:
1637	error = error_string
1638	id = ''
1639	mapping = item
1640	else:
1641	temp_item = item.copy()
1642	temp_item.update(dm)
1643	results = import_method(temp_item)
1644	id = results[0]
1645	error = results[1]
1646	mapping = results[2]
1647	#
1648	# restore some values
1649	#
1650	for k in empty_value_keys:
1651	mapping[k] = EMPTY
1652	if mapping.has_key('sex'):
1653	#import pdb;pdb.set_trace()
1654	if mapping['sex'] == True:
1655	mapping['sex'] = 'F'
1656	elif mapping['sex'] == False:
1657	mapping['sex'] = 'M'
1658	if pending_only:
1659	info['imported_from'] = item['imported_from']
1660	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1661	info['error'] = error
1662	info['import_record_no'] = count + 1
1663	mapping.update(info)
1664	log_list = []
1665	if error:
1666	error_count += 1
1667	digest = makeDigest(mapping,data_keys)
1668	if digest not in pending_digests:
1669	pending_digests += digest,
1670	pending.append(mapping)
1671	if not pending_only:
1672	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1673	else:
1674	already_in += 1
1675	pass
1676	else:
1677	imported_count += 1
1678	imported += mapping,
1679	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1680	if log_list:
1681	time_till_now = time.time() - elapse
1682	percent_finished = (error_count + imported_count)/tti_float*100
1683	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1684	logger.info(' '.join(log_list))
1685	finished = count > total_to_import - 1
1686	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1687	if must_commit:
1688	old_commit_count = imported_count
1689
1690	if must_commit or finished:
1691	if len(imported):
1692	transaction.commit()
1693	datafile = open(imported_path,"a")
1694	writer = csv.DictWriter(datafile,
1695	csv_keys,
1696	extrasaction='ignore')
1697	writer.writerows(imported)
1698	datafile.close()
1699	total_added_to_imported += len(imported)
1700	imported = []
1701	if len(pending) > 0:
1702	datafile = open(pending_tmp,"a")
1703	writer = csv.DictWriter(datafile,
1704	csv_keys,
1705	extrasaction='ignore')
1706	writer.writerows(pending)
1707	datafile.close()
1708	total_pending += len(pending)
1709	#total_added_to_pending += len(pending)
1710	pending = []
1711	if not finished:
1712	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1713	logger.info(msg)
1714	elapse = time.time() - elapse
1715	if os.path.exists(pending_path):
1716	copy2(pending_path,pending_backup)
1717	copy2(pending_tmp,pending_path)
1718	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1719	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1720	if pending_only:
1721	removed_pending = pending_at_start - total_pending
1722	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1723	else:
1724	added_pending = total_pending - pending_at_start
1725	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1726	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1727	logger.info(msg)
1728	if import_source_done:
1729	copy(import_source_path,import_source_done)
1730	os.remove(import_source_path)
1731	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1732	if upload is not None:
1733	upload_doc = upload.getContent()
1734	mapping = {}
1735	#mapping['import_date'] = DateTime.DateTime()
1736	mapping['import_date'] = import_date
1737	mapping['imported_by'] = importer.imported_by
1738	mapping['import_message'] = msg
1739	upload_doc.edit(mapping = mapping)
1740	os.remove(pending_tmp)
1741	return msg
1742	###)
1743
1744	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1745	def moveImagesToFS(self,student_id="O738726"):
1746	"move the images to the filesystem"
1747	images_dir = getImagesDir(student_id)
1748	#images_dir = os.path.join("%s" % images_base,student_id)
1749	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1750	stool = getToolByName(self, 'portal_schemas')
1751	schemas = ['student_application',
1752	'student_clearance',
1753	]
1754	created = False
1755	for schema_id in schemas:
1756	schema = stool._getOb(schema_id)
1757	object = getattr(student_folder,schema_id[len('student_'):],None)
1758	if object is None:
1759	continue
1760	doc = object.getContent()
1761	for key in schema.keys():
1762	if schema[key].meta_type != "CPS Image Field":
1763	continue
1764	#import pdb;pdb.set_trace()
1765	image = getattr(doc,key,None)
1766	if not image or not hasattr(image,"data"):
1767	continue
1768	if not created:
1769	if not os.path.exists(images_dir):
1770	os.mkdir(images_dir)
1771	created = True
1772	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1773	open(filename,"wb").write(str(image.data))
1774	###)
1775
1776	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1777	def movePassportToFS(self,student_id="O738726"):
1778	"move the passports to the filesystem"
1779	images_dir = os.path.join("%s" % i_home,'passports')
1780	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1781	stool = getToolByName(self, 'portal_schemas')
1782	schemas = ['student_application',
1783	#'student_clearance',
1784	]
1785	created = False
1786	for schema_id in schemas:
1787	schema = stool._getOb(schema_id)
1788	object = getattr(student_folder,schema_id[len('student_'):],None)
1789	if object is None:
1790	continue
1791	doc = object.getContent()
1792	for key in schema.keys():
1793	if schema[key].meta_type != "CPS Image Field":
1794	continue
1795	#import pdb;pdb.set_trace()
1796	image = getattr(doc,key)
1797	if not hasattr(image,"data"):
1798	continue
1799	if not created:
1800	if not os.path.exists(images_dir):
1801	os.mkdir(images_dir)
1802	created = True
1803	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1804	open(filename,"wb").write(str(image.data))
1805	###)
1806
1807	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3421

Download in other formats: