Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3452

Last change on this file since 3452 was 3452, checked in by joachim, 17 years ago
sanitize getClearanceInfo and clearance_edit, adopt views, infopenalty? is now set to False.
Property svn:keywords set to `Id`
File size: 72.8 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3452 2008-04-19 09:19:02Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2,copy
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108
109	###)
110
111	security.declareProtected(View,'updateRoleMappingsFor') ###(
112	def updateRoleMappingsFor(self,wf_definition,ob):
113	"do so for public"
114	wf_def = getattr(self.portal_workflow,wf_definition)
115	wf_def.updateRoleMappingsFor(ob)
116	###)
117
118	security.declareProtected(View,'getStatesLgas') ###(
119	def getStatesLgas(self):
120	"""return lga info"""
121	voc = getattr(self.portal_vocabularies,'local_gov_areas')
122	states = []
123	lgas = []
124	d = {}
125	wd = {}
126	for k,v in voc.items():
127	parts = v.split(' / ')
128	if len(parts) == 1:
129	state = parts[0].lower()
130	lga = ""
131	elif len(parts) == 2:
132	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134	else:
135	continue
136	if state not in states:
137	states.append(state)
138	if lga not in lgas:
139	lgas.append(lga)
140	words = re.split('[^a-zA-Z0-9/]',k)
141	words.sort()
142	wd[k] = words
143	d[k] = v
144	mapping = {}
145	mapping['word_dict'] = wd
146	mapping['lga_dict'] = d
147	mapping['states'] = states
148	mapping['lgas'] = lgas
149	return mapping
150	###)
151
152	security.declareProtected(View,'findLga') ###(
153	def findLga(self,words,words_dict):
154	words = re.split('[^a-zA-Z0-9/]',words)
155	lga_words = []
156	for word in words:
157	if word:
158	lga_words += word.strip().lower(),
159	lga_words.sort()
160	state_lga = ''
161	while not state_lga:
162	for k,l in words_dict.items():
163	if lga_words == l:
164	state_lga = k
165	break
166	break
167	return state_lga
168	###)
169	security.declareProtected(View,'getAccessInfo') ###(
170	def getAccessInfo(self,context):
171	"return a dict with access_info"
172	logger = logging.getLogger('WAeUPTool.getAccessInfo')
173	mtool = self.portal_membership
174	member = mtool.getAuthenticatedMember()
175	member_id = str(member)
176	info = {}
177	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179	is_staff = info['is_staff'] = not is_anonymous and not is_student
180	roles = member.getRolesInContext(context)
181	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182	"SectionManager" in roles or
183	"Manager" in roles)
184	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
185	is_allowed = info['is_allowed'] = not is_anonymous
186	requested_id = context.getStudentId()
187	student_id = None
188	if is_allowed:
189	if not is_student and requested_id:
190	student_id = requested_id
191	elif is_anonymous or member_id != requested_id:
192	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
193	else:
194	student_id = member_id
195	info['student_id'] = student_id
196	return info
197	###)
198
199	security.declareProtected(ModifyPortalContent,'openLog') ###(
200	def openLog(self,name):
201	"""open a log file"""
202	version = 1
203	path = "%s/log/%s_%d.log" % (i_home,name,version)
204	while os.path.exists(path):
205	version += 1
206	path = "%s/log/%s_%d.log" % (i_home,name,version)
207	log = open(path,"w")
208	return log
209	###)
210
211	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
212	def bypassQueueCatalog(self,enable=True):
213	"""bypass the QueueCatalog by setting all indexes to process imediate,
214	if enable is True (default) the old settings are restored
215	"""
216
217	###)
218
219	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
220	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
221	"""measure Object access Time"""
222	import random
223	if hasattr(self,'portal_catalog_real'):
224	aq_portal = self.portal_catalog_real.evalAdvancedQuery
225	else:
226	aq_portal = self.portal_catalog.evalAdvancedQuery
227	nr_pts = int(nr_pts)
228	probe = int(probe)
229	intervall = probe/10
230	objects = ("application","clearance","personal")
231	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
232	#i = random.randrange(num_objects)
233	count = 0
234	found = 0
235	not_found = 0
236	t_found = 0
237	t_not_found = 0
238	time_found = time_not_found = 0.0
239	t_time_found = t_time_not_found = 0.0
240	accessed = []
241	t_min = 1000
242	t_max = 0
243	#import pdb;pdb.set_trace()
244	students = self.portal_catalog(portal_type="Student")
245	num_students = len(students)
246	if method == "d":
247	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
248	res = aq_portal(query)
249	brains = {}
250	for r in res:
251	sid = r.relative_path.split('/')[-2]
252	if brains.has_key(sid):
253	brains[sid][r.portal_type] = r
254	else:
255	brains[sid] = {r.portal_type : r}
256	brains_list = brains.keys()
257	num_objects = len(brains_list)
258	else:
259	num_objects = num_students
260	print "="*40
261	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
262	probe,
263	nr_pts,
264	num_objects)
265	print "nr found/not time found/not min/max"
266	elapse = time.time()
267	i_elapse = time.time()
268	c_elapse = time.clock()
269	for c in range(1,probe + 1):
270	i = random.randrange(num_objects)
271	if method in ('a','b','c'):
272	student_brain = students[i]
273	elif method == "d":
274	#import pdb;pdb.set_trace()
275	student_brain = brains[brains_list[i]]
276	if method == "c":
277	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
278	res = aq_portal(query)
279	this_portal_types = [r.portal_type for r in res]
280	for i in range(nr_pts):
281	oid = objects[i]
282	if method == "a":
283	try:
284	student_path = student_brain.getPath()
285	path = "%s/%s" % (student_path,oid)
286	doc = self.unrestrictedTraverse(path).getContent()
287	found += 1
288	i_time = time.time() - i_elapse
289	time_found += i_time
290	except:
291	not_found += 1
292	i_time = time.time() - i_elapse
293	time_not_found += i_time
294	pass
295	elif method == "b":
296	try:
297	student_object = student_brain.getObject()
298	doc = getattr(student_object,oid).getContent()
299	found += 1
300	i_time = time.time() - i_elapse
301	time_found += i_time
302	except:
303	i_time = time.time() - i_elapse
304	time_not_found += i_time
305	not_found += 1
306	pass
307	elif method == "c":
308	if portal_types[i] in this_portal_types:
309	found += 1
310	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
311	i_time = time.time() - i_elapse
312	time_found += i_time
313	else:
314	not_found += 1
315	i_time = time.time() - i_elapse
316	time_not_found += i_time
317	elif method == "d":
318	if student_brain.has_key(portal_types[i]):
319	found += 1
320	doc = student_brain[portal_types[i]].getObject().getContent()
321	i_time = time.time() - i_elapse
322	time_found += i_time
323	else:
324	not_found += 1
325	i_time = time.time() - i_elapse
326	time_not_found += i_time
327	i_elapse = time.time()
328	if c and (c % intervall == 0):
329	#i_time = time.time() - i_elapse
330	t_per = 0.0
331	if found:
332	t_per = time_found/found
333	if t_per > t_max:
334	t_max = t_per
335	if t_per > 0.0 and t_per < t_min:
336	t_min = t_per
337	itf = 0.0
338	if found:
339	itf = time_found/found
340	itnf = 0.0
341	if not_found :
342	itnf = time_not_found / not_found
343	interval_time = time_found + time_not_found
344	s = "%(c)d: %(found)d/%(not_found)d " % vars()
345	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
346	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
347	print s
348	t_found += found
349	t_not_found += not_found
350	t_time_found += time_found
351	t_time_not_found += time_not_found
352	time_found = time_not_found = 0.0
353	found = not_found = 0
354	# t_found += found
355	# t_not_found += not_found
356	elapse = time.time() - elapse
357	itf = 0.0
358	if t_found:
359	itf = t_time_found/t_found
360	itnf = 0.0
361	if t_not_found:
362	itnf = t_time_not_found / t_not_found
363	#c_elapse = time.clock() - c_elapse
364	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
365	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
366	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
367	print "-"*40
368	print s
369	rel_found = float(t_found)/probe
370	rel_not_found = float(t_not_found)/probe
371	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
372	print estimated_total_time
373	###)
374
375	security.declareProtected(ModifyPortalContent,'writeLog') ###(
376	def writeLog(self,logfile,s):
377	"""write to the log file"""
378	logfile.write(s)
379
380	###)
381
382	def generateStudentId(self,letter): ###(
383	import random
384	r = random
385	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
386	if letter == '?':
387	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
388	sid = "%c%d" % (letter,r.randint(99999,1000000))
389	students = self.portal_url.getPortalObject().campus.students
390	## while hasattr(students, sid):
391	## sid = "%c%d" % (letter,r.randint(99999,1000000))
392	while self.students_catalog(id = sid):
393	sid = "%c%d" % (letter,r.randint(99999,1000000))
394	return sid
395	###)
396
397	def generatePassword(self,s=None): ###(
398	import random
399	r = random
400	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
401	if s is None:
402	s = 'abcdefghklmnpqrstuvwxy23456789'
403	pw = ''
404	while len(pw) < 6:
405	pw += r.choice(s)
406	return pw
407	###)
408
409	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
410	def dumpSchoolfeePayments(self):
411	"dump paid schoolfees"
412	mtool = self.portal_membership
413	member = mtool.getAuthenticatedMember()
414	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
415	aq_student = self.students_catalog.evalAdvancedQuery
416	query = In('review_state',('schoolfee_paid',
417	'courses_registered',
418	'courses_validated',
419	))
420	res = aq_student(query)
421	#import pdb;pdb.set_trace()
422	l = []
423	logger.info("start for %d" % len(res))
424	count = 1
425	log_after = 100
426	for student in res:
427	if not count % log_after:
428	logger.info("processed %d total %d" % (log_after,count))
429	count += 1
430	fee_dict =self.getSchoolFee(student)
431	fulltime = student.mode.endswith('_ft')
432	d = {}
433	d['student_id'] = student.id
434	d['name'] = student.name
435	d['amount'] = fee_dict.get(new_returning)
436	l += d,
437	csv_name = self.dumpListToCSV(l,'payments')
438	logger.info('%s dumped payments to %s' % (member,export_file))
439	###)
440
441	security.declarePublic('dumpListToCSV') ###(
442	def dumpListToCSV(self,l,filename,fields=None):
443	"""dump a list of dicts to a CSV file"""
444	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
445	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
446	if fields is None:
447	fields = l[0].keys()
448	headline = ','.join(fields)
449	out = open(export_file,"wb")
450	out.write(headline +'\n')
451	out.close()
452	out = open(export_file,"a")
453	csv_writer = csv.DictWriter(out,fields,)
454	csv_writer.writerows(l)
455	return export_file
456	###)
457
458	security.declareProtected(ManagePortal, 'listMembers') ###(
459	def listMembers(self):
460	"list all members"
461	mtool = self.portal_membership
462	member = mtool.getAuthenticatedMember()
463	logger = logging.getLogger('WAeUPTool.listMembers')
464	if str(member) not in ('admin','joachim'):
465	logger.info('%s tried to list members' % (member))
466	return None
467	members = self.portal_directories.members
468	all = members.listEntryIdsAndTitles()
469	l = []
470	for user_id,name in all:
471	d = {}
472	d['user_id'] = user_id
473	d['name'] = name
474	d['pw'] = getattr(getattr(members,user_id),'password')
475	d['email'] = getattr(getattr(members,user_id),'email')
476	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
477	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
478	l += d,
479	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
480	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
481	logger.info('%s dumped member list to %s' % (member,export_file))
482	fields = l[0].keys()
483	headline = ','.join(fields)
484	out = open(export_file,"wb")
485	out.write(headline +'\n')
486	out.close()
487	out = open(export_file,"a")
488	csv_writer = csv.DictWriter(out,fields,)
489	csv_writer.writerows(l)
490	###)
491
492	security.declareProtected(ManagePortal, 'listStudents') ###(
493	def listStudents(self):
494	"list all students"
495	mtool = self.portal_membership
496	member = mtool.getAuthenticatedMember()
497	logger = logging.getLogger('WAeUPTool.listStudents')
498	if str(member) not in ('admin','joachim'):
499	logger.info('%s tried to list students' % (member))
500	return None
501	students = self.portal_directories.students
502	all = students.listEntryIdsAndTitles()
503	l = []
504	for user_id,name in all:
505	d = {}
506	d['user_id'] = user_id
507	d['name'] = name
508	d['pw'] = getattr(getattr(students,user_id),'password')
509	d['email'] = getattr(getattr(students,user_id),'email')
510	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
511	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
512	l += d,
513	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
514	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
515	logger.info('%s dumped student list to %s' % (member,export_file))
516	fields = l[0].keys()
517	headline = ','.join(fields)
518	out = open(export_file,"wb")
519	out.write(headline +'\n')
520	out.close()
521	out = open(export_file,"a")
522	csv_writer = csv.DictWriter(out,fields,)
523	csv_writer.writerows(l)
524	###)
525
526	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
527	def removeDeletedDocIds(self, max=1000):
528	"""
529	remove deleted docids from repository commit after max
530	"""
531	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
532	repository = getToolByName(self, 'portal_repository')
533	pxtool = getToolByName(self, 'portal_proxies')
534	logger.info('start')
535	pxtool_infos = pxtool.getRevisionsUsed()
536	logger.info('found %d used revisions ' % (len(pxtool_infos)))
537
538	nb_revs = 0
539	docids_d = {} # all docids
540	unused_docids_d = {} # all docids that are unused
541	ids_unused_revs_docids = [] # ids for revs of unused docids
542	unused_ids = [] # ids for unused revs
543	total = 0
544	idlist = repository.objectIds()
545	to_delete = 0
546	found = False
547	for id in idlist:
548	docid, rev = repository._splitId(id)
549	if docid is None:
550	logger.info("invalid doc_id %s" % docid)
551	continue
552	nb_revs += 1
553	if not pxtool_infos.has_key(docid):
554	found = True
555	to_delete += 1
556	unused_ids.append(id)
557	elif not pxtool_infos[docid].has_key(rev):
558	found = True
559	to_delete += 1
560	unused_ids.append(id)
561	if found and not to_delete % max:
562	found = False
563	#import pdb;pdb.set_trace()
564	repository.manage_delObjects(unused_ids)
565	transaction.commit()
566	logger.info('removed %d total %d unused docids ' % (max,to_delete))
567	else:
568	if unused_ids:
569	repository.manage_delObjects(unused_ids)
570	transaction.commit()
571	logger.info('finished removing %d unused docids ' % (to_delete))
572
573
574	###)
575
576	security.declareProtected(View,'getCredential') ###(
577	def getCredential(self,student_id):
578	student_entry = getattr(self.portal_directories.students,student_id,None)
579	if not self.isStaff():
580	mtool = self.portal_membership
581	member = mtool.getAuthenticatedMember()
582	logger = logging.getLogger('WAeUPTool.getCredential')
583	logger.info('%s tried to access password of %s' % (member,student_id))
584	return None
585	if student_entry is None:
586	return None
587	return getattr(student_entry,"password","not set")
588	###)
589
590	security.declarePublic('checkPassword') ###(
591	def checkPassword(self,student_id,password):
592	student_entry = getattr(self.portal_directories.students,student_id,None)
593	if student_entry is None:
594	return False
595	return getattr(student_entry,"password","not set") == password
596	###)
597
598	security.declarePublic('checkGenericPassword') ###(
599	def checkGenericPassword(self,member_id):
600	member_entry = getattr(self.portal_directories.members,member_id,None)
601	if member_entry is None:
602	return False
603	ltool = getToolByName(self, 'portal_layouts')
604	unsecure_words = ltool._getOb('members')['w__password'].check_words
605	password = getattr(member_entry,"password","not set")
606	is_unsecure = password in unsecure_words
607	if is_unsecure:
608	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
609	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
610	return is_unsecure
611	###)
612
613	security.declareProtected(ModifyPortalContent,'editPassword') ###(
614	def editPassword(self,student_id,password):
615	"edit a student password"
616	student_entry = getattr(self.portal_directories.students,student_id,None)
617	if student_entry is None:
618	return
619	setattr(student_entry,'password',password)
620	###)
621
622	security.declareProtected(ModifyPortalContent,'doCommit') ###(
623	def doCommit(self,logger=None):
624	"commit some transactions"
625	transaction.commit()
626	###)
627
628	security.declarePublic('loadStudentFoto') ###(
629	def loadStudentFoto(self,student,filename,folder):
630	"return a student passport picture"
631	#import pdb;pdb.set_trace()
632	picture ="%s/import/%s/%s" % (i_home,folder,filename)
633	student_id = student.getId()
634	images_dir = getImagesDir(student_id)
635	if not os.path.exists(images_dir):
636	os.mkdir(images_dir)
637	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
638	for extension in ('.jpg','.JPG'):
639	fullname = "%(picture)s%(extension)s" % vars()
640	if os.path.exists(fullname):
641	copy2(fullname,image_name)
642	return "successfully copied passport picture"
643	return "passport picture not found: %s.jpg or .JPG" % picture
644	###)
645
646	def old____loadStudentFoto(self,student,filename,folder): ###(
647	"return a student passport picture"
648	app = student.application
649	app_doc = app.getContent()
650	#clear = student.clearance
651	#clear_doc = clear.getContent()
652	#matric_no = clear_doc.matric_no.upper()
653	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
654	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
655	#import pdb;pdb.set_trace()
656	if os.path.exists(picture1):
657	file = open(picture1)
658	elif os.path.exists(picture2):
659	file = open(picture2)
660	else:
661	return "passport picture not found %s" % picture1
662	reopened = False
663	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
664	self.portal_workflow.doActionFor(app,'open')
665	reopened = True
666	outfile = file.read()
667	app_doc.manage_addFile('passport',
668	file=outfile,
669	title="%s.jpg" % filename)
670	if reopened:
671	self.portal_workflow.doActionFor(app,'close')
672	return "successfully loaded passport picture"
673	###)
674
675	security.declareProtected(ModifyPortalContent,'createOne') ###(
676	def createOne(self,students_folder,student_brain,letter,commit=False):
677	sid = self.waeup_tool.generateStudentId(letter)
678	students_folder.invokeFactory('Student', sid)
679	student = getattr(students_folder,sid)
680	self.portal_workflow.doActionFor(student,'return')
681	student.manage_setLocalRoles(sid, ['Owner',])
682	matric_no = student_brain.matric_no
683	jamb_reg_no = student_brain.Entryregno
684	self.students_catalog.addRecord(id = sid,
685	matric_no = matric_no,
686	jamb_reg_no = jamb_reg_no,
687	sex = student_brain.Sex == "F",
688	name = "%s %s %s" % (student_brain.Firstname,
689	student_brain.Middlename,
690	student_brain.Lastname)
691	)
692	if commit:
693	transaction.commit()
694	return sid,jamb_reg_no
695	###)
696
697	security.declareProtected(ModifyPortalContent,'addStudent') ###(
698	def addStudent(self,dict):
699	students_folder = self.portal_url.getPortalObject().campus.students
700	sid = self.waeup_tool.generateStudentId('?')
701	students_folder.invokeFactory('Student', sid)
702	student_obj = getattr(students_folder,sid)
703	f2t = self.field2types_student
704	#from pdb import set_trace; set_trace()
705	d = {}
706	#d['jamb_sex'] = 'M'
707	#if dict.get('sex'):
708	# d['jamb_sex'] = 'F'
709
710	entry_session = dict.get('entry_session')
711	if entry_session == self.getSessionId()[0]:
712	wfaction = 'admit'
713	wft = 'wf_transition_admit'
714	password = None
715	else:
716	wfaction = 'return'
717	wft = 'wf_transition_return'
718	password = self.generatePassword()
719	self.makeStudentMember(sid,password)
720
721	for pt in f2t.keys():
722	student_obj.invokeFactory(pt,f2t[pt]['id'])
723	sub_obj = getattr(student_obj,f2t[pt]['id'])
724	sub_doc = sub_obj.getContent()
725	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
726	d['Title'] = f2t[pt]['title']
727	for field in f2t[pt]['fields']:
728	d[field] = dict.get(field,'')
729	sub_doc.edit(mapping = d)
730	new_state = f2t[pt][wft]
731	if new_state != "remain":
732	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
733	self.portal_workflow.doActionFor(student_obj,wfaction)
734	student_obj.manage_setLocalRoles(sid, ['Owner',])
735	return sid,password
736	###)
737
738	security.declarePublic('getCertificateBrain') ###(
739	def getCertificateBrain(self,cert_id):
740	"do it"
741	res = ZCatalog.searchResults(self.portal_catalog_real,
742	{'portal_type':"Certificate",
743	'id': cert_id})
744	if res:
745	return res[0]
746	return None
747	###)
748
749	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
750	def get_csv_filenames(self):
751	"do it"
752	files = [file for file in os.listdir("%s/import/" % (i_home))
753	if file.endswith('.csv') and (file.find('imported') == -1 and
754	file.find('pending') == -1)]
755	return files
756	###)
757
758	security.declarePublic('findStudentByMatricelNo') ###(
759	def findStudentByMatricelNo(self,matric_no):
760	"do it"
761	res = ZCatalog.searchResults(self.portal_catalog_real,
762	{'portal_type':"StudentClearance",
763	'SearchableText': matric_no})
764	if res:
765	return res[0]
766	return None
767	###)
768
769	security.declarePublic('makeStudentMember') ###(
770	def makeStudentMember(self,sid,password='uNsEt'):
771	"""make the student a member"""
772	membership = self.portal_membership
773	membership.addMember(sid,
774	password ,
775	roles=('Member',
776	'Student',
777	),
778	domains='',
779	properties = {'memberareaCreationFlag': False,
780	'homeless': True},)
781	member = membership.getMemberById(sid)
782	self.portal_registration.afterAdd(member, sid, password, None)
783	#self.manage_setLocalRoles(sid, ['Owner',])
784	###)
785
786	security.declareProtected(View,'makeStudentData') ###(
787	def makeStudentData(self,student_id,email=None,phone_nr=None):
788	"create Datastructure for a returning Student"
789	#import pdb;pdb.set_trace()
790	logger = logging.getLogger('WAeUPTool.makeStudentData')
791	students_folder = self.portal_url.getPortalObject().campus.students
792	#res = self.students_catalog(id=student_id)
793	#if res:
794	# st = res[0]
795	#res = self.returning_import(matric_no = st.matric_no)
796	res = self.returning_import(id = student_id)
797	if res:
798	student = res[0]
799	else:
800	logger.info('Id %s not found in returning_import' % student_id)
801	return
802	logger.info('%s creates data structure' % student_id)
803	s_results = self.results_import(matric_no = student.matric_no)
804	if s_results:
805	lnr = self.getLevelFromResultsCosCode(s_results)
806	level = "%d00" % lnr
807	verdict,eligible = self.getVerdict(s_results[0].Verdict)
808	#if eligible:
809	# level = "%d00" % (lnr + 1)
810	else:
811	logger.info('matric_no %s not found in results_import' % student.matric_no)
812	level = ''
813	verdict = ''
814	#student should not be allowed to perform this transition
815	#wftool = self.portal_workflow
816	#wftool.doActionFor(student,'return')
817	certcode_org = student.Coursemajorcode
818	certcode = makeCertificateCode(certcode_org)
819	certificate_brain = self.getCertificateBrain(certcode)
820	if not certificate_brain:
821	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
822	logger.info(em)
823	matric_no = student.matric_no
824	sid = student_id
825	student_obj = getattr(students_folder,sid)
826	if not getattr(student_obj,'application'):
827	student_obj.invokeFactory('StudentApplication','application')
828	application = student_obj.application
829	self.portal_workflow.doActionFor(application,'open',dest_container=application)
830	da = {'Title': 'Application Data'}
831	student_obj.invokeFactory('StudentPersonal','personal')
832	da['jamb_reg_no'] = student.Entryregno
833	em = self.getEntryMode(student.Entryregno)
834	da['entry_mode'] = em
835	personal = student_obj.personal
836	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
837	dp = {'Title': 'Personal Data'}
838	student_obj.invokeFactory('StudentClearance','clearance')
839	clearance = student_obj.clearance
840	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
841	dc = {'Title': 'Clearance/Eligibility Record'}
842	dc['matric_no'] = matric_no
843	state = student.State
844	lga = student.LGA
845	if state and lga:
846	lga = state + ' / ' + lga
847	else:
848	lga = "None"
849	da['jamb_lga'] = dc['lga'] = lga
850	da['app_email'] = dp['email'] = email
851	da['app_mobile'] = dp['phone'] = phone_nr
852	dp['firstname'] = student.Firstname
853	dp['middlename'] = student.Middlename
854	dp['lastname'] = student.Lastname
855	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
856	da['jamb_sex'] = student.Sex
857	dp['sex'] = student.Sex == 'F'
858	dp['perm_address'] = student.Permanent_Address
859	application.getContent().edit(mapping=da)
860	self.portal_workflow.doActionFor(application,'close',dest_container=application)
861	personal.getContent().edit(mapping=dp)
862	clearance.getContent().edit(mapping=dc)
863	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
864	#
865	# Study Course
866	#
867	student_obj.invokeFactory('StudentStudyCourse','study_course')
868	studycourse = student_obj.study_course
869	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
870	dsc = {}
871	dsc['study_course'] = certcode
872	dsc['current_level'] = level
873	dsc['current_verdict'] = verdict
874	dsc['current_mode'] = em
875	dsc['current_session'] = '05'
876	studycourse.getContent().edit(mapping=dsc)
877	#
878	# Level
879	#
880	## l = getattr(studycourse,level,None)
881	## if l is None:
882	## studycourse.invokeFactory('StudentStudyLevel', level)
883	## l = getattr(studycourse, level)
884	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
885	## l.getContent().edit(mapping={'Title': "Level %s" % level})
886	###)
887
888	def init_timing(self): ###(
889	if self.with_timing:
890	if not hasattr(self,'_v_step_times'):
891	self._v_timer_count = 0
892	self._v_total = 0
893	self._v_step_times = {}
894	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
895	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
896	self.timer_step = 0
897	self.total_time = 0
898	self.elapse = time.time()
899	self.i_elapse = time.time()
900	self.c_elapse = time.clock()
901
902	###)
903
904	def do_timing(self): ###(
905	if self.with_timing:
906	try:
907	raise 'dummy'
908	except:
909	frame = sys.exc_traceback.tb_frame.f_back
910	locals = frame.f_locals
911	globals = frame.f_globals
912	functionname = frame.f_code.co_name
913	filename = os.path.basename(frame.f_code.co_filename)
914	lineno = frame.f_lineno
915	mod_line = "%(functionname)s:%(lineno)s" % vars()
916	i_time = time.time() - self.i_elapse
917	td = {}
918	if self._v_step_times.has_key(mod_line):
919	a_time = self._v_step_times[mod_line]['a_time'] + i_time
920	td['a_time'] = a_time
921	else:
922	td['a_time'] = i_time
923	td['i_time'] = i_time
924	self._v_step_times[mod_line] = td
925	self.i_time = i_time
926	self.total_time += i_time
927	self.timer_step +=1
928	self.i_elapse = time.time()
929
930	###)
931
932	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
933	def print_timing(self):
934	if self.with_timing:
935	l = []
936	timer_count = self._v_timer_count + 1
937	mod_lines = self._v_step_times.keys()
938	mod_lines.sort(cmp,reverse=0)
939	for mod_line in mod_lines:
940	td = self._v_step_times[mod_line]
941	i_time = td['i_time']
942	a_time = td['a_time']/(self._v_timer_count + 1)
943	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
944	total_time = self.total_time
945	total_avarage = self._v_total / timer_count
946	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
947	print "\r\n".join(l)
948	out = open(self._v_timer_file,'a')
949	out.write("\r\n".join(l))
950	out.close()
951
952	###)
953
954	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
955	def get_timing_data(self):
956	if self.with_timing:
957	timer_count = self._v_timer_count + 1
958	results = {}
959	for k,d in self._v_step_times.items():
960	dd = {}
961	dd['a_time'] = d['a_time']/timer_count
962	dd['i_time'] = d['i_time']
963	dd['count'] = timer_count
964	results[k] = dd
965	dd = {}
966	dd['a_time'] = self._v_total / timer_count
967	dd['i_time'] = self.total_time
968	dd['count'] = timer_count
969	results["total"] = dd
970	return results
971	###)
972
973	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
974	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
975	"create Datastructure for an admitted Student"
976	#import pdb;pdb.set_trace()
977	logger = logging.getLogger('WAeUPTool.admitOneStudent')
978	self.with_timing = with_timing
979	if not hasattr(self,"_v_certificates"):
980	self._v_certificates = self.getCertificatesDict()
981	students_folder = self.portal_url.getPortalObject().campus.students
982	if brain.screening_type in ('cest','sandwich',):
983	reg_no = "%s%s/07" % (brain.course1[:3],brain.serial)
984	else:
985	reg_no = brain.reg_no
986	res = self.students_catalog(jamb_reg_no = reg_no)
987	if res:
988	logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
989	return
990	if brain.status != "admitted":
991	logger.info('status of %s is %s' % (reg_no,brain.status))
992	return
993	pin_parts = brain.pin.split('-')
994	if pin_parts and len(pin_parts) != 3:
995	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
996	return
997	if brain.course_admitted not in self._v_certificates:
998	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
999	return
1000	if brain.sex not in (True,False):
1001	logger.info('sex of %s not available' % (reg_no))
1002	return
1003	self.init_timing()
1004	student_id = self.generateStudentId('?')
1005	students_folder.invokeFactory('Student', student_id)
1006	student_object = getattr(students_folder,student_id)
1007	self.do_timing()
1008	if pin_password:
1009	password = pin_parts[2]
1010	self.makeStudentMember(student_id,password = password)
1011	student_object.manage_setLocalRoles(student_id, ['Owner',])
1012	self.do_timing()
1013	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
1014	#
1015	# application
1016	#
1017	student_object.invokeFactory('StudentApplication','application')
1018	application = student_object.application
1019	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
1020	#self.do_timing()
1021	da = {'Title': 'Application Data'}
1022	da['jamb_reg_no'] = reg_no
1023
1024	sex = 'M'
1025	if brain.sex:
1026	sex = 'F'
1027	da['jamb_sex'] = sex
1028	da['jamb_age'] = brain.jamb_age
1029	da['app_reg_pin'] = brain.pin
1030	da['jamb_lga'] = brain.jamb_lga
1031	da['jamb_state'] = brain.jamb_state
1032	da['jamb_score'] = brain.aggregate
1033	da['app_email'] = brain.email
1034	da['app_mobile'] = brain.phone
1035
1036	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1037
1038	da['entry_session'] = entry_session
1039	da['jamb_lastname'] = brain.lastname
1040	da['jamb_middlename'] = brain.middlenames # different field names!
1041	da['jamb_firstname'] = brain.firstname
1042	da['screening_application_date'] = brain.application_date
1043	da['date_of_birth'] = brain.date_of_birth
1044	da['jamb_first_cos'] = brain.course1
1045	da['jamb_second_cos'] = brain.course2
1046	da['course3'] = brain.course3
1047	da['screening_type'] = brain.screening_type
1048	da['screening_score'] = brain.screening_score
1049	da['screening_date'] = brain.screening_date
1050	da['hq_type'] = brain.hq_type
1051	da['hq_grade'] = brain.hq_grade
1052	da['aos'] = brain.aos
1053
1054	application.getContent().edit(mapping=da)
1055	self.do_timing()
1056	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
1057	#
1058	# personal
1059	#
1060	student_object.invokeFactory('StudentPersonal','personal')
1061	personal = student_object.personal
1062	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1063	#self.do_timing()
1064	dp = {'Title': 'Personal Data'}
1065	dp['sex'] = brain.sex
1066	dp['email'] = brain.email
1067	dp['phone'] = brain.phone
1068	dp['lastname'] = brain.lastname
1069	dp['middlename'] = brain.middlenames # different field names!
1070	dp['firstname'] = brain.firstname
1071	personal.getContent().edit(mapping=dp)
1072	self.do_timing()
1073	#
1074	# clearance
1075	#
1076	student_object.invokeFactory('StudentClearance','clearance')
1077	clearance = student_object.clearance
1078	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1079	dc = {'Title': 'Clearance/Eligibility Record'}
1080	dc['lga'] = brain.lga
1081	dc['birthday'] = brain.date_of_birth
1082	clearance.getContent().edit(mapping=dc)
1083	self.do_timing()
1084	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1085	#
1086	# study Course
1087	#
1088	student_object.invokeFactory('StudentStudyCourse','study_course')
1089	studycourse = student_object.study_course
1090	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1091	#self.do_timing()
1092	dsc = {}
1093	dsc['study_course'] = brain.course_admitted
1094	dsc['current_verdict'] = ''
1095	dsc['current_mode'] = da['entry_mode']
1096	if da['entry_mode'].startswith('de'):
1097	dsc['current_level'] = '200'
1098	elif da['entry_mode'].startswith('pre'):
1099	dsc['current_level'] = '000'
1100	else:
1101	dsc['current_level'] = '100'
1102	dsc['current_session'] = entry_session
1103	studycourse.getContent().edit(mapping=dsc)
1104	self.do_timing()
1105	#
1106	# payments folder
1107	student_object.invokeFactory('PaymentsFolder','payments')
1108	payments = getattr(student_object,'payments')
1109	#self.do_timing()
1110	dpay = {}
1111	dpay['Title'] = 'Payments'
1112	payments.getContent().edit(mapping=dpay)
1113	self.portal_workflow.doActionFor(payments,'open')
1114	self.do_timing()
1115	#
1116	# passport foto
1117	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1118	brain.screening_type,
1119	brain.reg_no)
1120	images_dir = getImagesDir(student_id)
1121	#images_dir = os.path.join("%s" % images_base,student_id)
1122	letter_dir,student_dir = os.path.split(images_dir)
1123	if not os.path.exists(letter_dir):
1124	os.mkdir(letter_dir)
1125	if not os.path.exists(images_dir):
1126	os.mkdir(images_dir)
1127	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1128	if os.path.exists(app_picture):
1129	copy2(app_picture,image_name)
1130	else:
1131	logger.info('passport of %s/%s not found: %s' % (student_id,
1132	brain.reg_no,
1133	app_picture))
1134
1135	self.do_timing()
1136	self.print_timing()
1137	if with_timing:
1138	self.timer_step = 0
1139	self._v_timer_count += 1
1140	self._v_total += self.total_time
1141	return student_id
1142	###)
1143
1144	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1145	def makeStudentLevel(self,student_id):
1146	"create the StudyLevel for a returning Student"
1147	#import pdb;pdb.set_trace()
1148	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1149	students_folder = self.portal_url.getPortalObject().campus.students
1150	res = self.students_catalog(id=student_id)
1151	if res:
1152	st = res[0]
1153	course = st.course
1154	matric_no = st.matric_no
1155	level = st.level
1156	res = self.results_import(matric_no = matric_no)
1157	if res:
1158	results = res
1159	logger.info('%s creating Level %s' % (student_id,level))
1160	#
1161	# Level
1162	#
1163	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1164	studycourse = getattr(student_obj,"study_course",None)
1165	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1166	l = getattr(studycourse,level,None)
1167	if l is None:
1168	studycourse.invokeFactory('StudentStudyLevel', level)
1169	l = getattr(studycourse, level)
1170	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1171	l.getContent().edit(mapping={'Title': "Level %s" % level})
1172	###)
1173
1174	security.declarePublic('getHallInfo') ###(
1175	def getHallInfo(self,bed):
1176	"""return Hall Info"""
1177	info = {}
1178	bedsplit = bed.split('_')
1179	if len(bedsplit) == 4:
1180	hall,block,room,letter = bed.split('_')
1181	else:
1182	info['maintenance_code'] = 'None'
1183	return info
1184	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1185	if res and len(res) == 1:
1186	hall_brain = res[0]
1187	hall_doc = hall_brain.getObject().getContent()
1188	else:
1189	return info
1190	info['hall_title'] = hall_brain.Title
1191	info['maintenance_code'] = hall_doc.maintenance_code
1192	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1193	batch_doc = None
1194	for brain in res:
1195	if brain.id.startswith(info['maintenance_code']):
1196	batch_doc = brain.getObject().getContent()
1197	break
1198	if batch_doc is None:
1199	info['maintenance_fee'] = ''
1200	else:
1201	info['maintenance_fee'] = batch_doc.cost
1202	return info
1203	###)
1204
1205	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1206	def removePictureFolder(self,student_id):
1207	"""remove picture_folder by renaming it"""
1208	path = 'images'
1209	picture_path = os.path.join(i_home,path,student_id)
1210	if not os.path.exists(picture_path):
1211	return False
1212	os.rename(picture_path,picture_path + "_removed")
1213	return True
1214	###)
1215
1216	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1217	def restorePictureFolder(self,student_id):
1218	"""restore picture_folder by renaming it"""
1219	path = 'images'
1220	picture_path = os.path.join(i_home,path,student_id)
1221	if not os.path.exists(picture_path + "_removed"):
1222	return False
1223	os.rename(picture_path + "_removed",picture_path)
1224	return True
1225	###)
1226
1227	security.declarePublic('picturesExist') ###(
1228	def picturesExist(self, ids,student_id=None):
1229	"""check if pictures exist in the filesystem"""
1230	if student_id is None:
1231	student_id = self.getStudentId()
1232	if student_id is None:
1233	return False
1234	picture_path = getImagesDir(student_id)
1235	#picture_path = os.path.join(images_base,student_id)
1236	if not os.path.exists(picture_path):
1237	return False
1238	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1239	return set(ids).issubset(set(pictures))
1240	###)
1241
1242	security.declarePublic('picturesList') ###(
1243	def picturesList(self):
1244	"""check if pictures exist in the filesystem"""
1245	path = 'images'
1246	student_id = self.getStudentId()
1247	#picture_path = os.path.join(i_home,path,student_id)
1248	picture_path = getImagesDir(student_id)
1249	if not os.path.exists(picture_path):
1250	return []
1251	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1252	###)
1253
1254	security.declarePublic('showFsPicture') ###(
1255	def showFsPicture(self,path):
1256	"""return a picture from the filesystem"""
1257	#picture_path = os.path.join(i_home,path)
1258	picture_path = os.path.join(images_base,path)
1259	response = self.REQUEST.RESPONSE
1260	#import pdb;pdb.set_trace()
1261	registry = getToolByName(self, 'mimetypes_registry')
1262	mimetype = str(registry.lookupExtension(path.lower()) or
1263	registry.lookupExtension('file.bin'))
1264	if os.path.exists(picture_path):
1265	response.setHeader('Content-type',mimetype)
1266	return open(picture_path).read()
1267	picture_path = os.path.join(i_home,'import',path)
1268	if os.path.exists(picture_path):
1269	return open(picture_path).read()
1270	###)
1271
1272	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1273	def deleteAllCourses(self,department="All"):
1274	''' delete the courses'''
1275	pm = self.portal_membership
1276	member = pm.getAuthenticatedMember()
1277
1278	if str(member) not in ("henrik","joachim"):
1279	return "not possible"
1280	if department == "All":
1281	res = self.portal_catalog({'meta_type': 'Department'})
1282	if len(res) < 1:
1283	return "No Departments found"
1284
1285	deleted = []
1286	for dep in res:
1287	cf = dep.getObject().courses
1288	if cf:
1289	cf.manage_delObjects(ids=cf.objectIds())
1290	deleted.append("deleted Courses in %s" % dep.getId)
1291	return "\r".join(deleted)
1292	###)
1293
1294	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1295	def getLogfileLines(self,filename="event.log",numlines=20):
1296	"""Get last NUMLINES lines of logfile FILENAME.
1297
1298	Return last lines' of a file in the instances logfile directory as
1299	a list. The number of returned lines equals `numlines' or less. If
1300	less than `numlines' lines are available, the whole file ist
1301	returned. If the file can not be opened or some other error
1302	occurs, empty list is returend.
1303	"""
1304	result = []
1305	lines_hit = 0
1306
1307	# We only handle files in instances' log directory...
1308	logpath = os.path.join(i_home, "log")
1309	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1310	if not filename.startswith( logpath ):
1311	# Attempt to access file outside log-dir...
1312	return []
1313
1314	try:
1315	fd = file( filename, "rb" )
1316	except IOError:
1317	return []
1318	if not fd:
1319	return []
1320
1321	if os.linesep == None:
1322	linesep = '\n'
1323	else:
1324	linesep = os.linesep
1325
1326	# Try to find 'numlines' times a lineseparator, searching from end
1327	# and moving to the beginning of file...
1328	fd.seek( 0, 2) # Move to end of file...
1329	while lines_hit < numlines:
1330	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1331	# one step forward...
1332	lines_hit += 1
1333	try:
1334	fd.seek( -2, 1) # Go two bytes back from current pos...
1335	except IOError:
1336	# We cannot go back two bytes. Maybe the file is too small...
1337	break
1338	fd.seek(2,1)
1339
1340	# Read all lines from current position...
1341	result = fd.readlines()
1342	# Remove line endings...
1343	result = [x.strip() for x in result]
1344	fd.close()
1345	return result
1346	###)
1347
1348	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1349	def getCallbacksFromLog(self,filename):
1350	"""fix Online Payment Transactions from Z2.log entries"""
1351	import transaction
1352	import random
1353	from cgi import parse_qs
1354	from urlparse import urlparse
1355	#from pdb import set_trace
1356	wftool = self.portal_workflow
1357	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1358	students_folder = self.portal_url.getPortalObject().campus.students
1359	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1360	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1361	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1362	data = re.compile(s)
1363	start = True
1364	tr_count = 1
1365	total = 0
1366	#name = 'pume_results'
1367	#name = 'epaymentsuccessful_z2log2'
1368	name = filename
1369	no_import = []
1370	imported = []
1371	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1372	try:
1373	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1374	except:
1375	logger.error('Error reading %s' % name)
1376	return
1377	tas = []
1378	for line in transactions:
1379	dict = {}
1380	items = data.search(line)
1381	dict['idict'] = idict = items.groupdict()
1382	#print idict
1383	#from pdb import set_trace;set_trace()
1384	urlparsed = urlparse(idict['get'][4:])
1385	#print urlparsed
1386	path = urlparsed[2].split('/')
1387	dict['student_id'] = student_id = path[8]
1388	dict['payment_id'] = payment_id = path[10]
1389	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1390	tas.append(dict)
1391	tr_count += 1
1392	return tas
1393	###)
1394
1395	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1396	def importOnlinePaymentTransactions(self):
1397	"""load Online Payment Transactions from CSV values"""
1398	import transaction
1399	import random
1400	#from pdb import set_trace
1401	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1402	opt = self.online_payments_import
1403	students_folder = self.portal_url.getPortalObject().campus.students
1404	start = True
1405	tr_count = 1
1406	total = 0
1407	#name = 'pume_results'
1408	name = 'OnlineTransactions'
1409	no_import = []
1410	imported = []
1411	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1412	try:
1413	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1414	except:
1415	logger.error('Error reading %s.csv' % name)
1416	return
1417	for pay_transaction in transactions:
1418	if start:
1419	start = False
1420	logger.info('Start loading from %s.csv' % name)
1421	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1422	no_import.append('%s,"Error"' % s)
1423	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1424	format_error = format + ',"%(Error)s"'
1425	data = {}
1426
1427	# format of the first file sent by Tayo
1428	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1429	#data['student_id'] = student_id = pay_transaction['Payer ID']
1430	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1431	#data['response_code'] = response_code = pay_transaction['Resp Code']
1432	#data['amount'] = amount = pay_transaction['Amount']
1433
1434	# format of the second file sent by Tayo
1435	#data['datetime'] = date = 0
1436	#data['student_id'] = student_id = pay_transaction['Payer ID']
1437	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1438	#data['response_code'] = response_code = '00'
1439	#data['amount'] = amount = pay_transaction['Amount']
1440
1441	# format of the third file sent by Kehinde
1442	data['datetime'] = date = 0
1443	data['student_id'] = student_id = pay_transaction['customer_id']
1444	data['order_id'] = order_id = pay_transaction['merchant_reference']
1445	data['response_code'] = response_code = '00'
1446	data['amount'] = amount = pay_transaction['Amount']
1447
1448	dup = False
1449	if response_code == "12":
1450	continue
1451	try:
1452	opt.addRecord(**data)
1453	except ValueError:
1454	dup = True
1455	#from pdb import set_trace;set_trace()
1456	if dup:
1457	if response_code == "00":
1458	try:
1459	opt.modifyRecord(**data)
1460	except:
1461	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1462	continue
1463	else:
1464	pay_transaction['Error'] = "Duplicate order_id"
1465	no_import.append( format_error % pay_transaction)
1466	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1467	continue
1468	tr_count += 1
1469	if tr_count > 1000:
1470	if len(no_import) > 0:
1471	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1472	'\n'.join(no_import) + '\n')
1473	no_import = []
1474	em = '%d transactions committed\n' % (tr_count)
1475	transaction.commit()
1476	regs = []
1477	logger.info(em)
1478	total += tr_count
1479	tr_count = 0
1480	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1481	'\n'.join(no_import))
1482	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1483	###)
1484
1485	security.declareProtected(ModifyPortalContent,"importData")###(
1486	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1487	"""load data from CSV values"""
1488	import transaction
1489	import random
1490	students_folder = self.portal_url.getPortalObject().campus.students
1491	uploads_folder = self.portal_url.getPortalObject().campus.uploads
1492	pending_only = False
1493	pend_str = '--'
1494	elapse = time.time()
1495	#
1496	# preparations
1497	#
1498	if filename == pend_str:
1499	pending_only = True
1500	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1501	importer = eval("%sImport" % importer_name)(self)
1502	logger = importer.logger
1503	if importer.init_errors:
1504	logger.info(importer.init_errors)
1505	return importer.init_errors
1506	member = importer.member
1507	#current = importer.current
1508	import_date = importer.import_date
1509	#
1510	# not_imported
1511	#
1512	info = importer.info
1513	data_keys = importer.data_keys
1514	csv_keys = importer.csv_keys
1515	#csv_keys.extend(info.keys())
1516	headline_mapping = dict((k,k) for k in csv_keys)
1517	#
1518	# pending
1519	#
1520	pending_path = importer.pending_path
1521	pending_tmp = importer.pending_tmp
1522	pending_backup = importer.pending_backup
1523	pending_fn = importer.pending_fn
1524	imported_path = importer.imported_path
1525	imported_fn = importer.imported_fn
1526	commit_after = importer.commit_after
1527	pending = []
1528	pending_digests = []
1529	#total_added_to_pending = 0
1530	if not pending_only:
1531	pending,pending_digests = importer.makeIdLists()
1532	pending_at_start = len(pending)
1533	datafile = open(pending_tmp,"w")
1534	pending_csv_writer = csv.DictWriter(datafile,
1535	csv_keys,
1536	extrasaction='ignore')
1537	pending_csv_writer.writerow(headline_mapping)
1538	datafile.close()
1539	#
1540	# imported
1541	#
1542	if not os.path.exists(imported_path):
1543	datafile = open(imported_path,"w")
1544	imported_csv_writer = csv.DictWriter(datafile,
1545	csv_keys,
1546	extrasaction='ignore')
1547	imported_csv_writer.writerow(headline_mapping)
1548	datafile.close()
1549	start = True
1550	tr_count = 0
1551	total = 0
1552	total_added_to_imported = 0
1553	total_pending = 0
1554	import_source_done = ""
1555	if pending_only:
1556	import_source_path = pending_path
1557	else:
1558	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1559	import_source_done = "%s/import/%s.done" % (i_home,filename)
1560	if not os.path.exists(import_source_path):
1561	fn = os.path.split(import_source_path)[1]
1562	em = 'no import file %(fn)s' % vars()
1563	return em
1564	import_source_fn = os.path.split(import_source_path)[1]
1565	if not pending_only:
1566	info['imported_from'] = import_source_fn
1567	headline = csv.reader(open(import_source_path,"rb")).next()
1568	if "import_mode" not in headline:
1569	msg = 'import_mode must be in heading'
1570	return msg
1571	invalid_keys = importer.checkHeadline(headline)
1572	if invalid_keys:
1573	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1574
1575	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1576	or k.strip() in info.keys())]
1577	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1578	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1579	# if diff2schema and diff2schema != set(['id',]):
1580	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1581	# return msg
1582	#
1583	# start importing
1584	#
1585	try:
1586	reader = csv.DictReader(open(import_source_path,"rb"))
1587	except:
1588	msg = 'Error reading %s.csv' % filename
1589	logger.error(msg)
1590	return msg
1591	items = [item for item in reader]
1592	total_to_import = len(items)
1593	tti_float = float(total_to_import)
1594	if pending_only:
1595	pending_at_start = total_to_import
1596	count = 0
1597	imported = []
1598	old_commit_count = 0
1599	error_count = imported_count = 0
1600	already_in = 0
1601	for record in items:
1602	item = {}
1603	empty_value_keys = []
1604	for k,v in record.items():
1605	if k is None:
1606	continue
1607	if v:
1608	if v == EMPTY:
1609	empty_value_keys += k,
1610	v = ''
1611	item[k.strip()] = v.strip()
1612	count += 1
1613	if start:
1614	start = False
1615	adapters = [MappingStorageAdapter(importer.schema, item)]
1616	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1617	dm = DataModel(item, adapters,context=self)
1618	ds = DataStructure(data=item,datamodel=dm)
1619	error_string = ""
1620	total += 1
1621	import_mode = item.get('import_mode','')
1622	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1623	if import_method is None:
1624	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1625	elif (import_mode in importer.required_modes and
1626	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1627	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1628	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1629	else:
1630	for k in import_keys:
1631	if k not in item.keys() or k not in importer.validators.keys():
1632	continue
1633	if not importer.validators[k](ds,mode=import_mode):
1634	error_string += ' ++ '
1635	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1636	ds.getErrorMapping(k)))
1637	if error_string:
1638	error = error_string
1639	id = ''
1640	mapping = item
1641	else:
1642	temp_item = item.copy()
1643	temp_item.update(dm)
1644	results = import_method(temp_item)
1645	id = results[0]
1646	error = results[1]
1647	mapping = results[2]
1648	#
1649	# restore some values
1650	#
1651	for k in empty_value_keys:
1652	mapping[k] = EMPTY
1653	if mapping.has_key('sex'):
1654	#import pdb;pdb.set_trace()
1655	if mapping['sex'] == True:
1656	mapping['sex'] = 'F'
1657	elif mapping['sex'] == False:
1658	mapping['sex'] = 'M'
1659	if pending_only:
1660	info['imported_from'] = item['imported_from']
1661	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1662	info['error'] = error
1663	info['import_record_no'] = count + 1
1664	mapping.update(info)
1665	log_list = []
1666	if error:
1667	error_count += 1
1668	digest = makeDigest(mapping,data_keys)
1669	if digest not in pending_digests:
1670	pending_digests += digest,
1671	pending.append(mapping)
1672	if not pending_only:
1673	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1674	else:
1675	already_in += 1
1676	pass
1677	else:
1678	imported_count += 1
1679	imported += mapping,
1680	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1681	if log_list:
1682	time_till_now = time.time() - elapse
1683	percent_finished = (error_count + imported_count)/tti_float*100
1684	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1685	logger.info(' '.join(log_list))
1686	finished = count > total_to_import - 1
1687	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1688	if must_commit:
1689	old_commit_count = imported_count
1690
1691	if must_commit or finished:
1692	if len(imported):
1693	transaction.commit()
1694	datafile = open(imported_path,"a")
1695	writer = csv.DictWriter(datafile,
1696	csv_keys,
1697	extrasaction='ignore')
1698	writer.writerows(imported)
1699	datafile.close()
1700	total_added_to_imported += len(imported)
1701	imported = []
1702	if len(pending) > 0:
1703	datafile = open(pending_tmp,"a")
1704	writer = csv.DictWriter(datafile,
1705	csv_keys,
1706	extrasaction='ignore')
1707	writer.writerows(pending)
1708	datafile.close()
1709	total_pending += len(pending)
1710	#total_added_to_pending += len(pending)
1711	pending = []
1712	if not finished:
1713	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1714	logger.info(msg)
1715	elapse = time.time() - elapse
1716	if os.path.exists(pending_path):
1717	copy2(pending_path,pending_backup)
1718	copy2(pending_tmp,pending_path)
1719	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1720	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1721	if pending_only:
1722	removed_pending = pending_at_start - total_pending
1723	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1724	else:
1725	added_pending = total_pending - pending_at_start
1726	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1727	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1728	logger.info(msg)
1729	if import_source_done:
1730	copy(import_source_path,import_source_done)
1731	os.remove(import_source_path)
1732	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1733	if upload is not None:
1734	upload_doc = upload.getContent()
1735	mapping = {}
1736	#mapping['import_date'] = DateTime.DateTime()
1737	mapping['import_date'] = import_date
1738	mapping['imported_by'] = importer.imported_by
1739	mapping['import_message'] = msg
1740	upload_doc.edit(mapping = mapping)
1741	os.remove(pending_tmp)
1742	return msg
1743	###)
1744
1745	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1746	def moveImagesToFS(self,student_id="O738726"):
1747	"move the images to the filesystem"
1748	images_dir = getImagesDir(student_id)
1749	#images_dir = os.path.join("%s" % images_base,student_id)
1750	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1751	stool = getToolByName(self, 'portal_schemas')
1752	schemas = ['student_application',
1753	'student_clearance',
1754	]
1755	created = False
1756	for schema_id in schemas:
1757	schema = stool._getOb(schema_id)
1758	object = getattr(student_folder,schema_id[len('student_'):],None)
1759	if object is None:
1760	continue
1761	doc = object.getContent()
1762	for key in schema.keys():
1763	if schema[key].meta_type != "CPS Image Field":
1764	continue
1765	#import pdb;pdb.set_trace()
1766	image = getattr(doc,key,None)
1767	if not image or not hasattr(image,"data"):
1768	continue
1769	if not created:
1770	if not os.path.exists(images_dir):
1771	os.mkdir(images_dir)
1772	created = True
1773	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1774	open(filename,"wb").write(str(image.data))
1775	###)
1776
1777	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1778	def movePassportToFS(self,student_id="O738726"):
1779	"move the passports to the filesystem"
1780	images_dir = os.path.join("%s" % i_home,'passports')
1781	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1782	stool = getToolByName(self, 'portal_schemas')
1783	schemas = ['student_application',
1784	#'student_clearance',
1785	]
1786	created = False
1787	for schema_id in schemas:
1788	schema = stool._getOb(schema_id)
1789	object = getattr(student_folder,schema_id[len('student_'):],None)
1790	if object is None:
1791	continue
1792	doc = object.getContent()
1793	for key in schema.keys():
1794	if schema[key].meta_type != "CPS Image Field":
1795	continue
1796	#import pdb;pdb.set_trace()
1797	image = getattr(doc,key)
1798	if not hasattr(image,"data"):
1799	continue
1800	if not created:
1801	if not os.path.exists(images_dir):
1802	os.mkdir(images_dir)
1803	created = True
1804	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1805	open(filename,"wb").write(str(image.data))
1806	###)
1807
1808	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: