Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 3265

Last change on this file since 3265 was 3264, checked in by joachim, 17 years ago
calculate application nr for sandwich students, passport-foto still uses brain.reg_no
Property svn:keywords set to `Id`
File size: 66.7 KB

Line
1	# -- mode: python; mode: fold; --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 3264 2008-03-03 10:01:52Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46	from WAeUPImport import CourseImport,CourseResultImport
47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48	from utils import makeDigest
49	import DateTime,time
50	import logging
51	import transaction
52	import csv,re,os,sys
53	import md5
54	from shutil import copy2
55	from Products.AdvancedQuery import Eq, Between, Le,In
56
57	p_home = package_home(globals())
58	i_home = INSTANCE_HOME
59	images_base = os.path.join(i_home,"images")
60	EMPTY = 'XXX'
61
62	def getImagesDir(student_id):
63	return os.path.join("%s" % images_base,student_id[0],student_id)
64
65	def getObject(object,name):
66	if object.hasObject(name):
67	return getattr(object,name)
68	return None
69
70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71	"""WAeUP tool"""
72
73	id = 'waeup_tool'
74	meta_type = 'WAeUP Tool'
75	_actions = ()
76	security = ClassSecurityInfo()
77	security.declareObjectProtected(View)
78	manage_options = ( ActionProviderBase.manage_options
79	+ SimpleItem.manage_options
80	)
81
82	security.declareProtected(View,'re_split') ###(
83	def re_split(self,split_string,string):
84	return re.split(split_string,string)
85	###)
86
87	security.declareProtected(View,'difference') ###(
88	def difference(self,l1,l2):
89	return set(l1).difference(set(l2))
90	###)
91
92	def rwrite(self,s): ###(
93	response = self.REQUEST.RESPONSE
94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95	response.write("%s<br />\r\n" % s)
96	###)
97
98	def addtodict(self,d,key,item): ###(
99	d[key].append(item)
100	return d[key]
101	###)
102
103	def sleep(self,secs): ###(
104	"sleep"
105	import time
106	time.sleep(secs)
107	return
108
109	###)
110
111	security.declareProtected(View,'updateRoleMappingsFor') ###(
112	def updateRoleMappingsFor(self,wf_definition,ob):
113	"do so for public"
114	wf_def = getattr(self.portal_workflow,wf_definition)
115	wf_def.updateRoleMappingsFor(ob)
116	###)
117
118	security.declareProtected(View,'getStatesLgas') ###(
119	def getStatesLgas(self):
120	"""return lga info"""
121	voc = getattr(self.portal_vocabularies,'local_gov_areas')
122	states = []
123	lgas = []
124	d = {}
125	wd = {}
126	for k,v in voc.items():
127	parts = v.split(' / ')
128	if len(parts) == 1:
129	state = parts[0].lower()
130	lga = ""
131	elif len(parts) == 2:
132	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134	else:
135	continue
136	if state not in states:
137	states.append(state)
138	if lga not in lgas:
139	lgas.append(lga)
140	words = re.split('[^a-zA-Z0-9/]',k)
141	words.sort()
142	wd[k] = words
143	d[k] = v
144	mapping = {}
145	mapping['word_dict'] = wd
146	mapping['lga_dict'] = d
147	mapping['states'] = states
148	mapping['lgas'] = lgas
149	return mapping
150	###)
151
152	security.declareProtected(View,'findLga') ###(
153	def findLga(self,words,words_dict):
154	words = re.split('[^a-zA-Z0-9/]',words)
155	lga_words = []
156	for word in words:
157	if word:
158	lga_words += word.strip().lower(),
159	lga_words.sort()
160	state_lga = ''
161	while not state_lga:
162	for k,l in words_dict.items():
163	if lga_words == l:
164	state_lga = k
165	break
166	break
167	return state_lga
168	###)
169	security.declareProtected(View,'getAccessInfo') ###(
170	def getAccessInfo(self,context):
171	"return a dict with access_info"
172	logger = logging.getLogger('WAeUPTool.getAccessInfo')
173	mtool = self.portal_membership
174	member = mtool.getAuthenticatedMember()
175	member_id = str(member)
176	info = {}
177	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179	is_staff = info['is_staff'] = not is_anonymous and not is_student
180	roles = member.getRolesInContext(context)
181	is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182	"SectionManager" in roles or
183	"Manager" in roles)
184	is_allowed = info['is_allowed'] = not is_anonymous
185	requested_id = context.getStudentId()
186	student_id = None
187	if is_allowed:
188	if not is_student and requested_id:
189	student_id = requested_id
190	elif is_anonymous or member_id != requested_id:
191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192	else:
193	student_id = member_id
194	info['student_id'] = student_id
195	return info
196	###)
197
198	security.declareProtected(ModifyPortalContent,'openLog') ###(
199	def openLog(self,name):
200	"""open a log file"""
201	version = 1
202	path = "%s/log/%s_%d.log" % (i_home,name,version)
203	while os.path.exists(path):
204	version += 1
205	path = "%s/log/%s_%d.log" % (i_home,name,version)
206	log = open(path,"w")
207	return log
208	###)
209
210	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211	def bypassQueueCatalog(self,enable=True):
212	"""bypass the QueueCatalog by setting all indexes to process imediate,
213	if enable is True (default) the old settings are restored
214	"""
215
216	###)
217
218	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220	"""measure Object access Time"""
221	import random
222	if hasattr(self,'portal_catalog_real'):
223	aq_portal = self.portal_catalog_real.evalAdvancedQuery
224	else:
225	aq_portal = self.portal_catalog.evalAdvancedQuery
226	nr_pts = int(nr_pts)
227	probe = int(probe)
228	intervall = probe/10
229	objects = ("application","clearance","personal")
230	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231	#i = random.randrange(num_objects)
232	count = 0
233	found = 0
234	not_found = 0
235	t_found = 0
236	t_not_found = 0
237	time_found = time_not_found = 0.0
238	t_time_found = t_time_not_found = 0.0
239	accessed = []
240	t_min = 1000
241	t_max = 0
242	#import pdb;pdb.set_trace()
243	students = self.portal_catalog(portal_type="Student")
244	num_students = len(students)
245	if method == "d":
246	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247	res = aq_portal(query)
248	brains = {}
249	for r in res:
250	sid = r.relative_path.split('/')[-2]
251	if brains.has_key(sid):
252	brains[sid][r.portal_type] = r
253	else:
254	brains[sid] = {r.portal_type : r}
255	brains_list = brains.keys()
256	num_objects = len(brains_list)
257	else:
258	num_objects = num_students
259	print "="*40
260	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261	probe,
262	nr_pts,
263	num_objects)
264	print "nr found/not time found/not min/max"
265	elapse = time.time()
266	i_elapse = time.time()
267	c_elapse = time.clock()
268	for c in range(1,probe + 1):
269	i = random.randrange(num_objects)
270	if method in ('a','b','c'):
271	student_brain = students[i]
272	elif method == "d":
273	#import pdb;pdb.set_trace()
274	student_brain = brains[brains_list[i]]
275	if method == "c":
276	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277	res = aq_portal(query)
278	this_portal_types = [r.portal_type for r in res]
279	for i in range(nr_pts):
280	oid = objects[i]
281	if method == "a":
282	try:
283	student_path = student_brain.getPath()
284	path = "%s/%s" % (student_path,oid)
285	doc = self.unrestrictedTraverse(path).getContent()
286	found += 1
287	i_time = time.time() - i_elapse
288	time_found += i_time
289	except:
290	not_found += 1
291	i_time = time.time() - i_elapse
292	time_not_found += i_time
293	pass
294	elif method == "b":
295	try:
296	student_object = student_brain.getObject()
297	doc = getattr(student_object,oid).getContent()
298	found += 1
299	i_time = time.time() - i_elapse
300	time_found += i_time
301	except:
302	i_time = time.time() - i_elapse
303	time_not_found += i_time
304	not_found += 1
305	pass
306	elif method == "c":
307	if portal_types[i] in this_portal_types:
308	found += 1
309	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310	i_time = time.time() - i_elapse
311	time_found += i_time
312	else:
313	not_found += 1
314	i_time = time.time() - i_elapse
315	time_not_found += i_time
316	elif method == "d":
317	if student_brain.has_key(portal_types[i]):
318	found += 1
319	doc = student_brain[portal_types[i]].getObject().getContent()
320	i_time = time.time() - i_elapse
321	time_found += i_time
322	else:
323	not_found += 1
324	i_time = time.time() - i_elapse
325	time_not_found += i_time
326	i_elapse = time.time()
327	if c and (c % intervall == 0):
328	#i_time = time.time() - i_elapse
329	t_per = 0.0
330	if found:
331	t_per = time_found/found
332	if t_per > t_max:
333	t_max = t_per
334	if t_per > 0.0 and t_per < t_min:
335	t_min = t_per
336	itf = 0.0
337	if found:
338	itf = time_found/found
339	itnf = 0.0
340	if not_found :
341	itnf = time_not_found / not_found
342	interval_time = time_found + time_not_found
343	s = "%(c)d: %(found)d/%(not_found)d " % vars()
344	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
346	print s
347	t_found += found
348	t_not_found += not_found
349	t_time_found += time_found
350	t_time_not_found += time_not_found
351	time_found = time_not_found = 0.0
352	found = not_found = 0
353	# t_found += found
354	# t_not_found += not_found
355	elapse = time.time() - elapse
356	itf = 0.0
357	if t_found:
358	itf = t_time_found/t_found
359	itnf = 0.0
360	if t_not_found:
361	itnf = t_time_not_found / t_not_found
362	#c_elapse = time.clock() - c_elapse
363	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
366	print "-"*40
367	print s
368	rel_found = float(t_found)/probe
369	rel_not_found = float(t_not_found)/probe
370	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
371	print estimated_total_time
372	###)
373
374	security.declareProtected(ModifyPortalContent,'writeLog') ###(
375	def writeLog(self,logfile,s):
376	"""write to the log file"""
377	logfile.write(s)
378
379	###)
380
381	def generateStudentId(self,letter): ###(
382	import random
383	r = random
384	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385	if letter == '?':
386	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387	sid = "%c%d" % (letter,r.randint(99999,1000000))
388	students = self.portal_url.getPortalObject().campus.students
389	## while hasattr(students, sid):
390	## sid = "%c%d" % (letter,r.randint(99999,1000000))
391	while self.students_catalog(id = sid):
392	sid = "%c%d" % (letter,r.randint(99999,1000000))
393	return sid
394	###)
395
396	def generatePassword(self,s=None): ###(
397	import random
398	r = random
399	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400	if s is None:
401	s = 'abcdefghklmnpqrstuvwxy23456789'
402	pw = ''
403	while len(pw) < 6:
404	pw += r.choice(s)
405	return pw
406	###)
407
408	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409	def dumpSchoolfeePayments(self):
410	"dump paid schoolfees"
411	mtool = self.portal_membership
412	member = mtool.getAuthenticatedMember()
413	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414	aq_student = self.students_catalog.evalAdvancedQuery
415	query = In('review_state',('schoolfee_paid',
416	'courses_registered',
417	'courses_validated',
418	))
419	res = aq_student(query)
420	#import pdb;pdb.set_trace()
421	l = []
422	logger.info("start for %d" % len(res))
423	count = 1
424	log_after = 100
425	for student in res:
426	if not count % log_after:
427	logger.info("processed %d total %d" % (log_after,count))
428	count += 1
429	fee_dict =self.getSchoolFee(student)
430	fulltime = student.mode.endswith('_ft')
431	d = {}
432	d['student_id'] = student.id
433	d['name'] = student.name
434	d['amount'] = fee_dict.get(new_returning)
435	l += d,
436	csv_name = self.dumpListToCSV(l,'payments')
437	logger.info('%s dumped payments to %s' % (member,export_file))
438	###)
439
440	security.declarePublic('dumpListToCSV') ###(
441	def dumpListToCSV(self,l,filename,fields=None):
442	"""dump a list of dicts to a CSV file"""
443	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445	if fields is None:
446	fields = l[0].keys()
447	headline = ','.join(fields)
448	out = open(export_file,"wb")
449	out.write(headline +'\n')
450	out.close()
451	out = open(export_file,"a")
452	csv_writer = csv.DictWriter(out,fields,)
453	csv_writer.writerows(l)
454	return export_file
455	###)
456
457	security.declareProtected(ManagePortal, 'listMembers') ###(
458	def listMembers(self):
459	"list all members"
460	mtool = self.portal_membership
461	member = mtool.getAuthenticatedMember()
462	logger = logging.getLogger('WAeUPTool.listMembers')
463	if str(member) not in ('admin','joachim'):
464	logger.info('%s tried to list members' % (member))
465	return None
466	members = self.portal_directories.members
467	all = members.listEntryIdsAndTitles()
468	l = []
469	for user_id,name in all:
470	d = {}
471	d['user_id'] = user_id
472	d['name'] = name
473	d['pw'] = getattr(getattr(members,user_id),'password')
474	d['email'] = getattr(getattr(members,user_id),'email')
475	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477	l += d,
478	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480	logger.info('%s dumped memberlist to %s' % (member,export_file))
481	fields = l[0].keys()
482	headline = ','.join(fields)
483	out = open(export_file,"wb")
484	out.write(headline +'\n')
485	out.close()
486	out = open(export_file,"a")
487	csv_writer = csv.DictWriter(out,fields,)
488	csv_writer.writerows(l)
489	###)
490
491	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
492	def removeDeletedDocIds(self, max=1000):
493	"""
494	remove deleted docids from repository commit after max
495	"""
496	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
497	repository = getToolByName(self, 'portal_repository')
498	pxtool = getToolByName(self, 'portal_proxies')
499	logger.info('start')
500	pxtool_infos = pxtool.getRevisionsUsed()
501	logger.info('found %d used revisions ' % (len(pxtool_infos)))
502
503	nb_revs = 0
504	docids_d = {} # all docids
505	unused_docids_d = {} # all docids that are unused
506	ids_unused_revs_docids = [] # ids for revs of unused docids
507	unused_ids = [] # ids for unused revs
508	total = 0
509	idlist = repository.objectIds()
510	to_delete = 0
511	found = False
512	for id in idlist:
513	docid, rev = repository._splitId(id)
514	if docid is None:
515	logger.info("invalid doc_id %s" % docid)
516	continue
517	nb_revs += 1
518	if not pxtool_infos.has_key(docid):
519	found = True
520	to_delete += 1
521	unused_ids.append(id)
522	elif not pxtool_infos[docid].has_key(rev):
523	found = True
524	to_delete += 1
525	unused_ids.append(id)
526	if found and not to_delete % max:
527	found = False
528	#import pdb;pdb.set_trace()
529	repository.manage_delObjects(unused_ids)
530	transaction.commit()
531	logger.info('removed %d total %d unused docids ' % (max,to_delete))
532	else:
533	if unused_ids:
534	repository.manage_delObjects(unused_ids)
535	transaction.commit()
536	logger.info('finished removing %d unused docids ' % (to_delete))
537
538
539	###)
540
541	security.declareProtected(View,'getCredential') ###(
542	def getCredential(self,student_id):
543	student_entry = getattr(self.portal_directories.students,student_id,None)
544	if not self.isStaff():
545	mtool = self.portal_membership
546	member = mtool.getAuthenticatedMember()
547	logger = logging.getLogger('WAeUPTool.getCredential')
548	logger.info('%s tried to access password of %s' % (member,student_id))
549	return None
550	if student_entry is None:
551	return None
552	return getattr(student_entry,"password","not set")
553	###)
554
555	security.declarePublic('checkPassword') ###(
556	def checkPassword(self,student_id,password):
557	student_entry = getattr(self.portal_directories.students,student_id,None)
558	if student_entry is None:
559	return False
560	return getattr(student_entry,"password","not set") == password
561	###)
562
563	security.declarePublic('checkGenericPassword') ###(
564	def checkGenericPassword(self,member_id):
565	member_entry = getattr(self.portal_directories.members,member_id,None)
566	if member_entry is None:
567	return False
568	ltool = getToolByName(self, 'portal_layouts')
569	unsecure_words = ltool._getOb('members')['w__password'].check_words
570	password = getattr(member_entry,"password","not set")
571	is_unsecure = password in unsecure_words
572	if is_unsecure:
573	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
574	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
575	return is_unsecure
576	###)
577
578	security.declareProtected(ModifyPortalContent,'editPassword') ###(
579	def editPassword(self,student_id,password):
580	"edit a student password"
581	student_entry = getattr(self.portal_directories.students,student_id,None)
582	if student_entry is None:
583	return
584	setattr(student_entry,'password',password)
585	###)
586
587	security.declareProtected(ModifyPortalContent,'doCommit') ###(
588	def doCommit(self,logger=None):
589	"commit some transactions"
590	transaction.commit()
591	###)
592
593	security.declarePublic('loadStudentFoto') ###(
594	def loadStudentFoto(self,student,filename,folder):
595	"return a student passport picture"
596	#import pdb;pdb.set_trace()
597	picture ="%s/import/%s/%s" % (i_home,folder,filename)
598	student_id = student.getId()
599	images_dir = getImagesDir(student_id)
600	if not os.path.exists(images_dir):
601	os.mkdir(images_dir)
602	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
603	for extension in ('.jpg','.JPG'):
604	fullname = "%(picture)s%(extension)s" % vars()
605	if os.path.exists(fullname):
606	copy2(fullname,image_name)
607	return "successfully copied passport picture"
608	return "passport picture not found: %s.jpg or .JPG" % picture
609	###)
610
611	def old____loadStudentFoto(self,student,filename,folder): ###(
612	"return a student passport picture"
613	app = student.application
614	app_doc = app.getContent()
615	#clear = student.clearance
616	#clear_doc = clear.getContent()
617	#matric_no = clear_doc.matric_no.upper()
618	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
619	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
620	#import pdb;pdb.set_trace()
621	if os.path.exists(picture1):
622	file = open(picture1)
623	elif os.path.exists(picture2):
624	file = open(picture2)
625	else:
626	return "passport picture not found %s" % picture1
627	reopened = False
628	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
629	self.portal_workflow.doActionFor(app,'open')
630	reopened = True
631	outfile = file.read()
632	app_doc.manage_addFile('passport',
633	file=outfile,
634	title="%s.jpg" % filename)
635	if reopened:
636	self.portal_workflow.doActionFor(app,'close')
637	return "successfully loaded passport picture"
638	###)
639
640	security.declareProtected(ModifyPortalContent,'createOne') ###(
641	def createOne(self,students_folder,student_brain,letter,commit=False):
642	sid = self.waeup_tool.generateStudentId(letter)
643	students_folder.invokeFactory('Student', sid)
644	student = getattr(students_folder,sid)
645	self.portal_workflow.doActionFor(student,'return')
646	student.manage_setLocalRoles(sid, ['Owner',])
647	matric_no = student_brain.matric_no
648	jamb_reg_no = student_brain.Entryregno
649	self.students_catalog.addRecord(id = sid,
650	matric_no = matric_no,
651	jamb_reg_no = jamb_reg_no,
652	sex = student_brain.Sex == "F",
653	name = "%s %s %s" % (student_brain.Firstname,
654	student_brain.Middlename,
655	student_brain.Lastname)
656	)
657	if commit:
658	transaction.commit()
659	return sid,jamb_reg_no
660	###)
661
662	security.declareProtected(ModifyPortalContent,'addStudent') ###(
663	def addStudent(self,dict):
664	students_folder = self.portal_url.getPortalObject().campus.students
665	sid = self.waeup_tool.generateStudentId('?')
666	students_folder.invokeFactory('Student', sid)
667	student_obj = getattr(students_folder,sid)
668	f2t = self.field2types_student
669	#from pdb import set_trace; set_trace()
670	d = {}
671	#d['jamb_sex'] = 'M'
672	#if dict.get('sex'):
673	# d['jamb_sex'] = 'F'
674
675	entry_session = dict.get('entry_session')
676	if entry_session == self.getSessionId()[0]:
677	wfaction = 'admit'
678	wft = 'wf_transition_admit'
679	password = None
680	else:
681	wfaction = 'return'
682	wft = 'wf_transition_return'
683	password = self.generatePassword()
684	self.makeStudentMember(sid,password)
685
686	for pt in f2t.keys():
687	student_obj.invokeFactory(pt,f2t[pt]['id'])
688	sub_obj = getattr(student_obj,f2t[pt]['id'])
689	sub_doc = sub_obj.getContent()
690	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
691	d['Title'] = f2t[pt]['title']
692	for field in f2t[pt]['fields']:
693	d[field] = dict.get(field,'')
694	sub_doc.edit(mapping = d)
695	new_state = f2t[pt][wft]
696	if new_state != "remain":
697	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
698	self.portal_workflow.doActionFor(student_obj,wfaction)
699	student_obj.manage_setLocalRoles(sid, ['Owner',])
700	return sid,password
701	###)
702
703	security.declarePublic('getCertificateBrain') ###(
704	def getCertificateBrain(self,cert_id):
705	"do it"
706	res = ZCatalog.searchResults(self.portal_catalog_real,
707	{'portal_type':"Certificate",
708	'id': cert_id})
709	if res:
710	return res[0]
711	return None
712	###)
713
714	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
715	def get_csv_filenames(self):
716	"do it"
717	files = [file for file in os.listdir("%s/import/" % (i_home))
718	if file.endswith('.csv') and (file.find('imported') == -1 and
719	file.find('pending') == -1)]
720	return files
721	###)
722
723	security.declarePublic('findStudentByMatricelNo') ###(
724	def findStudentByMatricelNo(self,matric_no):
725	"do it"
726	res = ZCatalog.searchResults(self.portal_catalog_real,
727	{'portal_type':"StudentClearance",
728	'SearchableText': matric_no})
729	if res:
730	return res[0]
731	return None
732	###)
733
734	security.declarePublic('makeStudentMember') ###(
735	def makeStudentMember(self,sid,password='uNsEt'):
736	"""make the student a member"""
737	membership = self.portal_membership
738	membership.addMember(sid,
739	password ,
740	roles=('Member',
741	'Student',
742	),
743	domains='',
744	properties = {'memberareaCreationFlag': False,
745	'homeless': True},)
746	member = membership.getMemberById(sid)
747	self.portal_registration.afterAdd(member, sid, password, None)
748	#self.manage_setLocalRoles(sid, ['Owner',])
749	###)
750
751	security.declareProtected(View,'makeStudentData') ###(
752	def makeStudentData(self,student_id,email=None,phone_nr=None):
753	"create Datastructure for a returning Student"
754	#import pdb;pdb.set_trace()
755	logger = logging.getLogger('WAeUPTool.makeStudentData')
756	students_folder = self.portal_url.getPortalObject().campus.students
757	#res = self.students_catalog(id=student_id)
758	#if res:
759	# st = res[0]
760	#res = self.returning_import(matric_no = st.matric_no)
761	res = self.returning_import(id = student_id)
762	if res:
763	student = res[0]
764	else:
765	logger.info('Id %s not found in returning_import' % student_id)
766	return
767	logger.info('%s creates data structure' % student_id)
768	s_results = self.results_import(matric_no = student.matric_no)
769	if s_results:
770	lnr = self.getLevelFromResultsCosCode(s_results)
771	level = "%d00" % lnr
772	verdict,eligible = self.getVerdict(s_results[0].Verdict)
773	#if eligible:
774	# level = "%d00" % (lnr + 1)
775	else:
776	logger.info('matric_no %s not found in results_import' % student.matric_no)
777	level = ''
778	verdict = ''
779	#student should not be allowed to perform this transition
780	#wftool = self.portal_workflow
781	#wftool.doActionFor(student,'return')
782	certcode_org = student.Coursemajorcode
783	certcode = makeCertificateCode(certcode_org)
784	certificate_brain = self.getCertificateBrain(certcode)
785	if not certificate_brain:
786	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
787	logger.info(em)
788	matric_no = student.matric_no
789	sid = student_id
790	student_obj = getattr(students_folder,sid)
791	if not getattr(student_obj,'application'):
792	student_obj.invokeFactory('StudentApplication','application')
793	application = student_obj.application
794	self.portal_workflow.doActionFor(application,'open',dest_container=application)
795	da = {'Title': 'Application Data'}
796	student_obj.invokeFactory('StudentPersonal','personal')
797	da['jamb_reg_no'] = student.Entryregno
798	em = self.getEntryMode(student.Entryregno)
799	da['entry_mode'] = em
800	personal = student_obj.personal
801	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
802	dp = {'Title': 'Personal Data'}
803	student_obj.invokeFactory('StudentClearance','clearance')
804	clearance = student_obj.clearance
805	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
806	dc = {'Title': 'Clearance/Eligibility Record'}
807	dc['matric_no'] = matric_no
808	state = student.State
809	lga = student.LGA
810	if state and lga:
811	lga = state + ' / ' + lga
812	else:
813	lga = "None"
814	da['jamb_lga'] = dc['lga'] = lga
815	da['app_email'] = dp['email'] = email
816	da['app_mobile'] = dp['phone'] = phone_nr
817	dp['firstname'] = student.Firstname
818	dp['middlename'] = student.Middlename
819	dp['lastname'] = student.Lastname
820	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
821	da['jamb_sex'] = student.Sex
822	dp['sex'] = student.Sex == 'F'
823	dp['perm_address'] = student.Permanent_Address
824	application.getContent().edit(mapping=da)
825	self.portal_workflow.doActionFor(application,'close',dest_container=application)
826	personal.getContent().edit(mapping=dp)
827	clearance.getContent().edit(mapping=dc)
828	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
829	#
830	# Study Course
831	#
832	student_obj.invokeFactory('StudentStudyCourse','study_course')
833	studycourse = student_obj.study_course
834	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
835	dsc = {}
836	dsc['study_course'] = certcode
837	dsc['current_level'] = level
838	dsc['current_verdict'] = verdict
839	dsc['current_mode'] = em
840	dsc['current_session'] = '05'
841	studycourse.getContent().edit(mapping=dsc)
842	#
843	# Level
844	#
845	## l = getattr(studycourse,level,None)
846	## if l is None:
847	## studycourse.invokeFactory('StudentStudyLevel', level)
848	## l = getattr(studycourse, level)
849	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
850	## l.getContent().edit(mapping={'Title': "Level %s" % level})
851	###)
852
853	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
854	def admitOneStudent(self,brain,entry_session,pin_password):
855	"create Datastructure for an admitted Student"
856	#import pdb;pdb.set_trace()
857	if not hasattr(self,"_v_certificates"):
858	self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
859	students_folder = self.portal_url.getPortalObject().campus.students
860	logger = logging.getLogger('WAeUPTool.admitOneStudent')
861	if brain.screening_type in ('cest','sandwich',):
862	reg_no = "%s%s/07" % (brain.course1,brain.serial)
863	else:
864	reg_no = brain.reg_no
865	res = self.students_catalog(jamb_reg_no = reg_no)
866	if res:
867	logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
868	return
869	if brain.status != "admitted":
870	logger.info('status of %s is %s' % (reg_no,brain.status))
871	return
872	pin_parts = brain.pin.split('-')
873	if pin_parts and len(pin_parts) != 3:
874	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
875	return
876	if brain.course_admitted not in self._v_certificates:
877	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
878	return
879	if brain.sex not in (True,False):
880	logger.info('sex of %s not available' % (reg_no))
881	return
882	student_id = self.generateStudentId('?')
883	students_folder.invokeFactory('Student', student_id)
884	student_object = getattr(students_folder,student_id)
885	if pin_password:
886	password = pin_parts[2]
887	self.makeStudentMember(student_id,password = password)
888	student_object.manage_setLocalRoles(student_id, ['Owner',])
889	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
890	#
891	# application
892	#
893	student_object.invokeFactory('StudentApplication','application')
894	application = student_object.application
895	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
896	da = {'Title': 'Application Data'}
897	da['jamb_reg_no'] = reg_no
898
899	sex = 'M'
900	if brain.sex:
901	sex = 'F'
902	da['jamb_sex'] = sex
903	da['jamb_age'] = brain.jamb_age
904	#da['app_ac_pin'] = brain.pin
905	da['jamb_lga'] = brain.jamb_lga
906	da['jamb_state'] = brain.jamb_state
907	da['jamb_score'] = brain.aggregate
908	da['app_email'] = brain.email
909	da['app_mobile'] = brain.phone
910	if brain.entry_mode:
911	da['entry_mode'] = brain.entry_mode
912	elif brain.screening_type == 'pume':
913	da['entry_mode'] = 'ume_ft'
914	elif brain.screening_type == 'pde':
915	da['entry_mode'] = 'de_ft'
916	elif brain.screening_type == 'pce':
917	da['entry_mode'] = 'pce'
918	elif brain.screening_type == 'prence':
919	da['entry_mode'] = 'prence'
920	else:
921	da['entry_mode'] = ''
922	da['entry_session'] = entry_session
923	da['jamb_lastname'] = brain.lastname
924	da['jamb_middlename'] = brain.middlenames # different field names!
925	da['jamb_firstname'] = brain.firstname
926	da['screening_application_date'] = brain.application_date
927	da['date_of_birth'] = brain.date_of_birth
928	da['jamb_first_cos'] = brain.course1
929	da['jamb_second_cos'] = brain.course2
930	da['course3'] = brain.course3
931	da['screening_type'] = brain.screening_type
932	da['screening_score'] = brain.screening_score
933	da['screening_date'] = brain.screening_date
934	da['hq_type'] = brain.hq_type
935	da['hq_grade'] = brain.hq_grade
936	da['aos'] = brain.aos
937
938	application.getContent().edit(mapping=da)
939	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
940	#
941	# personal
942	#
943	student_object.invokeFactory('StudentPersonal','personal')
944	personal = student_object.personal
945	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
946	dp = {'Title': 'Personal Data'}
947	dp['sex'] = brain.sex
948	dp['email'] = brain.email
949	dp['phone'] = brain.phone
950	dp['lastname'] = brain.lastname
951	dp['middlename'] = brain.middlenames # different field names!
952	dp['firstname'] = brain.firstname
953	personal.getContent().edit(mapping=dp)
954	#
955	# clearance
956	#
957	student_object.invokeFactory('StudentClearance','clearance')
958	clearance = student_object.clearance
959	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
960	dc = {'Title': 'Clearance/Eligibility Record'}
961	dc['lga'] = brain.lga
962	dc['birthday'] = brain.date_of_birth
963	clearance.getContent().edit(mapping=dc)
964	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
965	#
966	# study Course
967	#
968	student_object.invokeFactory('StudentStudyCourse','study_course')
969	studycourse = student_object.study_course
970	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
971	dsc = {}
972	dsc['study_course'] = brain.course_admitted
973	dsc['current_verdict'] = ''
974	dsc['current_mode'] = da['entry_mode']
975	if da['entry_mode'].startswith('de'):
976	dsc['current_level'] = '200'
977	elif da['entry_mode'].startswith('pre'):
978	dsc['current_level'] = '000'
979	else:
980	dsc['current_level'] = '100'
981	dsc['current_session'] = entry_session
982	studycourse.getContent().edit(mapping=dsc)
983	#
984	# payments folder
985	student_object.invokeFactory('PaymentsFolder','payments')
986	payments = getattr(student_object,'payments')
987	dpay = {}
988	dpay['Title'] = 'Payments'
989	payments.getContent().edit(mapping=dpay)
990	self.portal_workflow.doActionFor(payments,'open')
991	#
992	# passport foto
993	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
994	brain.screening_type,
995	brain.reg_no)
996	images_dir = getImagesDir(student_id)
997	#images_dir = os.path.join("%s" % images_base,student_id)
998	letter_dir,student_dir = os.path.split(images_dir)
999	if not os.path.exists(letter_dir):
1000	os.mkdir(letter_dir)
1001	if not os.path.exists(images_dir):
1002	os.mkdir(images_dir)
1003	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1004	if os.path.exists(app_picture):
1005	copy2(app_picture,image_name)
1006	else:
1007	logger.info('passport of %s/%s not found: %s' % (student_id,
1008	brain.reg_no,
1009	app_picture))
1010
1011	return student_id
1012	###)
1013
1014	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1015	def makeStudentLevel(self,student_id):
1016	"create the StudyLevel for a returning Student"
1017	#import pdb;pdb.set_trace()
1018	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1019	students_folder = self.portal_url.getPortalObject().campus.students
1020	res = self.students_catalog(id=student_id)
1021	if res:
1022	st = res[0]
1023	course = st.course
1024	matric_no = st.matric_no
1025	level = st.level
1026	res = self.results_import(matric_no = matric_no)
1027	if res:
1028	results = res
1029	logger.info('%s creating Level %s' % (student_id,level))
1030	#
1031	# Level
1032	#
1033	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1034	studycourse = getattr(student_obj,"study_course",None)
1035	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1036	l = getattr(studycourse,level,None)
1037	if l is None:
1038	studycourse.invokeFactory('StudentStudyLevel', level)
1039	l = getattr(studycourse, level)
1040	self.portal_workflow.doActionFor(l,'open',dest_container=l)
1041	l.getContent().edit(mapping={'Title': "Level %s" % level})
1042	###)
1043
1044	security.declarePublic('getHallInfo') ###(
1045	def getHallInfo(self,bed):
1046	"""return Hall Info"""
1047	info = {}
1048	hall,block,room,letter = bed.split('_')
1049	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1050	if res and len(res) == 1:
1051	hall_brain = res[0]
1052	hall_doc = hall_brain.getObject().getContent()
1053	else:
1054	return info
1055	info['hall_title'] = hall_brain.Title
1056	info['maintenance_code'] = hall_doc.maintenance_code
1057	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1058	batch_doc = None
1059	for brain in res:
1060	if brain.id.startswith(info['maintenance_code']):
1061	batch_doc = brain.getObject().getContent()
1062	break
1063	if batch_doc is None:
1064	info['maintenance_fee'] = ''
1065	else:
1066	info['maintenance_fee'] = batch_doc.cost
1067	return info
1068	###)
1069
1070	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1071	def removePictureFolder(self,student_id):
1072	"""remove picture_folder by renaming it"""
1073	path = 'images'
1074	picture_path = os.path.join(i_home,path,student_id)
1075	if not os.path.exists(picture_path):
1076	return False
1077	os.rename(picture_path,picture_path + "_removed")
1078	return True
1079	###)
1080
1081	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1082	def restorePictureFolder(self,student_id):
1083	"""restore picture_folder by renaming it"""
1084	path = 'images'
1085	picture_path = os.path.join(i_home,path,student_id)
1086	if not os.path.exists(picture_path + "_removed"):
1087	return False
1088	os.rename(picture_path + "_removed",picture_path)
1089	return True
1090	###)
1091
1092	security.declarePublic('picturesExist') ###(
1093	def picturesExist(self, ids,student_id=None):
1094	"""check if pictures exist in the filesystem"""
1095	if student_id is None:
1096	student_id = self.getStudentId()
1097	if student_id is None:
1098	return False
1099	picture_path = getImagesDir(student_id)
1100	#picture_path = os.path.join(images_base,student_id)
1101	if not os.path.exists(picture_path):
1102	return False
1103	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1104	return set(ids).issubset(set(pictures))
1105	###)
1106
1107	security.declarePublic('picturesList') ###(
1108	def picturesList(self):
1109	"""check if pictures exist in the filesystem"""
1110	path = 'images'
1111	student_id = self.getStudentId()
1112	#picture_path = os.path.join(i_home,path,student_id)
1113	picture_path = getImagesDir(student_id)
1114	if not os.path.exists(picture_path):
1115	return []
1116	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1117	###)
1118
1119	security.declarePublic('showFsPicture') ###(
1120	def showFsPicture(self,path):
1121	"""return a picture from the filesystem"""
1122	#picture_path = os.path.join(i_home,path)
1123	picture_path = os.path.join(images_base,path)
1124	response = self.REQUEST.RESPONSE
1125	#import pdb;pdb.set_trace()
1126	registry = getToolByName(self, 'mimetypes_registry')
1127	mimetype = str(registry.lookupExtension(path.lower()) or
1128	registry.lookupExtension('file.bin'))
1129	if os.path.exists(picture_path):
1130	response.setHeader('Content-type',mimetype)
1131	return open(picture_path).read()
1132	picture_path = os.path.join(i_home,'import',path)
1133	if os.path.exists(picture_path):
1134	return open(picture_path).read()
1135	###)
1136
1137	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1138	def deleteAllCourses(self,department="All"):
1139	''' delete the courses'''
1140	pm = self.portal_membership
1141	member = pm.getAuthenticatedMember()
1142
1143	if str(member) not in ("henrik","joachim"):
1144	return "not possible"
1145	if department == "All":
1146	res = self.portal_catalog({'meta_type': 'Department'})
1147	if len(res) < 1:
1148	return "No Departments found"
1149
1150	deleted = []
1151	for dep in res:
1152	cf = dep.getObject().courses
1153	if cf:
1154	cf.manage_delObjects(ids=cf.objectIds())
1155	deleted.append("deleted Courses in %s" % dep.getId)
1156	return "\r".join(deleted)
1157	###)
1158
1159	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1160	def getLogfileLines(self,filename="event.log",numlines=20):
1161	"""Get last NUMLINES lines of logfile FILENAME.
1162
1163	Return last lines' of a file in the instances logfile directory as
1164	a list. The number of returned lines equals `numlines' or less. If
1165	less than `numlines' lines are available, the whole file ist
1166	returned. If the file can not be opened or some other error
1167	occurs, empty list is returend.
1168	"""
1169	result = []
1170	lines_hit = 0
1171
1172	# We only handle files in instances' log directory...
1173	logpath = os.path.join(i_home, "log")
1174	filename = str(os.path.abspath( os.path.join( logpath, filename )))
1175	if not filename.startswith( logpath ):
1176	# Attempt to access file outside log-dir...
1177	return []
1178
1179	try:
1180	fd = file( filename, "rb" )
1181	except IOError:
1182	return []
1183	if not fd:
1184	return []
1185
1186	if os.linesep == None:
1187	linesep = '\n'
1188	else:
1189	linesep = os.linesep
1190
1191	# Try to find 'numlines' times a lineseparator, searching from end
1192	# and moving to the beginning of file...
1193	fd.seek( 0, 2) # Move to end of file...
1194	while lines_hit < numlines:
1195	if fd.read(1) == linesep[-1]: # This moves filedescriptor
1196	# one step forward...
1197	lines_hit += 1
1198	try:
1199	fd.seek( -2, 1) # Go two bytes back from current pos...
1200	except IOError:
1201	# We cannot go back two bytes. Maybe the file is too small...
1202	break
1203	fd.seek(2,1)
1204
1205	# Read all lines from current position...
1206	result = fd.readlines()
1207	# Remove line endings...
1208	result = [x.strip() for x in result]
1209	fd.close()
1210	return result
1211	###)
1212
1213	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1214	def getCallbacksFromLog(self,filename):
1215	"""fix Online Payment Transactions from Z2.log entries"""
1216	import transaction
1217	import random
1218	from cgi import parse_qs
1219	from urlparse import urlparse
1220	#from pdb import set_trace
1221	wftool = self.portal_workflow
1222	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1223	students_folder = self.portal_url.getPortalObject().campus.students
1224	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1225	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
1226	s += r'(?P<intersw>.)" "(?P<agent>.)"'
1227	data = re.compile(s)
1228	start = True
1229	tr_count = 1
1230	total = 0
1231	#name = 'pume_results'
1232	#name = 'epaymentsuccessful_z2log2'
1233	name = filename
1234	no_import = []
1235	imported = []
1236	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1237	try:
1238	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1239	except:
1240	logger.error('Error reading %s' % name)
1241	return
1242	tas = []
1243	for line in transactions:
1244	dict = {}
1245	items = data.search(line)
1246	dict['idict'] = idict = items.groupdict()
1247	#print idict
1248	#from pdb import set_trace;set_trace()
1249	urlparsed = urlparse(idict['get'][4:])
1250	#print urlparsed
1251	path = urlparsed[2].split('/')
1252	dict['student_id'] = student_id = path[8]
1253	dict['payment_id'] = payment_id = path[10]
1254	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1255	tas.append(dict)
1256	tr_count += 1
1257	return tas
1258	###)
1259
1260	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1261	def importOnlinePaymentTransactions(self):
1262	"""load Online Payment Transactions from CSV values"""
1263	import transaction
1264	import random
1265	#from pdb import set_trace
1266	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1267	opt = self.online_payments_import
1268	students_folder = self.portal_url.getPortalObject().campus.students
1269	start = True
1270	tr_count = 1
1271	total = 0
1272	#name = 'pume_results'
1273	name = 'OnlineTransactions'
1274	no_import = []
1275	imported = []
1276	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1277	try:
1278	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1279	except:
1280	logger.error('Error reading %s.csv' % name)
1281	return
1282	for pay_transaction in transactions:
1283	if start:
1284	start = False
1285	logger.info('Start loading from %s.csv' % name)
1286	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1287	no_import.append('%s,"Error"' % s)
1288	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1289	format_error = format + ',"%(Error)s"'
1290	data = {}
1291
1292	# format of the first file sent by Tayo
1293	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1294	#data['student_id'] = student_id = pay_transaction['Payer ID']
1295	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1296	#data['response_code'] = response_code = pay_transaction['Resp Code']
1297	#data['amount'] = amount = pay_transaction['Amount']
1298
1299	# format of the second file sent by Tayo
1300	#data['datetime'] = date = 0
1301	#data['student_id'] = student_id = pay_transaction['Payer ID']
1302	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1303	#data['response_code'] = response_code = '00'
1304	#data['amount'] = amount = pay_transaction['Amount']
1305
1306	# format of the third file sent by Kehinde
1307	data['datetime'] = date = 0
1308	data['student_id'] = student_id = pay_transaction['customer_id']
1309	data['order_id'] = order_id = pay_transaction['merchant_reference']
1310	data['response_code'] = response_code = '00'
1311	data['amount'] = amount = pay_transaction['Amount']
1312
1313	dup = False
1314	if response_code == "12":
1315	continue
1316	try:
1317	opt.addRecord(**data)
1318	except ValueError:
1319	dup = True
1320	#from pdb import set_trace;set_trace()
1321	if dup:
1322	if response_code == "00":
1323	try:
1324	opt.modifyRecord(**data)
1325	except:
1326	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1327	continue
1328	else:
1329	pay_transaction['Error'] = "Duplicate order_id"
1330	no_import.append( format_error % pay_transaction)
1331	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1332	continue
1333	tr_count += 1
1334	if tr_count > 1000:
1335	if len(no_import) > 0:
1336	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1337	'\n'.join(no_import) + '\n')
1338	no_import = []
1339	em = '%d transactions committed\n' % (tr_count)
1340	transaction.commit()
1341	regs = []
1342	logger.info(em)
1343	total += tr_count
1344	tr_count = 0
1345	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1346	'\n'.join(no_import))
1347	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1348	###)
1349
1350	security.declareProtected(ModifyPortalContent,"importData")###(
1351	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1352	"""load data from CSV values"""
1353	import transaction
1354	import random
1355	students_folder = self.portal_url.getPortalObject().campus.students
1356	pending_only = False
1357	pend_str = '--'
1358	elapse = time.time()
1359	#
1360	# preparations
1361	#
1362	if filename == pend_str:
1363	pending_only = True
1364	# mode = "create"
1365	# if edit:
1366	# if filename.endswith('_toDelete'):
1367	# mode = "delete"
1368	# else:
1369	# mode = "edit"
1370	importer_name = ''.join([part.capitalize() for part in name.split('_')])
1371	importer = eval("%sImport" % importer_name)(self)
1372	logger = importer.logger
1373	if importer.init_errors:
1374	logger.info(importer.init_errors)
1375	return importer.init_errors
1376	member = importer.member
1377	current = importer.current
1378	import_date = importer.import_date
1379	#
1380	# not_imported
1381	#
1382	info = importer.info
1383	data_keys = importer.data_keys
1384	csv_keys = importer.csv_keys
1385	#csv_keys.extend(info.keys())
1386	headline_mapping = dict((k,k) for k in csv_keys)
1387	#
1388	# pending
1389	#
1390	pending_path = importer.pending_path
1391	pending_tmp = importer.pending_tmp
1392	pending_backup = importer.pending_backup
1393	pending_fn = importer.pending_fn
1394	imported_path = importer.imported_path
1395	imported_fn = importer.imported_fn
1396	commit_after = importer.commit_after
1397	pending = []
1398	pending_digests = []
1399	#total_added_to_pending = 0
1400	if not pending_only:
1401	pending,pending_digests = importer.makeIdLists()
1402	pending_at_start = len(pending)
1403	datafile = open(pending_tmp,"w")
1404	pending_csv_writer = csv.DictWriter(datafile,
1405	csv_keys,
1406	extrasaction='ignore')
1407	pending_csv_writer.writerow(headline_mapping)
1408	datafile.close()
1409	#
1410	# imported
1411	#
1412	if not os.path.exists(imported_path):
1413	datafile = open(imported_path,"w")
1414	imported_csv_writer = csv.DictWriter(datafile,
1415	csv_keys,
1416	extrasaction='ignore')
1417	imported_csv_writer.writerow(headline_mapping)
1418	datafile.close()
1419	start = True
1420	tr_count = 0
1421	total = 0
1422	total_added_to_imported = 0
1423	total_pending = 0
1424	if pending_only:
1425	import_source_path = pending_path
1426	else:
1427	import_source_path = "%s/import/%s.csv" % (i_home,filename)
1428	if not os.path.exists(import_source_path):
1429	fn = os.path.split(import_source_path)[1]
1430	em = 'No %(fn)s' % vars()
1431	return em
1432	import_source_fn = os.path.split(import_source_path)[1]
1433	info['imported_from'] = import_source_fn
1434	attrs = csv.reader(open(import_source_path,"rb")).next()
1435	if "import_mode" not in attrs:
1436	msg = 'import_mode must be in heading'
1437	return msg
1438
1439	import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
1440	or k.strip() in info.keys())]
1441	diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1442	diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1443	if diff2schema and diff2schema != set(['id',]):
1444	msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1445	return msg
1446	#
1447	# start importing
1448	#
1449	try:
1450	reader = csv.DictReader(open(import_source_path,"rb"))
1451	except:
1452	msg = 'Error reading %s.csv' % filename
1453	logger.error(msg)
1454	return msg
1455	items = [item for item in reader]
1456	total_to_import = len(items)
1457	tti_float = float(total_to_import)
1458	if pending_only:
1459	pending_at_start = total_to_import
1460	count = 0
1461	imported = []
1462	old_commit_count = 0
1463	error_count = imported_count = 0
1464	already_in = 0
1465	for record in items:
1466	item = {}
1467	empty_value_keys = []
1468	for k,v in record.items():
1469	if v:
1470	if v == EMPTY:
1471	empty_value_keys += k,
1472	v = ''
1473	item[k.strip()] = v.strip()
1474	count += 1
1475	if start:
1476	start = False
1477	adapters = [MappingStorageAdapter(importer.schema, item)]
1478	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1479	dm = DataModel(item, adapters,context=self)
1480	ds = DataStructure(data=item,datamodel=dm)
1481	error_string = ""
1482	total += 1
1483	import_mode = item.get('import_mode','edit')
1484	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1485	if import_method is None:
1486	error_string += "no %(import_mode)s " % vars()
1487	elif (import_mode in importer.required_modes and
1488	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1489	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1490	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1491	else:
1492	for k in import_keys:
1493	if k not in item.keys() or k not in importer.validators.keys():
1494	continue
1495	if not importer.validators[k](ds,mode=import_mode):
1496	error_string += ' ++ '
1497	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1498	ds.getErrorMapping(k)))
1499	if error_string:
1500	error = error_string
1501	id = ''
1502	mapping = item
1503	else:
1504	temp_item = item.copy()
1505	temp_item.update(dm)
1506	results = import_method(temp_item)
1507	id = results[0]
1508	error = results[1]
1509	mapping = results[2]
1510	for k in empty_value_keys:
1511	mapping[k] = EMPTY
1512	if mapping.has_key('sex'):
1513	#import pdb;pdb.set_trace()
1514	if mapping['sex']:
1515	mapping['sex'] = 'F'
1516	else:
1517	mapping['sex'] = 'M'
1518	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1519	info['error'] = error
1520	mapping.update(info)
1521	log_list = []
1522	if error:
1523	error_count += 1
1524	digest = makeDigest(mapping,data_keys)
1525	if digest not in pending_digests:
1526	pending_digests += digest,
1527	pending.append(mapping)
1528	if not pending_only:
1529	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1530	else:
1531	already_in += 1
1532	pass
1533	else:
1534	imported_count += 1
1535	imported += mapping,
1536	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1537	if log_list:
1538	time_till_now = time.time() - elapse
1539	percent_finished = (error_count + imported_count)/tti_float*100
1540	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1541	logger.info(' '.join(log_list))
1542	finished = count > total_to_import - 1
1543	must_commit = False
1544	if len(imported) != old_commit_count:
1545	if not len(imported) % commit_after:
1546	must_commit = True
1547	old_commit_count = len(imported)
1548
1549	if must_commit or finished:
1550	if len(imported):
1551	transaction.commit()
1552	datafile = open(imported_path,"a")
1553	writer = csv.DictWriter(datafile,
1554	csv_keys,
1555	extrasaction='ignore')
1556	writer.writerows(imported)
1557	datafile.close()
1558	total_added_to_imported += len(imported)
1559	imported = []
1560	if len(pending) > 0:
1561	datafile = open(pending_tmp,"a")
1562	writer = csv.DictWriter(datafile,
1563	csv_keys,
1564	extrasaction='ignore')
1565	writer.writerows(pending)
1566	datafile.close()
1567	total_pending += len(pending)
1568	#total_added_to_pending += len(pending)
1569	pending = []
1570	if not finished:
1571	msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
1572	logger.info(msg)
1573	elapse = time.time() - elapse
1574	copy2(pending_tmp,pending_path)
1575	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1576	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1577	if pending_only:
1578	removed_pending = pending_at_start - total_pending
1579	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1580	else:
1581	added_pending = total_pending - pending_at_start
1582	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1583	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
1584	logger.info(msg)
1585	os.remove(pending_tmp)
1586	return msg
1587	###)
1588
1589
1590
1591	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1592	def moveImagesToFS(self,student_id="O738726"):
1593	"move the images to the filesystem"
1594	images_dir = getImagesDir(student_id)
1595	#images_dir = os.path.join("%s" % images_base,student_id)
1596	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1597	stool = getToolByName(self, 'portal_schemas')
1598	schemas = ['student_application',
1599	'student_clearance',
1600	]
1601	created = False
1602	for schema_id in schemas:
1603	schema = stool._getOb(schema_id)
1604	object = getattr(student_folder,schema_id[len('student_'):],None)
1605	if object is None:
1606	continue
1607	doc = object.getContent()
1608	for key in schema.keys():
1609	if schema[key].meta_type != "CPS Image Field":
1610	continue
1611	#import pdb;pdb.set_trace()
1612	image = getattr(doc,key,None)
1613	if not image or not hasattr(image,"data"):
1614	continue
1615	if not created:
1616	if not os.path.exists(images_dir):
1617	os.mkdir(images_dir)
1618	created = True
1619	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1620	open(filename,"wb").write(str(image.data))
1621	###)
1622
1623	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1624	def movePassportToFS(self,student_id="O738726"):
1625	"move the passports to the filesystem"
1626	images_dir = os.path.join("%s" % i_home,'passports')
1627	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1628	stool = getToolByName(self, 'portal_schemas')
1629	schemas = ['student_application',
1630	#'student_clearance',
1631	]
1632	created = False
1633	for schema_id in schemas:
1634	schema = stool._getOb(schema_id)
1635	object = getattr(student_folder,schema_id[len('student_'):],None)
1636	if object is None:
1637	continue
1638	doc = object.getContent()
1639	for key in schema.keys():
1640	if schema[key].meta_type != "CPS Image Field":
1641	continue
1642	#import pdb;pdb.set_trace()
1643	image = getattr(doc,key)
1644	if not hasattr(image,"data"):
1645	continue
1646	if not created:
1647	if not os.path.exists(images_dir):
1648	os.mkdir(images_dir)
1649	created = True
1650	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1651	open(filename,"wb").write(str(image.data))
1652	###)
1653
1654	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: