Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1900

Last change on this file since 1900 was 1900, checked in by Henrik Bettermann, 18 years ago
getSessionResults: avoid memory overflow WAeUPTool: make addStudent work
Property svn:keywords set to `Id`
File size: 41.6 KB

Line
1	3#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1900 2007-06-15 07:51:48Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app_doc = student.application.getContent()
215	#clear = student.clearance
216	#clear_doc = clear.getContent()
217	#matric_no = clear_doc.matric_no.upper()
218	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220	#import pdb;pdb.set_trace()
221	if os.path.exists(picture1):
222	file = open(picture1)
223	elif os.path.exists(picture2):
224	file = open(picture2)
225	else:
226	return "passport picture not found %s" % picture1
227
228	outfile = file.read()
229	app_doc.manage_addFile('passport',
230	file=outfile,
231	title="%s.jpg" % filename)
232	return "successfully loaded passport picture"
233	###)
234
235	security.declareProtected(ModifyPortalContent,'createOne') ###(
236	def createOne(self,students_folder,student_brain,letter,commit=False):
237	sid = self.waeup_tool.generateStudentId(letter)
238	students_folder.invokeFactory('Student', sid)
239	student = getattr(students_folder,sid)
240	self.portal_workflow.doActionFor(student,'return')
241	student.manage_setLocalRoles(sid, ['Owner',])
242	matric_no = student_brain.matric_no
243	jamb_reg_no = student_brain.Entryregno
244	self.students_catalog.addRecord(id = sid,
245	matric_no = matric_no,
246	jamb_reg_no = jamb_reg_no,
247	sex = student_brain.Sex == "F",
248	name = "%s %s %s" % (student_brain.Firstname,
249	student_brain.Middlename,
250	student_brain.Lastname)
251	)
252	if commit:
253	transaction.commit()
254	return sid,jamb_reg_no
255	###)
256
257	security.declareProtected(ModifyPortalContent,'addStudent') ###(
258	def addStudent(self,dict):
259	students_folder = self.portal_url.getPortalObject().campus.students
260	sid = self.waeup_tool.generateStudentId('?')
261	students_folder.invokeFactory('Student', sid)
262	student_obj = getattr(students_folder,sid)
263	f2t = self.student_field2types
264	#from pdb import set_trace; set_trace()
265	d = {}
266	d['jamb_sex'] = 'M'
267	if dict.get('sex'):
268	d['jamb_sex'] = 'F'
269
270	entry_session = dict.get('entry_session')
271	if entry_session == self.getSessionId()[-2:]:
272	wfaction = 'admit'
273	wft = 'wf_transition_admitted'
274	password = None
275	else:
276	wfaction = 'return'
277	wft = 'wf_transition_returning'
278	password = self.generatePassword()
279	self.makeStudentMember(sid,password)
280
281	for pt in f2t.keys():
282	student_obj.invokeFactory(pt,f2t[pt]['id'])
283	sub_obj = getattr(student_obj,f2t[pt]['id'])
284	sub_doc = sub_obj.getContent()
285	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
286	d['Title'] = f2t[pt]['title']
287	for field in f2t[pt]['fields']:
288	d[field] = dict.get(field,'')
289	sub_doc.edit(mapping = d)
290	new_state = f2t[pt][wft]
291	if new_state != "remain":
292	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
293	self.portal_workflow.doActionFor(student_obj,wfaction)
294	student_obj.manage_setLocalRoles(sid, ['Owner',])
295	return sid,password
296	###)
297
298	security.declarePublic('getCertificateBrain') ###(
299	def getCertificateBrain(self,cert_id):
300	"do it"
301	res = ZCatalog.searchResults(self.portal_catalog_real,
302	{'portal_type':"Certificate",
303	'id': cert_id})
304	if res:
305	return res[0]
306	return None
307	###)
308
309	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
310	def get_csv_filenames(self):
311	"do it"
312	files = [file for file in os.listdir("%s/import/" % (i_home))
313	if file.endswith('.csv') and file.find('imported') == -1]
314	return files
315	###)
316
317	security.declarePublic('findStudentByMatricelNo') ###(
318	def findStudentByMatricelNo(self,matric_no):
319	"do it"
320	res = ZCatalog.searchResults(self.portal_catalog_real,
321	{'portal_type':"StudentClearance",
322	'SearchableText': matric_no})
323	if res:
324	return res[0]
325	return None
326	###)
327
328	security.declarePublic('makeStudentMember') ###(
329	def makeStudentMember(self,sid,password='uNsEt'):
330	"""make the student a member"""
331	membership = self.portal_membership
332	membership.addMember(sid,
333	password ,
334	roles=('Member',
335	'Student',
336	),
337	domains='',
338	properties = {'memberareaCreationFlag': False,
339	'homeless': True},)
340	member = membership.getMemberById(sid)
341	self.portal_registration.afterAdd(member, sid, password, None)
342	#self.manage_setLocalRoles(sid, ['Owner',])
343	###)
344
345	security.declarePublic('makeStudentData') ###(
346	def makeStudentData(self,student_id,email=None,phone_nr=None):
347	"create Datastructure for a returning Student"
348	#import pdb;pdb.set_trace()
349	logger = logging.getLogger('WAeUPTool.makeStudentData')
350	students_folder = self.portal_url.getPortalObject().campus.students
351	#res = self.students_catalog(id=student_id)
352	#if res:
353	# st = res[0]
354	#res = self.returning_import(matric_no = st.matric_no)
355	res = self.returning_import(id = student_id)
356	if res:
357	student = res[0]
358	else:
359	logger.info('Id %s not found in returning_import' % student_id)
360	return
361	logger.info('%s creates data structure' % student_id)
362	s_results = self.results_import(matric_no = student.matric_no)
363	if s_results:
364	lnr = self.getLevelFromResultsCosCode(s_results)
365	level = "%d00" % lnr
366	verdict,eligible = self.getVerdict(s_results[0].Verdict)
367	if eligible:
368	level = "%d00" % (lnr + 1)
369	else:
370	logger.info('matric_no %s not found in results_import' % student.matric_no)
371	level = 0
372	verdict = 'N/A'
373	#student should not be allowed to perform this transition
374	#wftool = self.portal_workflow
375	#wftool.doActionFor(student,'return')
376	certcode_org = student.Coursemajorcode
377	certcode = makeCertificateCode(certcode_org)
378	certificate_brain = self.getCertificateBrain(certcode)
379	if not certificate_brain:
380	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
381	logger.info(em)
382	matric_no = student.matric_no
383	sid = student_id
384	student_obj = getattr(students_folder,sid)
385	student_obj.invokeFactory('StudentApplication','application')
386	application = student_obj.application
387	self.portal_workflow.doActionFor(application,'open',dest_container=application)
388	da = {'Title': 'Application Data'}
389	student_obj.invokeFactory('StudentPersonal','personal')
390	da['jamb_reg_no'] = student.Entryregno
391	em = self.getEntryMode(student.Entryregno)
392	## em = student.Mode_of_Entry
393	## if em in ('DIRECT', 'DIRECT ENTRY',):
394	## em = 'DE'
395	## elif em in ('U.M.E', 'UNE',):
396	## em = 'UME'
397	## elif not em:
398	## em = "unknown"
399	da['entry_mode'] = em
400	personal = student_obj.personal
401	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
402	dp = {'Title': 'Personal Data'}
403	student_obj.invokeFactory('StudentClearance','clearance')
404	clearance = student_obj.clearance
405	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
406	dc = {'Title': 'Clearance/Eligibility Record'}
407	dc['matric_no'] = matric_no
408	state = student.State
409	lga = student.LGA
410	if state and lga:
411	lga = state + ' / ' + lga
412	else:
413	lga = "None"
414	da['jamb_lga'] = dc['lga'] = lga
415	da['app_email'] = dp['email'] = email
416	da['app_mobile'] = dp['phone'] = phone_nr
417	dp['firstname'] = student.Firstname
418	dp['middlename'] = student.Middlename
419	dp['lastname'] = student.Lastname
420	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
421	da['jamb_sex'] = student.Sex
422	dp['sex'] = student.Sex == 'F'
423	dp['perm_address'] = student.Permanent_Address
424	application.getContent().edit(mapping=da)
425	self.portal_workflow.doActionFor(application,'close',dest_container=application)
426	personal.getContent().edit(mapping=dp)
427	clearance.getContent().edit(mapping=dc)
428	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
429	## catd = {}
430	## catd['id'] = sid
431	## catd['entry_mode']= da['entry_mode']
432	## catd['matric_no'] = matric_no
433	## catd['jamb_reg_no'] = da['jamb_reg_no']
434	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
435	## catd['sex'] = dp['sex']
436	## catd['level'] = level
437	## catd['verdict'] = verdict
438	## if certificate_brain:
439	## cpath = certificate_brain.getPath().split('/')
440	## catd['faculty'] = cpath[-4]
441	## catd['department'] = cpath[-3]
442	## catd['course'] = certcode
443	## self.students_catalog.modifyRecord(**catd)
444	#
445	# Study Course
446	#
447	student_obj.invokeFactory('StudentStudyCourse','study_course')
448	studycourse = student_obj.study_course
449	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
450	dsc = {}
451	dsc['study_course'] = certcode
452	dsc['current_level'] = level
453	dsc['current_verdict'] = verdict
454	dsc['current_mode'] = em
455	dsc['current_session'] = '06'
456	studycourse.getContent().edit(mapping=dsc)
457	#
458	# Level
459	#
460	## l = getattr(studycourse,level,None)
461	## if l is None:
462	## studycourse.invokeFactory('StudentStudyLevel', level)
463	## l = getattr(studycourse, level)
464	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
465	## l.getContent().edit(mapping={'Title': "Level %s" % level})
466	###)
467
468	security.declarePublic('makeStudentLevel') ###(
469	def makeStudentLevel(self,student_id):
470	"create the StudyLevel for a returning Student"
471	#import pdb;pdb.set_trace()
472	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
473	students_folder = self.portal_url.getPortalObject().campus.students
474	res = self.students_catalog(id=student_id)
475	if res:
476	st = res[0]
477	course = st.course
478	matric_no = st.matric_no
479	level = st.level
480	res = self.results_import(matric_no = matric_no)
481	if res:
482	results = res
483	logger.info('%s creating Level %s' % (student_id,level))
484	#
485	# Level
486	#
487	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
488	studycourse = getattr(student_obj,"study_course",None)
489	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
490	l = getattr(studycourse,level,None)
491	if l is None:
492	studycourse.invokeFactory('StudentStudyLevel', level)
493	l = getattr(studycourse, level)
494	self.portal_workflow.doActionFor(l,'open',dest_container=l)
495	l.getContent().edit(mapping={'Title': "Level %s" % level})
496	###)
497
498	security.declarePublic('getAccommodationInfo') ###(
499	def getAccommodationInfo(self,bed):
500	"""return Accommodation Info"""
501	info = {}
502	hall,block,room,letter = bed.split('_')
503	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
504	if res and len(res) == 1:
505	hall_brain = res[0]
506	hall_doc = hall_brain.getObject().getContent()
507	else:
508	return info
509	info['hall_title'] = hall_brain.Title
510	info['maintenance_code'] = hall_doc.maintenance_code
511	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
512	batch_doc = None
513	for brain in res:
514	if brain.id.startswith(info['maintenance_code']):
515	batch_doc = brain.getObject().getContent()
516	break
517	if batch_doc is None:
518	info['maintenance_fee'] = None
519	else:
520	info['maintenance_fee'] = batch_doc.cost
521	return info
522	###)
523
524	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
525	def deleteAllCourses(self,department="All"):
526	''' delete the courses'''
527	pm = self.portal_membership
528	member = pm.getAuthenticatedMember()
529
530	if str(member) not in ("henrik","joachim"):
531	return "not possible"
532	if department == "All":
533	res = self.portal_catalog({'meta_type': 'Department'})
534	if len(res) < 1:
535	return "No Departments found"
536
537	deleted = []
538	for dep in res:
539	cf = dep.getObject().courses
540	if cf:
541	cf.manage_delObjects(ids=cf.objectIds())
542	deleted.append("deleted Courses in %s" % dep.getId)
543	return "\r".join(deleted)
544	###)
545
546	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
547	def getLogfileLines(self,filename="event.log",numlines=20):
548	"""Get last NUMLINES lines of logfile FILENAME.
549
550	Return last lines' of a file in the instances logfile directory as
551	a list. The number of returned lines equals `numlines' or less. If
552	less than `numlines' lines are available, the whole file ist
553	returned. If the file can not be opened or some other error
554	occurs, empty list is returend.
555	"""
556	result = []
557	lines_hit = 0
558
559	# We only handle files in instances' log directory...
560	logpath = os.path.join(i_home, "log")
561	filename = str(os.path.abspath( os.path.join( logpath, filename )))
562	if not filename.startswith( logpath ):
563	# Attempt to access file outside log-dir...
564	return []
565
566	try:
567	fd = file( filename, "rb" )
568	except IOError:
569	return []
570	if not fd:
571	return []
572
573	if os.linesep == None:
574	linesep = '\n'
575	else:
576	linesep = os.linesep
577
578	# Try to find 'numlines' times a lineseparator, searching from end
579	# and moving to the beginning of file...
580	fd.seek( 0, 2) # Move to end of file...
581	while lines_hit < numlines:
582	if fd.read(1) == linesep[-1]: # This moves filedescriptor
583	# one step forward...
584	lines_hit += 1
585	try:
586	fd.seek( -2, 1) # Go two bytes back from current pos...
587	except IOError:
588	# We cannot go back two bytes. Maybe the file is too small...
589	break
590	fd.seek(2,1)
591
592	# Read all lines from current position...
593	result = fd.readlines()
594	# Remove line endings...
595	result = [x.strip() for x in result]
596	fd.close()
597	return result
598	###)
599
600	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
601	def getCallbacksFromLog(self,filename):
602	"""fix Online Payment Transactions from Z2.log entries"""
603	import transaction
604	import random
605	from cgi import parse_qs
606	from urlparse import urlparse
607	#from pdb import set_trace
608	wftool = self.portal_workflow
609	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
610	students_folder = self.portal_url.getPortalObject().campus.students
611	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
612	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
613	s += r'(?P<intersw>.)" "(?P<agent>.)"'
614	data = re.compile(s)
615	start = True
616	tr_count = 1
617	total = 0
618	#name = 'pume_results'
619	#name = 'epaymentsuccessful_z2log2'
620	name = filename
621	no_import = []
622	imported = []
623	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
624	try:
625	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
626	except:
627	logger.error('Error reading %s' % name)
628	return
629	tas = []
630	for line in transactions:
631	dict = {}
632	items = data.search(line)
633	dict['idict'] = idict = items.groupdict()
634	#print idict
635	#from pdb import set_trace;set_trace()
636	urlparsed = urlparse(idict['get'][4:])
637	#print urlparsed
638	path = urlparsed[2].split('/')
639	dict['student_id'] = student_id = path[8]
640	dict['payment_id'] = payment_id = path[10]
641	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
642	tas.append(dict)
643	tr_count += 1
644	return tas
645	###)
646
647	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
648	def importOnlinePaymentTransactions(self):
649	"""load Online Payment Transactions from CSV values"""
650	import transaction
651	import random
652	#from pdb import set_trace
653	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
654	opt = self.online_payments_import
655	students_folder = self.portal_url.getPortalObject().campus.students
656	start = True
657	tr_count = 1
658	total = 0
659	#name = 'pume_results'
660	name = 'OnlineTransactions'
661	no_import = []
662	imported = []
663	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
664	try:
665	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
666	except:
667	logger.error('Error reading %s.csv' % name)
668	return
669	for pay_transaction in transactions:
670	if start:
671	start = False
672	logger.info('Start loading from %s.csv' % name)
673	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
674	no_import.append('%s,"Error"' % s)
675	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
676	format_error = format + ',"%(Error)s"'
677	data = {}
678
679	# format of the first file sent by Tayo
680	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
681	#data['student_id'] = student_id = pay_transaction['Payer ID']
682	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
683	#data['response_code'] = response_code = pay_transaction['Resp Code']
684	#data['amount'] = amount = pay_transaction['Amount']
685
686	# format of the second file sent by Tayo
687	#data['datetime'] = date = 0
688	#data['student_id'] = student_id = pay_transaction['Payer ID']
689	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
690	#data['response_code'] = response_code = '00'
691	#data['amount'] = amount = pay_transaction['Amount']
692
693	# format of the third file sent by Kehinde
694	data['datetime'] = date = 0
695	data['student_id'] = student_id = pay_transaction['customer_id']
696	data['order_id'] = order_id = pay_transaction['merchant_reference']
697	data['response_code'] = response_code = '00'
698	data['amount'] = amount = pay_transaction['Amount']
699
700	dup = False
701	if response_code == "12":
702	continue
703	try:
704	opt.addRecord(**data)
705	except ValueError:
706	dup = True
707	#from pdb import set_trace;set_trace()
708	if dup:
709	if response_code == "00":
710	try:
711	opt.modifyRecord(**data)
712	except:
713	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
714	continue
715	else:
716	pay_transaction['Error'] = "Duplicate order_id"
717	no_import.append( format_error % pay_transaction)
718	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
719	continue
720	tr_count += 1
721	if tr_count > 1000:
722	if len(no_import) > 0:
723	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
724	'\n'.join(no_import) + '\n')
725	no_import = []
726	em = '%d transactions committed\n' % (tr_count)
727	transaction.commit()
728	regs = []
729	logger.info(em)
730	total += tr_count
731	tr_count = 0
732	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
733	'\n'.join(no_import))
734	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
735	###)
736
737
738	# all workflow transitions refer to students in state returning
739	student_field2types = { ###(
740	'StudentApplication':
741	{'id': 'application',
742	'title': 'Application Data',
743	'wf_transition_returning': 'close',
744	'wf_transition_admitted': 'remain',
745	'fields':
746	('jamb_reg_no',
747	'entry_mode',
748	'entry_session',
749	'jamb_score',
750	'app_email',
751	)
752	},
753	#'StudentPume':
754	# {'id': 'pume',
755	# 'title': 'Pume Data',
756	# 'wf_transition_returning': 'close',
757	# 'wf_transition_admitted': 'close',
758	# 'fields':
759	# ('pume_score',
760	# )
761	# },
762	'StudentClearance':
763	{'id': 'clearance',
764	'title': 'Clearance Data',
765	'wf_transition_returning': 'close',
766	'wf_transition_admitted': 'remain',
767	'fields':
768	('matric_no',
769	'nationality',
770	'lga',
771	'birthday',
772	)
773	},
774	'StudentPersonal':
775	{'id': 'personal',
776	'title': 'Personal Data',
777	'wf_transition_returning': 'open',
778	'wf_transition_admitted': 'remain',
779	'fields':
780	('firstname',
781	'middlename',
782	'lastname',
783	'sex',
784	'email',
785	'phone',
786	'perm_address',
787	)
788	},
789	'StudentStudyCourse':
790	{'id': 'study_course',
791	'title': 'Study Course',
792	'wf_transition_returning': 'open',
793	'wf_transition_admitted': 'remain',
794	'fields':
795	('study_course',
796	'current_level',
797	'current_session',
798	'current_mode',
799	'current_verdict',
800	)
801	},
802
803	'PaymentsFolder':
804	{'id': 'payments',
805	'title': 'Payments',
806	'wf_transition_returning': 'open',
807	'wf_transition_admitted': 'open',
808	'fields':
809	()
810	},
811	}
812	###)
813
814
815	security.declareProtected(ModifyPortalContent,'importStudent') ###(
816	def importStudent(self,mapping):
817	"create a students record due import"
818	logger = logging.getLogger('WAeUPTool.importStudent')
819	students_folder = self.portal_url.getPortalObject().campus.students
820	jamb_reg_no = mapping.get('jamb_reg_no',None)
821	if jamb_reg_no:
822	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
823	if res:
824	return '',"jamb_reg_no exists"
825	matric_no = mapping.get('matric_no',None)
826	if matric_no:
827	res = self.students_catalog(matric_no = matric_no)
828	if res:
829	return '',"matric_no exists"
830	sid = self.waeup_tool.generateStudentId('?')
831	students_folder.invokeFactory('Student', sid)
832	student_obj = getattr(students_folder,sid)
833	f2t = self.student_field2types
834	d = {}
835	d['jamb_sex'] = 'M'
836	if mapping.get('sex'):
837	d['jamb_sex'] = 'F'
838	for pt in f2t.keys():
839	student_obj.invokeFactory(pt,f2t[pt]['id'])
840	sub_obj = getattr(student_obj,f2t[pt]['id'])
841	sub_doc = sub_obj.getContent()
842	d['Title'] = f2t[pt]['title']
843	for field in f2t[pt]['fields']:
844	d[field] = mapping.get(field,'')
845	if pt == "StudyCourse":
846	for von,zu in (('entry_mode','current_mode'),
847	('entry_session','current_session')):
848	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
849	d[zu] = mapping[von]
850	sub_doc.edit(mapping = d)
851
852	new_state = f2t[pt]['wf_transition_admitted']
853	if new_state != "remain":
854	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
855	wfaction = 'admit'
856
857
858	self.portal_workflow.doActionFor(student_obj,wfaction)
859	student_obj.manage_setLocalRoles(sid, ['Owner',])
860	return sid,''
861	###)
862
863	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
864	def importEditStudent(self,mapping):
865	"edit a students record due import"
866	logger = logging.getLogger('WAeUPTool.importEditStudent')
867	students_folder = self.portal_url.getPortalObject().campus.students
868	sid = mapping.get('id',None)
869	jamb_reg_no = mapping.get('jamb_reg_no',None)
870	matric_no = mapping.get('matric_no',None)
871	editable_keys = mapping.keys()
872	if sid:
873	res = self.students_catalog(id = sid)
874	if not res:
875	return '',"no student with id %s" % sid
876	elif matric_no and res[0].matric_no and\
877	matric_no != res[0].matric_no:
878	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
879	elif jamb_reg_no and res[0].jamb_reg_no and\
880	jamb_reg_no != res[0].jamb_reg_no:
881	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
882	elif jamb_reg_no:
883	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
884	if not res:
885	return '',"no student with jamb_reg_no %s" % jamb_reg_no
886	editable_keys.remove('jamb_reg_no')
887	elif matric_no:
888	res = self.students_catalog(matric_no = matric_no)
889	if not res:
890	return '',"no student with matric_no %s" % matric_no
891	editable_keys.remove('matric_no')
892	sid = res[0].id
893	student_obj = getattr(students_folder,sid)
894	f2t = self.student_field2types
895	d = {}
896	#import pdb;pdb.set_trace()
897	any_change = False
898	for pt in f2t.keys():
899	if pt == "student_application":
900	d['jamb_sex'] = 'M'
901	if mapping.get('sex'):
902	d['jamb_sex'] = 'F'
903	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
904	if intersect:
905	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
906	if sub_obj is None:
907	try:
908	student_obj.invokeFactory(pt,f2t[pt]['id'])
909	except:
910	continue
911	sub_obj = getattr(student_obj,f2t[pt]['id'])
912	d['Title'] = f2t[pt]['title']
913	sub_doc = sub_obj.getContent()
914	for field in intersect:
915	changed = False
916	if getattr(sub_doc,field) != mapping.get(field,''):
917	any_change = True
918	changed = True
919	d[field] = mapping.get(field,'')
920	if changed:
921	sub_doc.edit(mapping = d)
922	if any_change:
923	return sid,''
924	else:
925	return sid,'not modified'
926	###)
927
928	security.declareProtected(ModifyPortalContent,"importData")###(
929	def importData(self,filename,name,edit=False):
930	"""load data from CSV values"""
931	import transaction
932	import random
933
934	pm = self.portal_membership
935	member = pm.getAuthenticatedMember()
936
937	logger = logging.getLogger('WAeUPTool.importData')
938	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
939	students_folder = self.portal_url.getPortalObject().campus.students
940	start = True
941	tr_count = 1
942	total_imported = 0
943	total_not_imported = 0
944	total = 0
945	iname = "import_%s" % name
946	stool = getToolByName(self, 'portal_schemas')
947	ltool = getToolByName(self, 'portal_layouts')
948	schema = stool._getOb(iname)
949	if schema is None:
950	em = 'No such schema %s' % iname
951	logger.error('No such schema %s' % iname)
952	return em
953	layout = ltool._getOb(iname)
954	if layout is None:
955	em = 'No such layout %s' % iname
956	logger.error(em)
957	return em
958	validators = {}
959	for widget in layout.keys():
960	validators[widget] = layout[widget].validate
961	if edit:
962	importer_name = "importEdit%s" % name.capitalize()
963	else:
964	importer_name = "import%s" % name.capitalize()
965	importer = getattr(self, '%s' % importer_name,None)
966	if importer is None:
967	em = 'No importer function %s' % importer_name
968	logger.error(em)
969	return em
970	not_imported = []
971	imported = []
972	try:
973	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
974	except:
975	em = 'Error reading %s.csv' % filename
976	logger.error(em)
977	return em
978	for item in items:
979	if start:
980	start = False
981	adapters = [MappingStorageAdapter(schema, item)]
982	dm = DataModel(item, adapters,context=self)
983	logger.info('%s starts import from %s.csv' % (member,filename))
984	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
985	diff2schema = set(import_keys).difference(set(schema.keys()))
986	diff2layout = set(import_keys).difference(set(layout.keys()))
987	if diff2schema:
988	em = "not ignorable key(s) %s found in heading" % diff2schema
989	return em
990	s = ','.join(['"%s"' % fn for fn in import_keys])
991	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
992	s = '"id",' + s
993	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
994	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
995	format_error = format + ',"%(Error)s"'
996	format = '"%(id)s",'+ format
997	ds = DataStructure(data=item,datamodel=dm)
998	error_string = ""
999	for k in import_keys:
1000	if not validators[k](ds):
1001	error_string += " %s : %s" % (k,ds.getError(k))
1002	if not error_string:
1003	item.update(dm)
1004	item['id'],error = importer(item)
1005	if error:
1006	error_string += error
1007	if error_string:
1008	item['Error'] = error_string
1009	not_imported.append(format_error % item)
1010	total_not_imported += 1
1011	else:
1012	em = format % item
1013	imported.append(em)
1014	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1015	tr_count += 1
1016	total_imported += 1
1017	total += 1
1018	if total_imported and not total_imported % 100:
1019	transaction.commit()
1020	if len(not_imported) > 0:
1021	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1022	'\n'.join(not_imported) + '\n')
1023	not_imported = []
1024	if len(imported) > 0:
1025	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1026	'\n'.join(imported) + '\n')
1027	imported = []
1028	em = '%d transactions committed\n' % (tr_count)
1029	regs = []
1030	logger.info(em)
1031	tr_count = 0
1032	if len(imported) > 0:
1033	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1034	'\n'.join(imported))
1035	if len(not_imported) > 0:
1036	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1037	'\n'.join(not_imported))
1038	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1039	logger.info(em)
1040	return em
1041	###)
1042
1043	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: