Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1904

Last change on this file since 1904 was 1904, checked in by Henrik Bettermann, 17 years ago
change wf state due import (not yet tested, please check)
Property svn:keywords set to `Id`
File size: 41.8 KB

Line
1	3#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1904 2007-06-16 06:17:44Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app_doc = student.application.getContent()
215	#clear = student.clearance
216	#clear_doc = clear.getContent()
217	#matric_no = clear_doc.matric_no.upper()
218	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220	#import pdb;pdb.set_trace()
221	if os.path.exists(picture1):
222	file = open(picture1)
223	elif os.path.exists(picture2):
224	file = open(picture2)
225	else:
226	return "passport picture not found %s" % picture1
227
228	outfile = file.read()
229	app_doc.manage_addFile('passport',
230	file=outfile,
231	title="%s.jpg" % filename)
232	return "successfully loaded passport picture"
233	###)
234
235	security.declareProtected(ModifyPortalContent,'createOne') ###(
236	def createOne(self,students_folder,student_brain,letter,commit=False):
237	sid = self.waeup_tool.generateStudentId(letter)
238	students_folder.invokeFactory('Student', sid)
239	student = getattr(students_folder,sid)
240	self.portal_workflow.doActionFor(student,'return')
241	student.manage_setLocalRoles(sid, ['Owner',])
242	matric_no = student_brain.matric_no
243	jamb_reg_no = student_brain.Entryregno
244	self.students_catalog.addRecord(id = sid,
245	matric_no = matric_no,
246	jamb_reg_no = jamb_reg_no,
247	sex = student_brain.Sex == "F",
248	name = "%s %s %s" % (student_brain.Firstname,
249	student_brain.Middlename,
250	student_brain.Lastname)
251	)
252	if commit:
253	transaction.commit()
254	return sid,jamb_reg_no
255	###)
256
257	security.declareProtected(ModifyPortalContent,'addStudent') ###(
258	def addStudent(self,dict):
259	students_folder = self.portal_url.getPortalObject().campus.students
260	sid = self.waeup_tool.generateStudentId('?')
261	students_folder.invokeFactory('Student', sid)
262	student_obj = getattr(students_folder,sid)
263	f2t = self.student_field2types
264	#from pdb import set_trace; set_trace()
265	d = {}
266	d['jamb_sex'] = 'M'
267	if dict.get('sex'):
268	d['jamb_sex'] = 'F'
269
270	entry_session = dict.get('entry_session')
271	if entry_session == self.getSessionId()[-2:]:
272	wfaction = 'admit'
273	wft = 'wf_transition_admitted'
274	password = None
275	else:
276	wfaction = 'return'
277	wft = 'wf_transition_returning'
278	password = self.generatePassword()
279	self.makeStudentMember(sid,password)
280
281	for pt in f2t.keys():
282	student_obj.invokeFactory(pt,f2t[pt]['id'])
283	sub_obj = getattr(student_obj,f2t[pt]['id'])
284	sub_doc = sub_obj.getContent()
285	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
286	d['Title'] = f2t[pt]['title']
287	for field in f2t[pt]['fields']:
288	d[field] = dict.get(field,'')
289	sub_doc.edit(mapping = d)
290	new_state = f2t[pt][wft]
291	if new_state != "remain":
292	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
293	self.portal_workflow.doActionFor(student_obj,wfaction)
294	student_obj.manage_setLocalRoles(sid, ['Owner',])
295	return sid,password
296	###)
297
298	security.declarePublic('getCertificateBrain') ###(
299	def getCertificateBrain(self,cert_id):
300	"do it"
301	res = ZCatalog.searchResults(self.portal_catalog_real,
302	{'portal_type':"Certificate",
303	'id': cert_id})
304	if res:
305	return res[0]
306	return None
307	###)
308
309	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
310	def get_csv_filenames(self):
311	"do it"
312	files = [file for file in os.listdir("%s/import/" % (i_home))
313	if file.endswith('.csv') and file.find('imported') == -1]
314	return files
315	###)
316
317	security.declarePublic('findStudentByMatricelNo') ###(
318	def findStudentByMatricelNo(self,matric_no):
319	"do it"
320	res = ZCatalog.searchResults(self.portal_catalog_real,
321	{'portal_type':"StudentClearance",
322	'SearchableText': matric_no})
323	if res:
324	return res[0]
325	return None
326	###)
327
328	security.declarePublic('makeStudentMember') ###(
329	def makeStudentMember(self,sid,password='uNsEt'):
330	"""make the student a member"""
331	membership = self.portal_membership
332	membership.addMember(sid,
333	password ,
334	roles=('Member',
335	'Student',
336	),
337	domains='',
338	properties = {'memberareaCreationFlag': False,
339	'homeless': True},)
340	member = membership.getMemberById(sid)
341	self.portal_registration.afterAdd(member, sid, password, None)
342	#self.manage_setLocalRoles(sid, ['Owner',])
343	###)
344
345	security.declarePublic('makeStudentData') ###(
346	def makeStudentData(self,student_id,email=None,phone_nr=None):
347	"create Datastructure for a returning Student"
348	#import pdb;pdb.set_trace()
349	logger = logging.getLogger('WAeUPTool.makeStudentData')
350	students_folder = self.portal_url.getPortalObject().campus.students
351	#res = self.students_catalog(id=student_id)
352	#if res:
353	# st = res[0]
354	#res = self.returning_import(matric_no = st.matric_no)
355	res = self.returning_import(id = student_id)
356	if res:
357	student = res[0]
358	else:
359	logger.info('Id %s not found in returning_import' % student_id)
360	return
361	logger.info('%s creates data structure' % student_id)
362	s_results = self.results_import(matric_no = student.matric_no)
363	if s_results:
364	lnr = self.getLevelFromResultsCosCode(s_results)
365	level = "%d00" % lnr
366	verdict,eligible = self.getVerdict(s_results[0].Verdict)
367	if eligible:
368	level = "%d00" % (lnr + 1)
369	else:
370	logger.info('matric_no %s not found in results_import' % student.matric_no)
371	level = 0
372	verdict = 'N/A'
373	#student should not be allowed to perform this transition
374	#wftool = self.portal_workflow
375	#wftool.doActionFor(student,'return')
376	certcode_org = student.Coursemajorcode
377	certcode = makeCertificateCode(certcode_org)
378	certificate_brain = self.getCertificateBrain(certcode)
379	if not certificate_brain:
380	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
381	logger.info(em)
382	matric_no = student.matric_no
383	sid = student_id
384	student_obj = getattr(students_folder,sid)
385	student_obj.invokeFactory('StudentApplication','application')
386	application = student_obj.application
387	self.portal_workflow.doActionFor(application,'open',dest_container=application)
388	da = {'Title': 'Application Data'}
389	student_obj.invokeFactory('StudentPersonal','personal')
390	da['jamb_reg_no'] = student.Entryregno
391	em = self.getEntryMode(student.Entryregno)
392	## em = student.Mode_of_Entry
393	## if em in ('DIRECT', 'DIRECT ENTRY',):
394	## em = 'DE'
395	## elif em in ('U.M.E', 'UNE',):
396	## em = 'UME'
397	## elif not em:
398	## em = "unknown"
399	da['entry_mode'] = em
400	personal = student_obj.personal
401	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
402	dp = {'Title': 'Personal Data'}
403	student_obj.invokeFactory('StudentClearance','clearance')
404	clearance = student_obj.clearance
405	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
406	dc = {'Title': 'Clearance/Eligibility Record'}
407	dc['matric_no'] = matric_no
408	state = student.State
409	lga = student.LGA
410	if state and lga:
411	lga = state + ' / ' + lga
412	else:
413	lga = "None"
414	da['jamb_lga'] = dc['lga'] = lga
415	da['app_email'] = dp['email'] = email
416	da['app_mobile'] = dp['phone'] = phone_nr
417	dp['firstname'] = student.Firstname
418	dp['middlename'] = student.Middlename
419	dp['lastname'] = student.Lastname
420	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
421	da['jamb_sex'] = student.Sex
422	dp['sex'] = student.Sex == 'F'
423	dp['perm_address'] = student.Permanent_Address
424	application.getContent().edit(mapping=da)
425	self.portal_workflow.doActionFor(application,'close',dest_container=application)
426	personal.getContent().edit(mapping=dp)
427	clearance.getContent().edit(mapping=dc)
428	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
429	## catd = {}
430	## catd['id'] = sid
431	## catd['entry_mode']= da['entry_mode']
432	## catd['matric_no'] = matric_no
433	## catd['jamb_reg_no'] = da['jamb_reg_no']
434	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
435	## catd['sex'] = dp['sex']
436	## catd['level'] = level
437	## catd['verdict'] = verdict
438	## if certificate_brain:
439	## cpath = certificate_brain.getPath().split('/')
440	## catd['faculty'] = cpath[-4]
441	## catd['department'] = cpath[-3]
442	## catd['course'] = certcode
443	## self.students_catalog.modifyRecord(**catd)
444	#
445	# Study Course
446	#
447	student_obj.invokeFactory('StudentStudyCourse','study_course')
448	studycourse = student_obj.study_course
449	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
450	dsc = {}
451	dsc['study_course'] = certcode
452	dsc['current_level'] = level
453	dsc['current_verdict'] = verdict
454	dsc['current_mode'] = em
455	dsc['current_session'] = '06'
456	studycourse.getContent().edit(mapping=dsc)
457	#
458	# Level
459	#
460	## l = getattr(studycourse,level,None)
461	## if l is None:
462	## studycourse.invokeFactory('StudentStudyLevel', level)
463	## l = getattr(studycourse, level)
464	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
465	## l.getContent().edit(mapping={'Title': "Level %s" % level})
466	###)
467
468	security.declarePublic('makeStudentLevel') ###(
469	def makeStudentLevel(self,student_id):
470	"create the StudyLevel for a returning Student"
471	#import pdb;pdb.set_trace()
472	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
473	students_folder = self.portal_url.getPortalObject().campus.students
474	res = self.students_catalog(id=student_id)
475	if res:
476	st = res[0]
477	course = st.course
478	matric_no = st.matric_no
479	level = st.level
480	res = self.results_import(matric_no = matric_no)
481	if res:
482	results = res
483	logger.info('%s creating Level %s' % (student_id,level))
484	#
485	# Level
486	#
487	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
488	studycourse = getattr(student_obj,"study_course",None)
489	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
490	l = getattr(studycourse,level,None)
491	if l is None:
492	studycourse.invokeFactory('StudentStudyLevel', level)
493	l = getattr(studycourse, level)
494	self.portal_workflow.doActionFor(l,'open',dest_container=l)
495	l.getContent().edit(mapping={'Title': "Level %s" % level})
496	###)
497
498	security.declarePublic('getAccommodationInfo') ###(
499	def getAccommodationInfo(self,bed):
500	"""return Accommodation Info"""
501	info = {}
502	hall,block,room,letter = bed.split('_')
503	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
504	if res and len(res) == 1:
505	hall_brain = res[0]
506	hall_doc = hall_brain.getObject().getContent()
507	else:
508	return info
509	info['hall_title'] = hall_brain.Title
510	info['maintenance_code'] = hall_doc.maintenance_code
511	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
512	batch_doc = None
513	for brain in res:
514	if brain.id.startswith(info['maintenance_code']):
515	batch_doc = brain.getObject().getContent()
516	break
517	if batch_doc is None:
518	info['maintenance_fee'] = None
519	else:
520	info['maintenance_fee'] = batch_doc.cost
521	return info
522	###)
523
524	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
525	def deleteAllCourses(self,department="All"):
526	''' delete the courses'''
527	pm = self.portal_membership
528	member = pm.getAuthenticatedMember()
529
530	if str(member) not in ("henrik","joachim"):
531	return "not possible"
532	if department == "All":
533	res = self.portal_catalog({'meta_type': 'Department'})
534	if len(res) < 1:
535	return "No Departments found"
536
537	deleted = []
538	for dep in res:
539	cf = dep.getObject().courses
540	if cf:
541	cf.manage_delObjects(ids=cf.objectIds())
542	deleted.append("deleted Courses in %s" % dep.getId)
543	return "\r".join(deleted)
544	###)
545
546	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
547	def getLogfileLines(self,filename="event.log",numlines=20):
548	"""Get last NUMLINES lines of logfile FILENAME.
549
550	Return last lines' of a file in the instances logfile directory as
551	a list. The number of returned lines equals `numlines' or less. If
552	less than `numlines' lines are available, the whole file ist
553	returned. If the file can not be opened or some other error
554	occurs, empty list is returend.
555	"""
556	result = []
557	lines_hit = 0
558
559	# We only handle files in instances' log directory...
560	logpath = os.path.join(i_home, "log")
561	filename = str(os.path.abspath( os.path.join( logpath, filename )))
562	if not filename.startswith( logpath ):
563	# Attempt to access file outside log-dir...
564	return []
565
566	try:
567	fd = file( filename, "rb" )
568	except IOError:
569	return []
570	if not fd:
571	return []
572
573	if os.linesep == None:
574	linesep = '\n'
575	else:
576	linesep = os.linesep
577
578	# Try to find 'numlines' times a lineseparator, searching from end
579	# and moving to the beginning of file...
580	fd.seek( 0, 2) # Move to end of file...
581	while lines_hit < numlines:
582	if fd.read(1) == linesep[-1]: # This moves filedescriptor
583	# one step forward...
584	lines_hit += 1
585	try:
586	fd.seek( -2, 1) # Go two bytes back from current pos...
587	except IOError:
588	# We cannot go back two bytes. Maybe the file is too small...
589	break
590	fd.seek(2,1)
591
592	# Read all lines from current position...
593	result = fd.readlines()
594	# Remove line endings...
595	result = [x.strip() for x in result]
596	fd.close()
597	return result
598	###)
599
600	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
601	def getCallbacksFromLog(self,filename):
602	"""fix Online Payment Transactions from Z2.log entries"""
603	import transaction
604	import random
605	from cgi import parse_qs
606	from urlparse import urlparse
607	#from pdb import set_trace
608	wftool = self.portal_workflow
609	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
610	students_folder = self.portal_url.getPortalObject().campus.students
611	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
612	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
613	s += r'(?P<intersw>.)" "(?P<agent>.)"'
614	data = re.compile(s)
615	start = True
616	tr_count = 1
617	total = 0
618	#name = 'pume_results'
619	#name = 'epaymentsuccessful_z2log2'
620	name = filename
621	no_import = []
622	imported = []
623	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
624	try:
625	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
626	except:
627	logger.error('Error reading %s' % name)
628	return
629	tas = []
630	for line in transactions:
631	dict = {}
632	items = data.search(line)
633	dict['idict'] = idict = items.groupdict()
634	#print idict
635	#from pdb import set_trace;set_trace()
636	urlparsed = urlparse(idict['get'][4:])
637	#print urlparsed
638	path = urlparsed[2].split('/')
639	dict['student_id'] = student_id = path[8]
640	dict['payment_id'] = payment_id = path[10]
641	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
642	tas.append(dict)
643	tr_count += 1
644	return tas
645	###)
646
647	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
648	def importOnlinePaymentTransactions(self):
649	"""load Online Payment Transactions from CSV values"""
650	import transaction
651	import random
652	#from pdb import set_trace
653	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
654	opt = self.online_payments_import
655	students_folder = self.portal_url.getPortalObject().campus.students
656	start = True
657	tr_count = 1
658	total = 0
659	#name = 'pume_results'
660	name = 'OnlineTransactions'
661	no_import = []
662	imported = []
663	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
664	try:
665	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
666	except:
667	logger.error('Error reading %s.csv' % name)
668	return
669	for pay_transaction in transactions:
670	if start:
671	start = False
672	logger.info('Start loading from %s.csv' % name)
673	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
674	no_import.append('%s,"Error"' % s)
675	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
676	format_error = format + ',"%(Error)s"'
677	data = {}
678
679	# format of the first file sent by Tayo
680	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
681	#data['student_id'] = student_id = pay_transaction['Payer ID']
682	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
683	#data['response_code'] = response_code = pay_transaction['Resp Code']
684	#data['amount'] = amount = pay_transaction['Amount']
685
686	# format of the second file sent by Tayo
687	#data['datetime'] = date = 0
688	#data['student_id'] = student_id = pay_transaction['Payer ID']
689	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
690	#data['response_code'] = response_code = '00'
691	#data['amount'] = amount = pay_transaction['Amount']
692
693	# format of the third file sent by Kehinde
694	data['datetime'] = date = 0
695	data['student_id'] = student_id = pay_transaction['customer_id']
696	data['order_id'] = order_id = pay_transaction['merchant_reference']
697	data['response_code'] = response_code = '00'
698	data['amount'] = amount = pay_transaction['Amount']
699
700	dup = False
701	if response_code == "12":
702	continue
703	try:
704	opt.addRecord(**data)
705	except ValueError:
706	dup = True
707	#from pdb import set_trace;set_trace()
708	if dup:
709	if response_code == "00":
710	try:
711	opt.modifyRecord(**data)
712	except:
713	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
714	continue
715	else:
716	pay_transaction['Error'] = "Duplicate order_id"
717	no_import.append( format_error % pay_transaction)
718	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
719	continue
720	tr_count += 1
721	if tr_count > 1000:
722	if len(no_import) > 0:
723	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
724	'\n'.join(no_import) + '\n')
725	no_import = []
726	em = '%d transactions committed\n' % (tr_count)
727	transaction.commit()
728	regs = []
729	logger.info(em)
730	total += tr_count
731	tr_count = 0
732	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
733	'\n'.join(no_import))
734	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
735	###)
736
737
738	# all workflow transitions refer to students in state returning
739	student_field2types = { ###(
740	'StudentApplication':
741	{'id': 'application',
742	'title': 'Application Data',
743	'wf_transition_returning': 'close',
744	'wf_transition_admitted': 'remain',
745	'fields':
746	('jamb_reg_no',
747	'entry_mode',
748	'entry_session',
749	'jamb_score',
750	'app_email',
751	)
752	},
753	#'StudentPume':
754	# {'id': 'pume',
755	# 'title': 'Pume Data',
756	# 'wf_transition_returning': 'close',
757	# 'wf_transition_admitted': 'close',
758	# 'fields':
759	# ('pume_score',
760	# )
761	# },
762	'StudentClearance':
763	{'id': 'clearance',
764	'title': 'Clearance Data',
765	'wf_transition_returning': 'close',
766	'wf_transition_admitted': 'remain',
767	'fields':
768	('matric_no',
769	'nationality',
770	'lga',
771	'birthday',
772	)
773	},
774	'StudentPersonal':
775	{'id': 'personal',
776	'title': 'Personal Data',
777	'wf_transition_returning': 'open',
778	'wf_transition_admitted': 'remain',
779	'fields':
780	('firstname',
781	'middlename',
782	'lastname',
783	'sex',
784	'email',
785	'phone',
786	'perm_address',
787	)
788	},
789	'StudentStudyCourse':
790	{'id': 'study_course',
791	'title': 'Study Course',
792	'wf_transition_returning': 'open',
793	'wf_transition_admitted': 'remain',
794	'fields':
795	('study_course',
796	'current_level',
797	'current_session',
798	'current_mode',
799	'current_verdict',
800	)
801	},
802
803	'PaymentsFolder':
804	{'id': 'payments',
805	'title': 'Payments',
806	'wf_transition_returning': 'open',
807	'wf_transition_admitted': 'open',
808	'fields':
809	()
810	},
811	}
812	###)
813
814
815	security.declareProtected(ModifyPortalContent,'importStudent') ###(
816	def importStudent(self,mapping):
817	"create a students record due import"
818	logger = logging.getLogger('WAeUPTool.importStudent')
819	students_folder = self.portal_url.getPortalObject().campus.students
820	jamb_reg_no = mapping.get('jamb_reg_no',None)
821	if jamb_reg_no:
822	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
823	if res:
824	return '',"jamb_reg_no exists"
825	matric_no = mapping.get('matric_no',None)
826	if matric_no:
827	res = self.students_catalog(matric_no = matric_no)
828	if res:
829	return '',"matric_no exists"
830	sid = self.waeup_tool.generateStudentId('?')
831	students_folder.invokeFactory('Student', sid)
832	student_obj = getattr(students_folder,sid)
833	f2t = self.student_field2types
834	d = {}
835	d['jamb_sex'] = 'M'
836	if mapping.get('sex'):
837	d['jamb_sex'] = 'F'
838	for pt in f2t.keys():
839	student_obj.invokeFactory(pt,f2t[pt]['id'])
840	sub_obj = getattr(student_obj,f2t[pt]['id'])
841	sub_doc = sub_obj.getContent()
842	d['Title'] = f2t[pt]['title']
843	for field in f2t[pt]['fields']:
844	d[field] = mapping.get(field,'')
845	if pt == "StudyCourse":
846	for von,zu in (('entry_mode','current_mode'),
847	('entry_session','current_session')):
848	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
849	d[zu] = mapping[von]
850	sub_doc.edit(mapping = d)
851
852	new_state = f2t[pt]['wf_transition_admitted']
853	if new_state != "remain":
854	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
855	wfaction = 'admit'
856	self.portal_workflow.doActionFor(student_obj,wfaction)
857	student_obj.manage_setLocalRoles(sid, ['Owner',])
858	return sid,''
859	###)
860
861	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
862	def importEditStudent(self,mapping):
863	"edit a students record due import"
864	logger = logging.getLogger('WAeUPTool.importEditStudent')
865	students_folder = self.portal_url.getPortalObject().campus.students
866	sid = mapping.get('id',None)
867	jamb_reg_no = mapping.get('jamb_reg_no',None)
868	matric_no = mapping.get('matric_no',None)
869	editable_keys = mapping.keys()
870	if sid:
871	res = self.students_catalog(id = sid)
872	if not res:
873	return '',"no student with id %s" % sid
874	elif matric_no and res[0].matric_no and\
875	matric_no != res[0].matric_no:
876	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
877	elif jamb_reg_no and res[0].jamb_reg_no and\
878	jamb_reg_no != res[0].jamb_reg_no:
879	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
880	elif jamb_reg_no:
881	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
882	if not res:
883	return '',"no student with jamb_reg_no %s" % jamb_reg_no
884	editable_keys.remove('jamb_reg_no')
885	elif matric_no:
886	res = self.students_catalog(matric_no = matric_no)
887	if not res:
888	return '',"no student with matric_no %s" % matric_no
889	editable_keys.remove('matric_no')
890	sid = res[0].id
891	student_obj = getattr(students_folder,sid)
892	f2t = self.student_field2types
893	d = {}
894	#import pdb;pdb.set_trace()
895	any_change = False
896	for pt in f2t.keys():
897	if pt == "student_application":
898	d['jamb_sex'] = 'M'
899	if mapping.get('sex'):
900	d['jamb_sex'] = 'F'
901	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
902	if intersect:
903	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
904	if sub_obj is None:
905	try:
906	student_obj.invokeFactory(pt,f2t[pt]['id'])
907	except:
908	continue
909	sub_obj = getattr(student_obj,f2t[pt]['id'])
910	d['Title'] = f2t[pt]['title']
911	sub_doc = sub_obj.getContent()
912	for field in intersect:
913	changed = False
914	if getattr(sub_doc,field) != mapping.get(field,''):
915	any_change = True
916	changed = True
917	d[field] = mapping.get(field,'')
918	if changed:
919	sub_doc.edit(mapping = d)
920
921	wfaction = 'return'
922	try:
923	self.portal_workflow.doActionFor(student_obj,wfaction)
924	logger.info('%s, wf state changed' % sid)
925	any_change = True
926	except:
927	pass
928
929	if any_change:
930	return sid,''
931	else:
932	return sid,'not modified'
933	###)
934
935
936	security.declareProtected(ModifyPortalContent,"importData")###(
937	def importData(self,filename,name,edit=False):
938	"""load data from CSV values"""
939	import transaction
940	import random
941
942	pm = self.portal_membership
943	member = pm.getAuthenticatedMember()
944
945	logger = logging.getLogger('WAeUPTool.importData')
946	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
947	students_folder = self.portal_url.getPortalObject().campus.students
948	start = True
949	tr_count = 1
950	total_imported = 0
951	total_not_imported = 0
952	total = 0
953	iname = "import_%s" % name
954	stool = getToolByName(self, 'portal_schemas')
955	ltool = getToolByName(self, 'portal_layouts')
956	schema = stool._getOb(iname)
957	if schema is None:
958	em = 'No such schema %s' % iname
959	logger.error('No such schema %s' % iname)
960	return em
961	layout = ltool._getOb(iname)
962	if layout is None:
963	em = 'No such layout %s' % iname
964	logger.error(em)
965	return em
966	validators = {}
967	for widget in layout.keys():
968	validators[widget] = layout[widget].validate
969	if edit:
970	importer_name = "importEdit%s" % name.capitalize()
971	else:
972	importer_name = "import%s" % name.capitalize()
973	importer = getattr(self, '%s' % importer_name,None)
974	if importer is None:
975	em = 'No importer function %s' % importer_name
976	logger.error(em)
977	return em
978	not_imported = []
979	imported = []
980	try:
981	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
982	except:
983	em = 'Error reading %s.csv' % filename
984	logger.error(em)
985	return em
986	for item in items:
987	if start:
988	start = False
989	adapters = [MappingStorageAdapter(schema, item)]
990	dm = DataModel(item, adapters,context=self)
991	logger.info('%s starts import from %s.csv' % (member,filename))
992	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
993	diff2schema = set(import_keys).difference(set(schema.keys()))
994	diff2layout = set(import_keys).difference(set(layout.keys()))
995	if diff2schema:
996	em = "not ignorable key(s) %s found in heading" % diff2schema
997	return em
998	s = ','.join(['"%s"' % fn for fn in import_keys])
999	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1000	s = '"id",' + s
1001	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1002	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1003	format_error = format + ',"%(Error)s"'
1004	format = '"%(id)s",'+ format
1005	ds = DataStructure(data=item,datamodel=dm)
1006	error_string = ""
1007	for k in import_keys:
1008	if not validators[k](ds):
1009	error_string += " %s : %s" % (k,ds.getError(k))
1010	if not error_string:
1011	item.update(dm)
1012	item['id'],error = importer(item)
1013	if error:
1014	error_string += error
1015	if error_string:
1016	item['Error'] = error_string
1017	not_imported.append(format_error % item)
1018	total_not_imported += 1
1019	else:
1020	em = format % item
1021	imported.append(em)
1022	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1023	tr_count += 1
1024	total_imported += 1
1025	total += 1
1026	if total_imported and not total_imported % 100:
1027	transaction.commit()
1028	if len(not_imported) > 0:
1029	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1030	'\n'.join(not_imported) + '\n')
1031	not_imported = []
1032	if len(imported) > 0:
1033	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1034	'\n'.join(imported) + '\n')
1035	imported = []
1036	em = '%d transactions committed\n' % (tr_count)
1037	regs = []
1038	logger.info(em)
1039	tr_count = 0
1040	if len(imported) > 0:
1041	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1042	'\n'.join(imported))
1043	if len(not_imported) > 0:
1044	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1045	'\n'.join(not_imported))
1046	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1047	logger.info(em)
1048	return em
1049	###)
1050
1051	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: