Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1905

Last change on this file since 1905 was 1905, checked in by Henrik Bettermann, 17 years ago
change also state of subobjects (please check)
Property svn:keywords set to `Id`
File size: 42.2 KB

Line
1	3#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1905 2007-06-16 08:04:27Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app_doc = student.application.getContent()
215	#clear = student.clearance
216	#clear_doc = clear.getContent()
217	#matric_no = clear_doc.matric_no.upper()
218	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220	#import pdb;pdb.set_trace()
221	if os.path.exists(picture1):
222	file = open(picture1)
223	elif os.path.exists(picture2):
224	file = open(picture2)
225	else:
226	return "passport picture not found %s" % picture1
227
228	outfile = file.read()
229	app_doc.manage_addFile('passport',
230	file=outfile,
231	title="%s.jpg" % filename)
232	return "successfully loaded passport picture"
233	###)
234
235	security.declareProtected(ModifyPortalContent,'createOne') ###(
236	def createOne(self,students_folder,student_brain,letter,commit=False):
237	sid = self.waeup_tool.generateStudentId(letter)
238	students_folder.invokeFactory('Student', sid)
239	student = getattr(students_folder,sid)
240	self.portal_workflow.doActionFor(student,'return')
241	student.manage_setLocalRoles(sid, ['Owner',])
242	matric_no = student_brain.matric_no
243	jamb_reg_no = student_brain.Entryregno
244	self.students_catalog.addRecord(id = sid,
245	matric_no = matric_no,
246	jamb_reg_no = jamb_reg_no,
247	sex = student_brain.Sex == "F",
248	name = "%s %s %s" % (student_brain.Firstname,
249	student_brain.Middlename,
250	student_brain.Lastname)
251	)
252	if commit:
253	transaction.commit()
254	return sid,jamb_reg_no
255	###)
256
257	security.declareProtected(ModifyPortalContent,'addStudent') ###(
258	def addStudent(self,dict):
259	students_folder = self.portal_url.getPortalObject().campus.students
260	sid = self.waeup_tool.generateStudentId('?')
261	students_folder.invokeFactory('Student', sid)
262	student_obj = getattr(students_folder,sid)
263	f2t = self.student_field2types
264	#from pdb import set_trace; set_trace()
265	d = {}
266	d['jamb_sex'] = 'M'
267	if dict.get('sex'):
268	d['jamb_sex'] = 'F'
269
270	entry_session = dict.get('entry_session')
271	if entry_session == self.getSessionId()[-2:]:
272	wfaction = 'admit'
273	wft = 'wf_transition_admitted'
274	password = None
275	else:
276	wfaction = 'return'
277	wft = 'wf_transition_returning'
278	password = self.generatePassword()
279	self.makeStudentMember(sid,password)
280
281	for pt in f2t.keys():
282	student_obj.invokeFactory(pt,f2t[pt]['id'])
283	sub_obj = getattr(student_obj,f2t[pt]['id'])
284	sub_doc = sub_obj.getContent()
285	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
286	d['Title'] = f2t[pt]['title']
287	for field in f2t[pt]['fields']:
288	d[field] = dict.get(field,'')
289	sub_doc.edit(mapping = d)
290	new_state = f2t[pt][wft]
291	if new_state != "remain":
292	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
293	self.portal_workflow.doActionFor(student_obj,wfaction)
294	student_obj.manage_setLocalRoles(sid, ['Owner',])
295	return sid,password
296	###)
297
298	security.declarePublic('getCertificateBrain') ###(
299	def getCertificateBrain(self,cert_id):
300	"do it"
301	res = ZCatalog.searchResults(self.portal_catalog_real,
302	{'portal_type':"Certificate",
303	'id': cert_id})
304	if res:
305	return res[0]
306	return None
307	###)
308
309	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
310	def get_csv_filenames(self):
311	"do it"
312	files = [file for file in os.listdir("%s/import/" % (i_home))
313	if file.endswith('.csv') and file.find('imported') == -1]
314	return files
315	###)
316
317	security.declarePublic('findStudentByMatricelNo') ###(
318	def findStudentByMatricelNo(self,matric_no):
319	"do it"
320	res = ZCatalog.searchResults(self.portal_catalog_real,
321	{'portal_type':"StudentClearance",
322	'SearchableText': matric_no})
323	if res:
324	return res[0]
325	return None
326	###)
327
328	security.declarePublic('makeStudentMember') ###(
329	def makeStudentMember(self,sid,password='uNsEt'):
330	"""make the student a member"""
331	membership = self.portal_membership
332	membership.addMember(sid,
333	password ,
334	roles=('Member',
335	'Student',
336	),
337	domains='',
338	properties = {'memberareaCreationFlag': False,
339	'homeless': True},)
340	member = membership.getMemberById(sid)
341	self.portal_registration.afterAdd(member, sid, password, None)
342	#self.manage_setLocalRoles(sid, ['Owner',])
343	###)
344
345	security.declarePublic('makeStudentData') ###(
346	def makeStudentData(self,student_id,email=None,phone_nr=None):
347	"create Datastructure for a returning Student"
348	#import pdb;pdb.set_trace()
349	logger = logging.getLogger('WAeUPTool.makeStudentData')
350	students_folder = self.portal_url.getPortalObject().campus.students
351	#res = self.students_catalog(id=student_id)
352	#if res:
353	# st = res[0]
354	#res = self.returning_import(matric_no = st.matric_no)
355	res = self.returning_import(id = student_id)
356	if res:
357	student = res[0]
358	else:
359	logger.info('Id %s not found in returning_import' % student_id)
360	return
361	logger.info('%s creates data structure' % student_id)
362	s_results = self.results_import(matric_no = student.matric_no)
363	if s_results:
364	lnr = self.getLevelFromResultsCosCode(s_results)
365	level = "%d00" % lnr
366	verdict,eligible = self.getVerdict(s_results[0].Verdict)
367	if eligible:
368	level = "%d00" % (lnr + 1)
369	else:
370	logger.info('matric_no %s not found in results_import' % student.matric_no)
371	level = 0
372	verdict = 'N/A'
373	#student should not be allowed to perform this transition
374	#wftool = self.portal_workflow
375	#wftool.doActionFor(student,'return')
376	certcode_org = student.Coursemajorcode
377	certcode = makeCertificateCode(certcode_org)
378	certificate_brain = self.getCertificateBrain(certcode)
379	if not certificate_brain:
380	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
381	logger.info(em)
382	matric_no = student.matric_no
383	sid = student_id
384	student_obj = getattr(students_folder,sid)
385	student_obj.invokeFactory('StudentApplication','application')
386	application = student_obj.application
387	self.portal_workflow.doActionFor(application,'open',dest_container=application)
388	da = {'Title': 'Application Data'}
389	student_obj.invokeFactory('StudentPersonal','personal')
390	da['jamb_reg_no'] = student.Entryregno
391	em = self.getEntryMode(student.Entryregno)
392	## em = student.Mode_of_Entry
393	## if em in ('DIRECT', 'DIRECT ENTRY',):
394	## em = 'DE'
395	## elif em in ('U.M.E', 'UNE',):
396	## em = 'UME'
397	## elif not em:
398	## em = "unknown"
399	da['entry_mode'] = em
400	personal = student_obj.personal
401	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
402	dp = {'Title': 'Personal Data'}
403	student_obj.invokeFactory('StudentClearance','clearance')
404	clearance = student_obj.clearance
405	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
406	dc = {'Title': 'Clearance/Eligibility Record'}
407	dc['matric_no'] = matric_no
408	state = student.State
409	lga = student.LGA
410	if state and lga:
411	lga = state + ' / ' + lga
412	else:
413	lga = "None"
414	da['jamb_lga'] = dc['lga'] = lga
415	da['app_email'] = dp['email'] = email
416	da['app_mobile'] = dp['phone'] = phone_nr
417	dp['firstname'] = student.Firstname
418	dp['middlename'] = student.Middlename
419	dp['lastname'] = student.Lastname
420	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
421	da['jamb_sex'] = student.Sex
422	dp['sex'] = student.Sex == 'F'
423	dp['perm_address'] = student.Permanent_Address
424	application.getContent().edit(mapping=da)
425	self.portal_workflow.doActionFor(application,'close',dest_container=application)
426	personal.getContent().edit(mapping=dp)
427	clearance.getContent().edit(mapping=dc)
428	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
429	## catd = {}
430	## catd['id'] = sid
431	## catd['entry_mode']= da['entry_mode']
432	## catd['matric_no'] = matric_no
433	## catd['jamb_reg_no'] = da['jamb_reg_no']
434	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
435	## catd['sex'] = dp['sex']
436	## catd['level'] = level
437	## catd['verdict'] = verdict
438	## if certificate_brain:
439	## cpath = certificate_brain.getPath().split('/')
440	## catd['faculty'] = cpath[-4]
441	## catd['department'] = cpath[-3]
442	## catd['course'] = certcode
443	## self.students_catalog.modifyRecord(**catd)
444	#
445	# Study Course
446	#
447	student_obj.invokeFactory('StudentStudyCourse','study_course')
448	studycourse = student_obj.study_course
449	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
450	dsc = {}
451	dsc['study_course'] = certcode
452	dsc['current_level'] = level
453	dsc['current_verdict'] = verdict
454	dsc['current_mode'] = em
455	dsc['current_session'] = '06'
456	studycourse.getContent().edit(mapping=dsc)
457	#
458	# Level
459	#
460	## l = getattr(studycourse,level,None)
461	## if l is None:
462	## studycourse.invokeFactory('StudentStudyLevel', level)
463	## l = getattr(studycourse, level)
464	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
465	## l.getContent().edit(mapping={'Title': "Level %s" % level})
466	###)
467
468	security.declarePublic('makeStudentLevel') ###(
469	def makeStudentLevel(self,student_id):
470	"create the StudyLevel for a returning Student"
471	#import pdb;pdb.set_trace()
472	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
473	students_folder = self.portal_url.getPortalObject().campus.students
474	res = self.students_catalog(id=student_id)
475	if res:
476	st = res[0]
477	course = st.course
478	matric_no = st.matric_no
479	level = st.level
480	res = self.results_import(matric_no = matric_no)
481	if res:
482	results = res
483	logger.info('%s creating Level %s' % (student_id,level))
484	#
485	# Level
486	#
487	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
488	studycourse = getattr(student_obj,"study_course",None)
489	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
490	l = getattr(studycourse,level,None)
491	if l is None:
492	studycourse.invokeFactory('StudentStudyLevel', level)
493	l = getattr(studycourse, level)
494	self.portal_workflow.doActionFor(l,'open',dest_container=l)
495	l.getContent().edit(mapping={'Title': "Level %s" % level})
496	###)
497
498	security.declarePublic('getAccommodationInfo') ###(
499	def getAccommodationInfo(self,bed):
500	"""return Accommodation Info"""
501	info = {}
502	hall,block,room,letter = bed.split('_')
503	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
504	if res and len(res) == 1:
505	hall_brain = res[0]
506	hall_doc = hall_brain.getObject().getContent()
507	else:
508	return info
509	info['hall_title'] = hall_brain.Title
510	info['maintenance_code'] = hall_doc.maintenance_code
511	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
512	batch_doc = None
513	for brain in res:
514	if brain.id.startswith(info['maintenance_code']):
515	batch_doc = brain.getObject().getContent()
516	break
517	if batch_doc is None:
518	info['maintenance_fee'] = None
519	else:
520	info['maintenance_fee'] = batch_doc.cost
521	return info
522	###)
523
524	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
525	def deleteAllCourses(self,department="All"):
526	''' delete the courses'''
527	pm = self.portal_membership
528	member = pm.getAuthenticatedMember()
529
530	if str(member) not in ("henrik","joachim"):
531	return "not possible"
532	if department == "All":
533	res = self.portal_catalog({'meta_type': 'Department'})
534	if len(res) < 1:
535	return "No Departments found"
536
537	deleted = []
538	for dep in res:
539	cf = dep.getObject().courses
540	if cf:
541	cf.manage_delObjects(ids=cf.objectIds())
542	deleted.append("deleted Courses in %s" % dep.getId)
543	return "\r".join(deleted)
544	###)
545
546	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
547	def getLogfileLines(self,filename="event.log",numlines=20):
548	"""Get last NUMLINES lines of logfile FILENAME.
549
550	Return last lines' of a file in the instances logfile directory as
551	a list. The number of returned lines equals `numlines' or less. If
552	less than `numlines' lines are available, the whole file ist
553	returned. If the file can not be opened or some other error
554	occurs, empty list is returend.
555	"""
556	result = []
557	lines_hit = 0
558
559	# We only handle files in instances' log directory...
560	logpath = os.path.join(i_home, "log")
561	filename = str(os.path.abspath( os.path.join( logpath, filename )))
562	if not filename.startswith( logpath ):
563	# Attempt to access file outside log-dir...
564	return []
565
566	try:
567	fd = file( filename, "rb" )
568	except IOError:
569	return []
570	if not fd:
571	return []
572
573	if os.linesep == None:
574	linesep = '\n'
575	else:
576	linesep = os.linesep
577
578	# Try to find 'numlines' times a lineseparator, searching from end
579	# and moving to the beginning of file...
580	fd.seek( 0, 2) # Move to end of file...
581	while lines_hit < numlines:
582	if fd.read(1) == linesep[-1]: # This moves filedescriptor
583	# one step forward...
584	lines_hit += 1
585	try:
586	fd.seek( -2, 1) # Go two bytes back from current pos...
587	except IOError:
588	# We cannot go back two bytes. Maybe the file is too small...
589	break
590	fd.seek(2,1)
591
592	# Read all lines from current position...
593	result = fd.readlines()
594	# Remove line endings...
595	result = [x.strip() for x in result]
596	fd.close()
597	return result
598	###)
599
600	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
601	def getCallbacksFromLog(self,filename):
602	"""fix Online Payment Transactions from Z2.log entries"""
603	import transaction
604	import random
605	from cgi import parse_qs
606	from urlparse import urlparse
607	#from pdb import set_trace
608	wftool = self.portal_workflow
609	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
610	students_folder = self.portal_url.getPortalObject().campus.students
611	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
612	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
613	s += r'(?P<intersw>.)" "(?P<agent>.)"'
614	data = re.compile(s)
615	start = True
616	tr_count = 1
617	total = 0
618	#name = 'pume_results'
619	#name = 'epaymentsuccessful_z2log2'
620	name = filename
621	no_import = []
622	imported = []
623	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
624	try:
625	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
626	except:
627	logger.error('Error reading %s' % name)
628	return
629	tas = []
630	for line in transactions:
631	dict = {}
632	items = data.search(line)
633	dict['idict'] = idict = items.groupdict()
634	#print idict
635	#from pdb import set_trace;set_trace()
636	urlparsed = urlparse(idict['get'][4:])
637	#print urlparsed
638	path = urlparsed[2].split('/')
639	dict['student_id'] = student_id = path[8]
640	dict['payment_id'] = payment_id = path[10]
641	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
642	tas.append(dict)
643	tr_count += 1
644	return tas
645	###)
646
647	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
648	def importOnlinePaymentTransactions(self):
649	"""load Online Payment Transactions from CSV values"""
650	import transaction
651	import random
652	#from pdb import set_trace
653	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
654	opt = self.online_payments_import
655	students_folder = self.portal_url.getPortalObject().campus.students
656	start = True
657	tr_count = 1
658	total = 0
659	#name = 'pume_results'
660	name = 'OnlineTransactions'
661	no_import = []
662	imported = []
663	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
664	try:
665	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
666	except:
667	logger.error('Error reading %s.csv' % name)
668	return
669	for pay_transaction in transactions:
670	if start:
671	start = False
672	logger.info('Start loading from %s.csv' % name)
673	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
674	no_import.append('%s,"Error"' % s)
675	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
676	format_error = format + ',"%(Error)s"'
677	data = {}
678
679	# format of the first file sent by Tayo
680	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
681	#data['student_id'] = student_id = pay_transaction['Payer ID']
682	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
683	#data['response_code'] = response_code = pay_transaction['Resp Code']
684	#data['amount'] = amount = pay_transaction['Amount']
685
686	# format of the second file sent by Tayo
687	#data['datetime'] = date = 0
688	#data['student_id'] = student_id = pay_transaction['Payer ID']
689	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
690	#data['response_code'] = response_code = '00'
691	#data['amount'] = amount = pay_transaction['Amount']
692
693	# format of the third file sent by Kehinde
694	data['datetime'] = date = 0
695	data['student_id'] = student_id = pay_transaction['customer_id']
696	data['order_id'] = order_id = pay_transaction['merchant_reference']
697	data['response_code'] = response_code = '00'
698	data['amount'] = amount = pay_transaction['Amount']
699
700	dup = False
701	if response_code == "12":
702	continue
703	try:
704	opt.addRecord(**data)
705	except ValueError:
706	dup = True
707	#from pdb import set_trace;set_trace()
708	if dup:
709	if response_code == "00":
710	try:
711	opt.modifyRecord(**data)
712	except:
713	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
714	continue
715	else:
716	pay_transaction['Error'] = "Duplicate order_id"
717	no_import.append( format_error % pay_transaction)
718	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
719	continue
720	tr_count += 1
721	if tr_count > 1000:
722	if len(no_import) > 0:
723	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
724	'\n'.join(no_import) + '\n')
725	no_import = []
726	em = '%d transactions committed\n' % (tr_count)
727	transaction.commit()
728	regs = []
729	logger.info(em)
730	total += tr_count
731	tr_count = 0
732	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
733	'\n'.join(no_import))
734	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
735	###)
736
737
738	# all workflow transitions refer to students in state returning
739	student_field2types = { ###(
740	'StudentApplication':
741	{'id': 'application',
742	'title': 'Application Data',
743	'wf_transition_returning': 'close',
744	'wf_transition_admitted': 'remain',
745	'fields':
746	('jamb_reg_no',
747	'entry_mode',
748	'entry_session',
749	'jamb_score',
750	'app_email',
751	)
752	},
753	#'StudentPume':
754	# {'id': 'pume',
755	# 'title': 'Pume Data',
756	# 'wf_transition_returning': 'close',
757	# 'wf_transition_admitted': 'close',
758	# 'fields':
759	# ('pume_score',
760	# )
761	# },
762	'StudentClearance':
763	{'id': 'clearance',
764	'title': 'Clearance Data',
765	'wf_transition_returning': 'close',
766	'wf_transition_admitted': 'remain',
767	'fields':
768	('matric_no',
769	'nationality',
770	'lga',
771	'birthday',
772	)
773	},
774	'StudentPersonal':
775	{'id': 'personal',
776	'title': 'Personal Data',
777	'wf_transition_returning': 'open',
778	'wf_transition_admitted': 'remain',
779	'fields':
780	('firstname',
781	'middlename',
782	'lastname',
783	'sex',
784	'email',
785	'phone',
786	'perm_address',
787	)
788	},
789	'StudentStudyCourse':
790	{'id': 'study_course',
791	'title': 'Study Course',
792	'wf_transition_returning': 'open',
793	'wf_transition_admitted': 'remain',
794	'fields':
795	('study_course',
796	'current_level',
797	'current_session',
798	'current_mode',
799	'current_verdict',
800	)
801	},
802
803	'PaymentsFolder':
804	{'id': 'payments',
805	'title': 'Payments',
806	'wf_transition_returning': 'open',
807	'wf_transition_admitted': 'open',
808	'fields':
809	()
810	},
811	}
812	###)
813
814
815	security.declareProtected(ModifyPortalContent,'importStudent') ###(
816	def importStudent(self,mapping):
817	"create a students record due import"
818	logger = logging.getLogger('WAeUPTool.importStudent')
819	students_folder = self.portal_url.getPortalObject().campus.students
820	jamb_reg_no = mapping.get('jamb_reg_no',None)
821	if jamb_reg_no:
822	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
823	if res:
824	return '',"jamb_reg_no exists"
825	matric_no = mapping.get('matric_no',None)
826	if matric_no:
827	res = self.students_catalog(matric_no = matric_no)
828	if res:
829	return '',"matric_no exists"
830	sid = self.waeup_tool.generateStudentId('?')
831	students_folder.invokeFactory('Student', sid)
832	student_obj = getattr(students_folder,sid)
833	f2t = self.student_field2types
834	d = {}
835	d['jamb_sex'] = 'M'
836	if mapping.get('sex'):
837	d['jamb_sex'] = 'F'
838	for pt in f2t.keys():
839	student_obj.invokeFactory(pt,f2t[pt]['id'])
840	sub_obj = getattr(student_obj,f2t[pt]['id'])
841	sub_doc = sub_obj.getContent()
842	d['Title'] = f2t[pt]['title']
843	for field in f2t[pt]['fields']:
844	d[field] = mapping.get(field,'')
845	if pt == "StudyCourse":
846	for von,zu in (('entry_mode','current_mode'),
847	('entry_session','current_session')):
848	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
849	d[zu] = mapping[von]
850	sub_doc.edit(mapping = d)
851
852	new_state = f2t[pt]['wf_transition_admitted']
853	if new_state != "remain":
854	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
855	wfaction = 'admit'
856	self.portal_workflow.doActionFor(student_obj,wfaction)
857	student_obj.manage_setLocalRoles(sid, ['Owner',])
858	return sid,''
859	###)
860
861	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
862	def importEditStudent(self,mapping):
863	"edit a students record due import"
864	logger = logging.getLogger('WAeUPTool.importEditStudent')
865	students_folder = self.portal_url.getPortalObject().campus.students
866	sid = mapping.get('id',None)
867	jamb_reg_no = mapping.get('jamb_reg_no',None)
868	matric_no = mapping.get('matric_no',None)
869	editable_keys = mapping.keys()
870	if sid:
871	res = self.students_catalog(id = sid)
872	if not res:
873	return '',"no student with id %s" % sid
874	elif matric_no and res[0].matric_no and\
875	matric_no != res[0].matric_no:
876	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
877	elif jamb_reg_no and res[0].jamb_reg_no and\
878	jamb_reg_no != res[0].jamb_reg_no:
879	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
880	elif jamb_reg_no:
881	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
882	if not res:
883	return '',"no student with jamb_reg_no %s" % jamb_reg_no
884	editable_keys.remove('jamb_reg_no')
885	elif matric_no:
886	res = self.students_catalog(matric_no = matric_no)
887	if not res:
888	return '',"no student with matric_no %s" % matric_no
889	editable_keys.remove('matric_no')
890	sid = res[0].id
891	student_obj = getattr(students_folder,sid)
892	f2t = self.student_field2types
893	d = {}
894	#import pdb;pdb.set_trace()
895	any_change = False
896	for pt in f2t.keys():
897	if pt == "student_application":
898	d['jamb_sex'] = 'M'
899	if mapping.get('sex'):
900	d['jamb_sex'] = 'F'
901	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
902	if intersect:
903	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
904	if sub_obj is None:
905	try:
906	student_obj.invokeFactory(pt,f2t[pt]['id'])
907	except:
908	continue
909	sub_obj = getattr(student_obj,f2t[pt]['id'])
910	d['Title'] = f2t[pt]['title']
911	sub_doc = sub_obj.getContent()
912	for field in intersect:
913	changed = False
914	if getattr(sub_doc,field) != mapping.get(field,''):
915	any_change = True
916	changed = True
917	d[field] = mapping.get(field,'')
918	if changed:
919	sub_doc.edit(mapping = d)
920
921
922	# included only to change wf state from admitted to returning
923	new_state = f2t[pt]['wf_transition_returning']
924	if new_state != "remain":
925	try:
926	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
927	except:
928	pass
929	wfaction = 'return'
930	try:
931	self.portal_workflow.doActionFor(student_obj,wfaction)
932	logger.info('%s, wf state changed' % sid)
933	any_change = True
934	except:
935	pass
936	# end inclusion
937
938
939	if any_change:
940	return sid,''
941	else:
942	return sid,'not modified'
943	###)
944
945
946	security.declareProtected(ModifyPortalContent,"importData")###(
947	def importData(self,filename,name,edit=False):
948	"""load data from CSV values"""
949	import transaction
950	import random
951
952	pm = self.portal_membership
953	member = pm.getAuthenticatedMember()
954
955	logger = logging.getLogger('WAeUPTool.importData')
956	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
957	students_folder = self.portal_url.getPortalObject().campus.students
958	start = True
959	tr_count = 1
960	total_imported = 0
961	total_not_imported = 0
962	total = 0
963	iname = "import_%s" % name
964	stool = getToolByName(self, 'portal_schemas')
965	ltool = getToolByName(self, 'portal_layouts')
966	schema = stool._getOb(iname)
967	if schema is None:
968	em = 'No such schema %s' % iname
969	logger.error('No such schema %s' % iname)
970	return em
971	layout = ltool._getOb(iname)
972	if layout is None:
973	em = 'No such layout %s' % iname
974	logger.error(em)
975	return em
976	validators = {}
977	for widget in layout.keys():
978	validators[widget] = layout[widget].validate
979	if edit:
980	importer_name = "importEdit%s" % name.capitalize()
981	else:
982	importer_name = "import%s" % name.capitalize()
983	importer = getattr(self, '%s' % importer_name,None)
984	if importer is None:
985	em = 'No importer function %s' % importer_name
986	logger.error(em)
987	return em
988	not_imported = []
989	imported = []
990	try:
991	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
992	except:
993	em = 'Error reading %s.csv' % filename
994	logger.error(em)
995	return em
996	for item in items:
997	if start:
998	start = False
999	adapters = [MappingStorageAdapter(schema, item)]
1000	dm = DataModel(item, adapters,context=self)
1001	logger.info('%s starts import from %s.csv' % (member,filename))
1002	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1003	diff2schema = set(import_keys).difference(set(schema.keys()))
1004	diff2layout = set(import_keys).difference(set(layout.keys()))
1005	if diff2schema:
1006	em = "not ignorable key(s) %s found in heading" % diff2schema
1007	return em
1008	s = ','.join(['"%s"' % fn for fn in import_keys])
1009	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1010	s = '"id",' + s
1011	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1012	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1013	format_error = format + ',"%(Error)s"'
1014	format = '"%(id)s",'+ format
1015	ds = DataStructure(data=item,datamodel=dm)
1016	error_string = ""
1017	for k in import_keys:
1018	if not validators[k](ds):
1019	error_string += " %s : %s" % (k,ds.getError(k))
1020	if not error_string:
1021	item.update(dm)
1022	item['id'],error = importer(item)
1023	if error:
1024	error_string += error
1025	if error_string:
1026	item['Error'] = error_string
1027	not_imported.append(format_error % item)
1028	total_not_imported += 1
1029	else:
1030	em = format % item
1031	imported.append(em)
1032	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1033	tr_count += 1
1034	total_imported += 1
1035	total += 1
1036	if total_imported and not total_imported % 100:
1037	transaction.commit()
1038	if len(not_imported) > 0:
1039	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1040	'\n'.join(not_imported) + '\n')
1041	not_imported = []
1042	if len(imported) > 0:
1043	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1044	'\n'.join(imported) + '\n')
1045	imported = []
1046	em = '%d transactions committed\n' % (tr_count)
1047	regs = []
1048	logger.info(em)
1049	tr_count = 0
1050	if len(imported) > 0:
1051	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1052	'\n'.join(imported))
1053	if len(not_imported) > 0:
1054	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1055	'\n'.join(not_imported))
1056	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1057	logger.info(em)
1058	return em
1059	###)
1060
1061	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: