Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1898

Last change on this file since 1898 was 1898, checked in by Henrik Bettermann, 18 years ago
resolve ticket #274 (not yet tested)
Property svn:keywords set to `Id`
File size: 41.5 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1898 2007-06-14 19:26:02Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app_doc = student.application.getContent()
215	#clear = student.clearance
216	#clear_doc = clear.getContent()
217	#matric_no = clear_doc.matric_no.upper()
218	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220	#import pdb;pdb.set_trace()
221	if os.path.exists(picture1):
222	file = open(picture1)
223	elif os.path.exists(picture2):
224	file = open(picture2)
225	else:
226	return "passport picture not found %s" % picture1
227
228	outfile = file.read()
229	app_doc.manage_addFile('passport',
230	file=outfile,
231	title="%s.jpg" % filename)
232	return "successfully loaded passport picture"
233	###)
234
235	security.declareProtected(ModifyPortalContent,'createOne') ###(
236	def createOne(self,students_folder,student_brain,letter,commit=False):
237	sid = self.waeup_tool.generateStudentId(letter)
238	students_folder.invokeFactory('Student', sid)
239	student = getattr(students_folder,sid)
240	self.portal_workflow.doActionFor(student,'return')
241	student.manage_setLocalRoles(sid, ['Owner',])
242	matric_no = student_brain.matric_no
243	jamb_reg_no = student_brain.Entryregno
244	self.students_catalog.addRecord(id = sid,
245	matric_no = matric_no,
246	jamb_reg_no = jamb_reg_no,
247	sex = student_brain.Sex == "F",
248	name = "%s %s %s" % (student_brain.Firstname,
249	student_brain.Middlename,
250	student_brain.Lastname)
251	)
252	if commit:
253	transaction.commit()
254	return sid,jamb_reg_no
255	###)
256
257	security.declareProtected(ModifyPortalContent,'addStudent') ###(
258	def addStudent(self,dict):
259	students_folder = self.portal_url.getPortalObject().campus.students
260	sid = self.waeup_tool.generateStudentId('?')
261	students_folder.invokeFactory('Student', sid)
262	student_obj = getattr(students_folder,sid)
263	password = self.generatePassword()
264	self.makeStudentMember(sid,password)
265	f2t = self.student_field2types
266	#from pdb import set_trace; set_trace()
267	d = {}
268	d['jamb_sex'] = 'M'
269	if dict.get('sex'):
270	d['jamb_sex'] = 'F'
271	entry_session = dict.get('entry_session')
272	wfaction = 'return'
273	wft = 'wf_transition_returning'
274	if entry_session == self.getSessionId():
275	wfaction = 'admit'
276	wft = 'wf_transition_admitted'
277	for pt in f2t.keys():
278	student_obj.invokeFactory(pt,f2t[pt]['id'])
279	sub_obj = getattr(student_obj,f2t[pt]['id'])
280	sub_doc = sub_obj.getContent()
281	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
282	d['Title'] = f2t[pt]['title']
283	for field in f2t[pt]['fields']:
284	d[field] = dict.get(field,'')
285	sub_doc.edit(mapping = d)
286	new_state = f2t[pt][wft]
287	if new_state != "remain":
288	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
289	self.portal_workflow.doActionFor(student_obj,wfaction)
290	student_obj.manage_setLocalRoles(sid, ['Owner',])
291	return sid,password
292	###)
293
294	security.declarePublic('getCertificateBrain') ###(
295	def getCertificateBrain(self,cert_id):
296	"do it"
297	res = ZCatalog.searchResults(self.portal_catalog_real,
298	{'portal_type':"Certificate",
299	'id': cert_id})
300	if res:
301	return res[0]
302	return None
303	###)
304
305	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
306	def get_csv_filenames(self):
307	"do it"
308	files = [file for file in os.listdir("%s/import/" % (i_home))
309	if file.endswith('.csv') and file.find('imported') == -1]
310	return files
311	###)
312
313	security.declarePublic('findStudentByMatricelNo') ###(
314	def findStudentByMatricelNo(self,matric_no):
315	"do it"
316	res = ZCatalog.searchResults(self.portal_catalog_real,
317	{'portal_type':"StudentClearance",
318	'SearchableText': matric_no})
319	if res:
320	return res[0]
321	return None
322	###)
323
324	security.declarePublic('makeStudentMember') ###(
325	def makeStudentMember(self,sid,password='uNsEt'):
326	"""make the student a member"""
327	membership = self.portal_membership
328	membership.addMember(sid,
329	password ,
330	roles=('Member',
331	'Student',
332	),
333	domains='',
334	properties = {'memberareaCreationFlag': False,
335	'homeless': True},)
336	member = membership.getMemberById(sid)
337	self.portal_registration.afterAdd(member, sid, password, None)
338	#self.manage_setLocalRoles(sid, ['Owner',])
339	###)
340
341	security.declarePublic('makeStudentData') ###(
342	def makeStudentData(self,student_id,email=None,phone_nr=None):
343	"create Datastructure for a returning Student"
344	#import pdb;pdb.set_trace()
345	logger = logging.getLogger('WAeUPTool.makeStudentData')
346	students_folder = self.portal_url.getPortalObject().campus.students
347	#res = self.students_catalog(id=student_id)
348	#if res:
349	# st = res[0]
350	#res = self.returning_import(matric_no = st.matric_no)
351	res = self.returning_import(id = student_id)
352	if res:
353	student = res[0]
354	else:
355	logger.info('Id %s not found in returning_import' % student_id)
356	return
357	logger.info('%s creates data structure' % student_id)
358	s_results = self.results_import(matric_no = student.matric_no)
359	if s_results:
360	lnr = self.getLevelFromResultsCosCode(s_results)
361	level = "%d00" % lnr
362	verdict,eligible = self.getVerdict(s_results[0].Verdict)
363	if eligible:
364	level = "%d00" % (lnr + 1)
365	else:
366	logger.info('matric_no %s not found in results_import' % student.matric_no)
367	level = 0
368	verdict = 'N/A'
369	#student should not be allowed to perform this transition
370	#wftool = self.portal_workflow
371	#wftool.doActionFor(student,'return')
372	certcode_org = student.Coursemajorcode
373	certcode = makeCertificateCode(certcode_org)
374	certificate_brain = self.getCertificateBrain(certcode)
375	if not certificate_brain:
376	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
377	logger.info(em)
378	matric_no = student.matric_no
379	sid = student_id
380	student_obj = getattr(students_folder,sid)
381	student_obj.invokeFactory('StudentApplication','application')
382	application = student_obj.application
383	self.portal_workflow.doActionFor(application,'open',dest_container=application)
384	da = {'Title': 'Application Data'}
385	student_obj.invokeFactory('StudentPersonal','personal')
386	da['jamb_reg_no'] = student.Entryregno
387	em = self.getEntryMode(student.Entryregno)
388	## em = student.Mode_of_Entry
389	## if em in ('DIRECT', 'DIRECT ENTRY',):
390	## em = 'DE'
391	## elif em in ('U.M.E', 'UNE',):
392	## em = 'UME'
393	## elif not em:
394	## em = "unknown"
395	da['entry_mode'] = em
396	personal = student_obj.personal
397	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
398	dp = {'Title': 'Personal Data'}
399	student_obj.invokeFactory('StudentClearance','clearance')
400	clearance = student_obj.clearance
401	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
402	dc = {'Title': 'Clearance/Eligibility Record'}
403	dc['matric_no'] = matric_no
404	state = student.State
405	lga = student.LGA
406	if state and lga:
407	lga = state + ' / ' + lga
408	else:
409	lga = "None"
410	da['jamb_lga'] = dc['lga'] = lga
411	da['app_email'] = dp['email'] = email
412	da['app_mobile'] = dp['phone'] = phone_nr
413	dp['firstname'] = student.Firstname
414	dp['middlename'] = student.Middlename
415	dp['lastname'] = student.Lastname
416	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
417	da['jamb_sex'] = student.Sex
418	dp['sex'] = student.Sex == 'F'
419	dp['perm_address'] = student.Permanent_Address
420	application.getContent().edit(mapping=da)
421	self.portal_workflow.doActionFor(application,'close',dest_container=application)
422	personal.getContent().edit(mapping=dp)
423	clearance.getContent().edit(mapping=dc)
424	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
425	## catd = {}
426	## catd['id'] = sid
427	## catd['entry_mode']= da['entry_mode']
428	## catd['matric_no'] = matric_no
429	## catd['jamb_reg_no'] = da['jamb_reg_no']
430	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
431	## catd['sex'] = dp['sex']
432	## catd['level'] = level
433	## catd['verdict'] = verdict
434	## if certificate_brain:
435	## cpath = certificate_brain.getPath().split('/')
436	## catd['faculty'] = cpath[-4]
437	## catd['department'] = cpath[-3]
438	## catd['course'] = certcode
439	## self.students_catalog.modifyRecord(**catd)
440	#
441	# Study Course
442	#
443	student_obj.invokeFactory('StudentStudyCourse','study_course')
444	studycourse = student_obj.study_course
445	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
446	dsc = {}
447	dsc['study_course'] = certcode
448	dsc['current_level'] = level
449	dsc['current_verdict'] = verdict
450	dsc['current_mode'] = em
451	dsc['current_session'] = '06'
452	studycourse.getContent().edit(mapping=dsc)
453	#
454	# Level
455	#
456	## l = getattr(studycourse,level,None)
457	## if l is None:
458	## studycourse.invokeFactory('StudentStudyLevel', level)
459	## l = getattr(studycourse, level)
460	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
461	## l.getContent().edit(mapping={'Title': "Level %s" % level})
462	###)
463
464	security.declarePublic('makeStudentLevel') ###(
465	def makeStudentLevel(self,student_id):
466	"create the StudyLevel for a returning Student"
467	#import pdb;pdb.set_trace()
468	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
469	students_folder = self.portal_url.getPortalObject().campus.students
470	res = self.students_catalog(id=student_id)
471	if res:
472	st = res[0]
473	course = st.course
474	matric_no = st.matric_no
475	level = st.level
476	res = self.results_import(matric_no = matric_no)
477	if res:
478	results = res
479	logger.info('%s creating Level %s' % (student_id,level))
480	#
481	# Level
482	#
483	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
484	studycourse = getattr(student_obj,"study_course",None)
485	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
486	l = getattr(studycourse,level,None)
487	if l is None:
488	studycourse.invokeFactory('StudentStudyLevel', level)
489	l = getattr(studycourse, level)
490	self.portal_workflow.doActionFor(l,'open',dest_container=l)
491	l.getContent().edit(mapping={'Title': "Level %s" % level})
492	###)
493
494	security.declarePublic('getAccommodationInfo') ###(
495	def getAccommodationInfo(self,bed):
496	"""return Accommodation Info"""
497	info = {}
498	hall,block,room,letter = bed.split('_')
499	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
500	if res and len(res) == 1:
501	hall_brain = res[0]
502	hall_doc = hall_brain.getObject().getContent()
503	else:
504	return info
505	info['hall_title'] = hall_brain.Title
506	info['maintenance_code'] = hall_doc.maintenance_code
507	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
508	batch_doc = None
509	for brain in res:
510	if brain.id.startswith(info['maintenance_code']):
511	batch_doc = brain.getObject().getContent()
512	break
513	if batch_doc is None:
514	info['maintenance_fee'] = None
515	else:
516	info['maintenance_fee'] = batch_doc.cost
517	return info
518	###)
519
520	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
521	def deleteAllCourses(self,department="All"):
522	''' delete the courses'''
523	pm = self.portal_membership
524	member = pm.getAuthenticatedMember()
525
526	if str(member) not in ("henrik","joachim"):
527	return "not possible"
528	if department == "All":
529	res = self.portal_catalog({'meta_type': 'Department'})
530	if len(res) < 1:
531	return "No Departments found"
532
533	deleted = []
534	for dep in res:
535	cf = dep.getObject().courses
536	if cf:
537	cf.manage_delObjects(ids=cf.objectIds())
538	deleted.append("deleted Courses in %s" % dep.getId)
539	return "\r".join(deleted)
540	###)
541
542	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
543	def getLogfileLines(self,filename="event.log",numlines=20):
544	"""Get last NUMLINES lines of logfile FILENAME.
545
546	Return last lines' of a file in the instances logfile directory as
547	a list. The number of returned lines equals `numlines' or less. If
548	less than `numlines' lines are available, the whole file ist
549	returned. If the file can not be opened or some other error
550	occurs, empty list is returend.
551	"""
552	result = []
553	lines_hit = 0
554
555	# We only handle files in instances' log directory...
556	logpath = os.path.join(i_home, "log")
557	filename = str(os.path.abspath( os.path.join( logpath, filename )))
558	if not filename.startswith( logpath ):
559	# Attempt to access file outside log-dir...
560	return []
561
562	try:
563	fd = file( filename, "rb" )
564	except IOError:
565	return []
566	if not fd:
567	return []
568
569	if os.linesep == None:
570	linesep = '\n'
571	else:
572	linesep = os.linesep
573
574	# Try to find 'numlines' times a lineseparator, searching from end
575	# and moving to the beginning of file...
576	fd.seek( 0, 2) # Move to end of file...
577	while lines_hit < numlines:
578	if fd.read(1) == linesep[-1]: # This moves filedescriptor
579	# one step forward...
580	lines_hit += 1
581	try:
582	fd.seek( -2, 1) # Go two bytes back from current pos...
583	except IOError:
584	# We cannot go back two bytes. Maybe the file is too small...
585	break
586	fd.seek(2,1)
587
588	# Read all lines from current position...
589	result = fd.readlines()
590	# Remove line endings...
591	result = [x.strip() for x in result]
592	fd.close()
593	return result
594	###)
595
596	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
597	def getCallbacksFromLog(self,filename):
598	"""fix Online Payment Transactions from Z2.log entries"""
599	import transaction
600	import random
601	from cgi import parse_qs
602	from urlparse import urlparse
603	#from pdb import set_trace
604	wftool = self.portal_workflow
605	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
606	students_folder = self.portal_url.getPortalObject().campus.students
607	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
608	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
609	s += r'(?P<intersw>.)" "(?P<agent>.)"'
610	data = re.compile(s)
611	start = True
612	tr_count = 1
613	total = 0
614	#name = 'pume_results'
615	#name = 'epaymentsuccessful_z2log2'
616	name = filename
617	no_import = []
618	imported = []
619	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
620	try:
621	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
622	except:
623	logger.error('Error reading %s' % name)
624	return
625	tas = []
626	for line in transactions:
627	dict = {}
628	items = data.search(line)
629	dict['idict'] = idict = items.groupdict()
630	#print idict
631	#from pdb import set_trace;set_trace()
632	urlparsed = urlparse(idict['get'][4:])
633	#print urlparsed
634	path = urlparsed[2].split('/')
635	dict['student_id'] = student_id = path[8]
636	dict['payment_id'] = payment_id = path[10]
637	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
638	tas.append(dict)
639	tr_count += 1
640	return tas
641	###)
642
643	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
644	def importOnlinePaymentTransactions(self):
645	"""load Online Payment Transactions from CSV values"""
646	import transaction
647	import random
648	#from pdb import set_trace
649	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
650	opt = self.online_payments_import
651	students_folder = self.portal_url.getPortalObject().campus.students
652	start = True
653	tr_count = 1
654	total = 0
655	#name = 'pume_results'
656	name = 'OnlineTransactions'
657	no_import = []
658	imported = []
659	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
660	try:
661	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
662	except:
663	logger.error('Error reading %s.csv' % name)
664	return
665	for pay_transaction in transactions:
666	if start:
667	start = False
668	logger.info('Start loading from %s.csv' % name)
669	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
670	no_import.append('%s,"Error"' % s)
671	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
672	format_error = format + ',"%(Error)s"'
673	data = {}
674
675	# format of the first file sent by Tayo
676	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
677	#data['student_id'] = student_id = pay_transaction['Payer ID']
678	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
679	#data['response_code'] = response_code = pay_transaction['Resp Code']
680	#data['amount'] = amount = pay_transaction['Amount']
681
682	# format of the second file sent by Tayo
683	#data['datetime'] = date = 0
684	#data['student_id'] = student_id = pay_transaction['Payer ID']
685	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
686	#data['response_code'] = response_code = '00'
687	#data['amount'] = amount = pay_transaction['Amount']
688
689	# format of the third file sent by Kehinde
690	data['datetime'] = date = 0
691	data['student_id'] = student_id = pay_transaction['customer_id']
692	data['order_id'] = order_id = pay_transaction['merchant_reference']
693	data['response_code'] = response_code = '00'
694	data['amount'] = amount = pay_transaction['Amount']
695
696	dup = False
697	if response_code == "12":
698	continue
699	try:
700	opt.addRecord(**data)
701	except ValueError:
702	dup = True
703	#from pdb import set_trace;set_trace()
704	if dup:
705	if response_code == "00":
706	try:
707	opt.modifyRecord(**data)
708	except:
709	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
710	continue
711	else:
712	pay_transaction['Error'] = "Duplicate order_id"
713	no_import.append( format_error % pay_transaction)
714	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
715	continue
716	tr_count += 1
717	if tr_count > 1000:
718	if len(no_import) > 0:
719	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
720	'\n'.join(no_import) + '\n')
721	no_import = []
722	em = '%d transactions committed\n' % (tr_count)
723	transaction.commit()
724	regs = []
725	logger.info(em)
726	total += tr_count
727	tr_count = 0
728	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
729	'\n'.join(no_import))
730	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
731	###)
732
733
734	# all workflow transitions refer to students in state returning
735	student_field2types = { ###(
736	'StudentApplication':
737	{'id': 'application',
738	'title': 'Application Data',
739	'wf_transition_returning': 'close',
740	'wf_transition_admitted': 'remain',
741	'fields':
742	('jamb_reg_no',
743	'entry_mode',
744	'entry_session',
745	'jamb_score',
746	'app_email',
747	)
748	},
749	#'StudentPume':
750	# {'id': 'pume',
751	# 'title': 'Pume Data',
752	# 'wf_transition_returning': 'close',
753	# 'wf_transition_admitted': 'close',
754	# 'fields':
755	# ('pume_score',
756	# )
757	# },
758	'StudentClearance':
759	{'id': 'clearance',
760	'title': 'Clearance Data',
761	'wf_transition_returning': 'close',
762	'wf_transition_admitted': 'remain',
763	'fields':
764	('matric_no',
765	'nationality',
766	'lga',
767	'birthday',
768	)
769	},
770	'StudentPersonal':
771	{'id': 'personal',
772	'title': 'Personal Data',
773	'wf_transition_returning': 'open',
774	'wf_transition_admitted': 'remain',
775	'fields':
776	('firstname',
777	'middlename',
778	'lastname',
779	'sex',
780	'email',
781	'phone',
782	'perm_address',
783	)
784	},
785	'StudentStudyCourse':
786	{'id': 'study_course',
787	'title': 'Study Course',
788	'wf_transition_returning': 'open',
789	'wf_transition_admitted': 'remain',
790	'fields':
791	('study_course',
792	'current_level',
793	'current_session',
794	'current_mode',
795	'current_verdict',
796	)
797	},
798
799	'PaymentsFolder':
800	{'id': 'payments',
801	'title': 'Payments',
802	'wf_transition_returning': 'open',
803	'wf_transition_admitted': 'open',
804	'fields':
805	()
806	},
807	}
808	###)
809
810
811	security.declareProtected(ModifyPortalContent,'importStudent') ###(
812	def importStudent(self,mapping):
813	"create a students record due import"
814	logger = logging.getLogger('WAeUPTool.importStudent')
815	students_folder = self.portal_url.getPortalObject().campus.students
816	jamb_reg_no = mapping.get('jamb_reg_no',None)
817	if jamb_reg_no:
818	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
819	if res:
820	return '',"jamb_reg_no exists"
821	matric_no = mapping.get('matric_no',None)
822	if matric_no:
823	res = self.students_catalog(matric_no = matric_no)
824	if res:
825	return '',"matric_no exists"
826	sid = self.waeup_tool.generateStudentId('?')
827	students_folder.invokeFactory('Student', sid)
828	student_obj = getattr(students_folder,sid)
829	f2t = self.student_field2types
830	d = {}
831	d['jamb_sex'] = 'M'
832	if mapping.get('sex'):
833	d['jamb_sex'] = 'F'
834	for pt in f2t.keys():
835	student_obj.invokeFactory(pt,f2t[pt]['id'])
836	sub_obj = getattr(student_obj,f2t[pt]['id'])
837	sub_doc = sub_obj.getContent()
838	d['Title'] = f2t[pt]['title']
839	for field in f2t[pt]['fields']:
840	d[field] = mapping.get(field,'')
841	if pt == "StudyCourse":
842	for von,zu in (('entry_mode','current_mode'),
843	('entry_session','current_session')):
844	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
845	d[zu] = mapping[von]
846	sub_doc.edit(mapping = d)
847
848	new_state = f2t[pt]['wf_transition_admitted']
849	if new_state != "remain":
850	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
851	wfaction = 'admit'
852
853
854	self.portal_workflow.doActionFor(student_obj,wfaction)
855	student_obj.manage_setLocalRoles(sid, ['Owner',])
856	return sid,''
857	###)
858
859	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
860	def importEditStudent(self,mapping):
861	"edit a students record due import"
862	logger = logging.getLogger('WAeUPTool.importEditStudent')
863	students_folder = self.portal_url.getPortalObject().campus.students
864	sid = mapping.get('id',None)
865	jamb_reg_no = mapping.get('jamb_reg_no',None)
866	matric_no = mapping.get('matric_no',None)
867	editable_keys = mapping.keys()
868	if sid:
869	res = self.students_catalog(id = sid)
870	if not res:
871	return '',"no student with id %s" % sid
872	elif matric_no and res[0].matric_no and\
873	matric_no != res[0].matric_no:
874	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
875	elif jamb_reg_no and res[0].jamb_reg_no and\
876	jamb_reg_no != res[0].jamb_reg_no:
877	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
878	elif jamb_reg_no:
879	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
880	if not res:
881	return '',"no student with jamb_reg_no %s" % jamb_reg_no
882	editable_keys.remove('jamb_reg_no')
883	elif matric_no:
884	res = self.students_catalog(matric_no = matric_no)
885	if not res:
886	return '',"no student with matric_no %s" % matric_no
887	editable_keys.remove('matric_no')
888	sid = res[0].id
889	student_obj = getattr(students_folder,sid)
890	f2t = self.student_field2types
891	d = {}
892	#import pdb;pdb.set_trace()
893	any_change = False
894	for pt in f2t.keys():
895	if pt == "student_application":
896	d['jamb_sex'] = 'M'
897	if mapping.get('sex'):
898	d['jamb_sex'] = 'F'
899	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
900	if intersect:
901	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
902	if sub_obj is None:
903	try:
904	student_obj.invokeFactory(pt,f2t[pt]['id'])
905	except:
906	continue
907	sub_obj = getattr(student_obj,f2t[pt]['id'])
908	d['Title'] = f2t[pt]['title']
909	sub_doc = sub_obj.getContent()
910	for field in intersect:
911	changed = False
912	if getattr(sub_doc,field) != mapping.get(field,''):
913	any_change = True
914	changed = True
915	d[field] = mapping.get(field,'')
916	if changed:
917	sub_doc.edit(mapping = d)
918	if any_change:
919	return sid,''
920	else:
921	return sid,'not modified'
922	###)
923
924	security.declareProtected(ModifyPortalContent,"importData")###(
925	def importData(self,filename,name,edit=False):
926	"""load data from CSV values"""
927	import transaction
928	import random
929
930	pm = self.portal_membership
931	member = pm.getAuthenticatedMember()
932
933	logger = logging.getLogger('WAeUPTool.importData')
934	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
935	students_folder = self.portal_url.getPortalObject().campus.students
936	start = True
937	tr_count = 1
938	total_imported = 0
939	total_not_imported = 0
940	total = 0
941	iname = "import_%s" % name
942	stool = getToolByName(self, 'portal_schemas')
943	ltool = getToolByName(self, 'portal_layouts')
944	schema = stool._getOb(iname)
945	if schema is None:
946	em = 'No such schema %s' % iname
947	logger.error('No such schema %s' % iname)
948	return em
949	layout = ltool._getOb(iname)
950	if layout is None:
951	em = 'No such layout %s' % iname
952	logger.error(em)
953	return em
954	validators = {}
955	for widget in layout.keys():
956	validators[widget] = layout[widget].validate
957	if edit:
958	importer_name = "importEdit%s" % name.capitalize()
959	else:
960	importer_name = "import%s" % name.capitalize()
961	importer = getattr(self, '%s' % importer_name,None)
962	if importer is None:
963	em = 'No importer function %s' % importer_name
964	logger.error(em)
965	return em
966	not_imported = []
967	imported = []
968	try:
969	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
970	except:
971	em = 'Error reading %s.csv' % filename
972	logger.error(em)
973	return em
974	for item in items:
975	if start:
976	start = False
977	adapters = [MappingStorageAdapter(schema, item)]
978	dm = DataModel(item, adapters,context=self)
979	logger.info('%s starts import from %s.csv' % (member,filename))
980	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
981	diff2schema = set(import_keys).difference(set(schema.keys()))
982	diff2layout = set(import_keys).difference(set(layout.keys()))
983	if diff2schema:
984	em = "not ignorable key(s) %s found in heading" % diff2schema
985	return em
986	s = ','.join(['"%s"' % fn for fn in import_keys])
987	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
988	s = '"id",' + s
989	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
990	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
991	format_error = format + ',"%(Error)s"'
992	format = '"%(id)s",'+ format
993	ds = DataStructure(data=item,datamodel=dm)
994	error_string = ""
995	for k in import_keys:
996	if not validators[k](ds):
997	error_string += " %s : %s" % (k,ds.getError(k))
998	if not error_string:
999	item.update(dm)
1000	item['id'],error = importer(item)
1001	if error:
1002	error_string += error
1003	if error_string:
1004	item['Error'] = error_string
1005	not_imported.append(format_error % item)
1006	total_not_imported += 1
1007	else:
1008	em = format % item
1009	imported.append(em)
1010	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1011	tr_count += 1
1012	total_imported += 1
1013	total += 1
1014	if total_imported and not total_imported % 100:
1015	transaction.commit()
1016	if len(not_imported) > 0:
1017	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1018	'\n'.join(not_imported) + '\n')
1019	not_imported = []
1020	if len(imported) > 0:
1021	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1022	'\n'.join(imported) + '\n')
1023	imported = []
1024	em = '%d transactions committed\n' % (tr_count)
1025	regs = []
1026	logger.info(em)
1027	tr_count = 0
1028	if len(imported) > 0:
1029	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1030	'\n'.join(imported))
1031	if len(not_imported) > 0:
1032	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1033	'\n'.join(not_imported))
1034	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1035	logger.info(em)
1036	return em
1037	###)
1038
1039	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: