Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1896

Last change on this file since 1896 was 1895, checked in by Henrik Bettermann, 18 years ago
was very slow in live system
Property svn:keywords set to `Id`
File size: 41.2 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1895 2007-06-14 04:13:33Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app_doc = student.application.getContent()
215	#clear = student.clearance
216	#clear_doc = clear.getContent()
217	#matric_no = clear_doc.matric_no.upper()
218	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220	#import pdb;pdb.set_trace()
221	if os.path.exists(picture1):
222	file = open(picture1)
223	elif os.path.exists(picture2):
224	file = open(picture2)
225	else:
226	return "passport picture not found %s" % picture1
227
228	outfile = file.read()
229	app_doc.manage_addFile('passport',
230	file=outfile,
231	title="%s.jpg" % filename)
232	return "successfully loaded passport picture"
233	###)
234
235	security.declareProtected(ModifyPortalContent,'createOne') ###(
236	def createOne(self,students_folder,student_brain,letter,commit=False):
237	sid = self.waeup_tool.generateStudentId(letter)
238	students_folder.invokeFactory('Student', sid)
239	student = getattr(students_folder,sid)
240	self.portal_workflow.doActionFor(student,'return')
241	student.manage_setLocalRoles(sid, ['Owner',])
242	matric_no = student_brain.matric_no
243	jamb_reg_no = student_brain.Entryregno
244	self.students_catalog.addRecord(id = sid,
245	matric_no = matric_no,
246	jamb_reg_no = jamb_reg_no,
247	sex = student_brain.Sex == "F",
248	name = "%s %s %s" % (student_brain.Firstname,
249	student_brain.Middlename,
250	student_brain.Lastname)
251	)
252	if commit:
253	transaction.commit()
254	return sid,jamb_reg_no
255	###)
256
257	security.declareProtected(ModifyPortalContent,'addStudent') ###(
258	def addStudent(self,dict):
259	students_folder = self.portal_url.getPortalObject().campus.students
260	sid = self.waeup_tool.generateStudentId('?')
261	students_folder.invokeFactory('Student', sid)
262	student_obj = getattr(students_folder,sid)
263	password = self.generatePassword()
264	self.makeStudentMember(sid,password)
265	f2t = self.student_field2types
266	#from pdb import set_trace; set_trace()
267	d = {}
268	d['jamb_sex'] = 'M'
269	if dict.get('sex'):
270	d['jamb_sex'] = 'F'
271	for pt in f2t.keys():
272	student_obj.invokeFactory(pt,f2t[pt]['id'])
273	sub_obj = getattr(student_obj,f2t[pt]['id'])
274	sub_doc = sub_obj.getContent()
275	self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
276	d['Title'] = f2t[pt]['title']
277	for field in f2t[pt]['fields']:
278	d[field] = dict.get(field,'')
279	sub_doc.edit(mapping = d)
280	new_state = f2t[pt]['wf_transition']
281	if new_state != "remain":
282	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
283	entry_session = dict.get('entry_session')
284	wfaction = 'return'
285	if entry_session == self.getSessionId():
286	wfaction = 'admit'
287	self.portal_workflow.doActionFor(student_obj,wfaction)
288	student_obj.manage_setLocalRoles(sid, ['Owner',])
289	return sid,password
290	###)
291
292	security.declarePublic('getCertificateBrain') ###(
293	def getCertificateBrain(self,cert_id):
294	"do it"
295	res = ZCatalog.searchResults(self.portal_catalog_real,
296	{'portal_type':"Certificate",
297	'id': cert_id})
298	if res:
299	return res[0]
300	return None
301	###)
302
303	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
304	def get_csv_filenames(self):
305	"do it"
306	files = [file for file in os.listdir("%s/import/" % (i_home))
307	if file.endswith('.csv') and file.find('imported') == -1]
308	return files
309	###)
310
311	security.declarePublic('findStudentByMatricelNo') ###(
312	def findStudentByMatricelNo(self,matric_no):
313	"do it"
314	res = ZCatalog.searchResults(self.portal_catalog_real,
315	{'portal_type':"StudentClearance",
316	'SearchableText': matric_no})
317	if res:
318	return res[0]
319	return None
320	###)
321
322	security.declarePublic('makeStudentMember') ###(
323	def makeStudentMember(self,sid,password='uNsEt'):
324	"""make the student a member"""
325	membership = self.portal_membership
326	membership.addMember(sid,
327	password ,
328	roles=('Member',
329	'Student',
330	),
331	domains='',
332	properties = {'memberareaCreationFlag': False,
333	'homeless': True},)
334	member = membership.getMemberById(sid)
335	self.portal_registration.afterAdd(member, sid, password, None)
336	#self.manage_setLocalRoles(sid, ['Owner',])
337	###)
338
339	security.declarePublic('makeStudentData') ###(
340	def makeStudentData(self,student_id,email=None,phone_nr=None):
341	"create Datastructure for a returning Student"
342	#import pdb;pdb.set_trace()
343	logger = logging.getLogger('WAeUPTool.makeStudentData')
344	students_folder = self.portal_url.getPortalObject().campus.students
345	#res = self.students_catalog(id=student_id)
346	#if res:
347	# st = res[0]
348	#res = self.returning_import(matric_no = st.matric_no)
349	res = self.returning_import(id = student_id)
350	if res:
351	student = res[0]
352	else:
353	logger.info('Id %s not found in returning_import' % student_id)
354	return
355	logger.info('%s creates data structure' % student_id)
356	s_results = self.results_import(matric_no = student.matric_no)
357	if s_results:
358	lnr = self.getLevelFromResultsCosCode(s_results)
359	level = "%d00" % lnr
360	verdict,eligible = self.getVerdict(s_results[0].Verdict)
361	if eligible:
362	level = "%d00" % (lnr + 1)
363	else:
364	logger.info('matric_no %s not found in results_import' % student.matric_no)
365	level = 0
366	verdict = 'N/A'
367	#student should not be allowed to perform this transition
368	#wftool = self.portal_workflow
369	#wftool.doActionFor(student,'return')
370	certcode_org = student.Coursemajorcode
371	certcode = makeCertificateCode(certcode_org)
372	certificate_brain = self.getCertificateBrain(certcode)
373	if not certificate_brain:
374	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
375	logger.info(em)
376	matric_no = student.matric_no
377	sid = student_id
378	student_obj = getattr(students_folder,sid)
379	student_obj.invokeFactory('StudentApplication','application')
380	application = student_obj.application
381	self.portal_workflow.doActionFor(application,'open',dest_container=application)
382	da = {'Title': 'Application Data'}
383	student_obj.invokeFactory('StudentPersonal','personal')
384	da['jamb_reg_no'] = student.Entryregno
385	em = self.getEntryMode(student.Entryregno)
386	## em = student.Mode_of_Entry
387	## if em in ('DIRECT', 'DIRECT ENTRY',):
388	## em = 'DE'
389	## elif em in ('U.M.E', 'UNE',):
390	## em = 'UME'
391	## elif not em:
392	## em = "unknown"
393	da['entry_mode'] = em
394	personal = student_obj.personal
395	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
396	dp = {'Title': 'Personal Data'}
397	student_obj.invokeFactory('StudentClearance','clearance')
398	clearance = student_obj.clearance
399	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
400	dc = {'Title': 'Clearance/Eligibility Record'}
401	dc['matric_no'] = matric_no
402	state = student.State
403	lga = student.LGA
404	if state and lga:
405	lga = state + ' / ' + lga
406	else:
407	lga = "None"
408	da['jamb_lga'] = dc['lga'] = lga
409	da['app_email'] = dp['email'] = email
410	da['app_mobile'] = dp['phone'] = phone_nr
411	dp['firstname'] = student.Firstname
412	dp['middlename'] = student.Middlename
413	dp['lastname'] = student.Lastname
414	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
415	da['jamb_sex'] = student.Sex
416	dp['sex'] = student.Sex == 'F'
417	dp['perm_address'] = student.Permanent_Address
418	application.getContent().edit(mapping=da)
419	self.portal_workflow.doActionFor(application,'close',dest_container=application)
420	personal.getContent().edit(mapping=dp)
421	clearance.getContent().edit(mapping=dc)
422	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
423	## catd = {}
424	## catd['id'] = sid
425	## catd['entry_mode']= da['entry_mode']
426	## catd['matric_no'] = matric_no
427	## catd['jamb_reg_no'] = da['jamb_reg_no']
428	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
429	## catd['sex'] = dp['sex']
430	## catd['level'] = level
431	## catd['verdict'] = verdict
432	## if certificate_brain:
433	## cpath = certificate_brain.getPath().split('/')
434	## catd['faculty'] = cpath[-4]
435	## catd['department'] = cpath[-3]
436	## catd['course'] = certcode
437	## self.students_catalog.modifyRecord(**catd)
438	#
439	# Study Course
440	#
441	student_obj.invokeFactory('StudentStudyCourse','study_course')
442	studycourse = student_obj.study_course
443	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
444	dsc = {}
445	dsc['study_course'] = certcode
446	dsc['current_level'] = level
447	dsc['current_verdict'] = verdict
448	dsc['current_mode'] = em
449	dsc['current_session'] = '06'
450	studycourse.getContent().edit(mapping=dsc)
451	#
452	# Level
453	#
454	## l = getattr(studycourse,level,None)
455	## if l is None:
456	## studycourse.invokeFactory('StudentStudyLevel', level)
457	## l = getattr(studycourse, level)
458	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
459	## l.getContent().edit(mapping={'Title': "Level %s" % level})
460	###)
461
462	security.declarePublic('makeStudentLevel') ###(
463	def makeStudentLevel(self,student_id):
464	"create the StudyLevel for a returning Student"
465	#import pdb;pdb.set_trace()
466	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
467	students_folder = self.portal_url.getPortalObject().campus.students
468	res = self.students_catalog(id=student_id)
469	if res:
470	st = res[0]
471	course = st.course
472	matric_no = st.matric_no
473	level = st.level
474	res = self.results_import(matric_no = matric_no)
475	if res:
476	results = res
477	logger.info('%s creating Level %s' % (student_id,level))
478	#
479	# Level
480	#
481	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
482	studycourse = getattr(student_obj,"study_course",None)
483	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
484	l = getattr(studycourse,level,None)
485	if l is None:
486	studycourse.invokeFactory('StudentStudyLevel', level)
487	l = getattr(studycourse, level)
488	self.portal_workflow.doActionFor(l,'open',dest_container=l)
489	l.getContent().edit(mapping={'Title': "Level %s" % level})
490	###)
491
492	security.declarePublic('getAccommodationInfo') ###(
493	def getAccommodationInfo(self,bed):
494	"""return Accommodation Info"""
495	info = {}
496	hall,block,room,letter = bed.split('_')
497	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
498	if res and len(res) == 1:
499	hall_brain = res[0]
500	hall_doc = hall_brain.getObject().getContent()
501	else:
502	return info
503	info['hall_title'] = hall_brain.Title
504	info['maintenance_code'] = hall_doc.maintenance_code
505	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
506	batch_doc = None
507	for brain in res:
508	if brain.id.startswith(info['maintenance_code']):
509	batch_doc = brain.getObject().getContent()
510	break
511	if batch_doc is None:
512	info['maintenance_fee'] = None
513	else:
514	info['maintenance_fee'] = batch_doc.cost
515	return info
516	###)
517
518	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
519	def deleteAllCourses(self,department="All"):
520	''' delete the courses'''
521	pm = self.portal_membership
522	member = pm.getAuthenticatedMember()
523
524	if str(member) not in ("henrik","joachim"):
525	return "not possible"
526	if department == "All":
527	res = self.portal_catalog({'meta_type': 'Department'})
528	if len(res) < 1:
529	return "No Departments found"
530
531	deleted = []
532	for dep in res:
533	cf = dep.getObject().courses
534	if cf:
535	cf.manage_delObjects(ids=cf.objectIds())
536	deleted.append("deleted Courses in %s" % dep.getId)
537	return "\r".join(deleted)
538	###)
539
540	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
541	def getLogfileLines(self,filename="event.log",numlines=20):
542	"""Get last NUMLINES lines of logfile FILENAME.
543
544	Return last lines' of a file in the instances logfile directory as
545	a list. The number of returned lines equals `numlines' or less. If
546	less than `numlines' lines are available, the whole file ist
547	returned. If the file can not be opened or some other error
548	occurs, empty list is returend.
549	"""
550	result = []
551	lines_hit = 0
552
553	# We only handle files in instances' log directory...
554	logpath = os.path.join(i_home, "log")
555	filename = str(os.path.abspath( os.path.join( logpath, filename )))
556	if not filename.startswith( logpath ):
557	# Attempt to access file outside log-dir...
558	return []
559
560	try:
561	fd = file( filename, "rb" )
562	except IOError:
563	return []
564	if not fd:
565	return []
566
567	if os.linesep == None:
568	linesep = '\n'
569	else:
570	linesep = os.linesep
571
572	# Try to find 'numlines' times a lineseparator, searching from end
573	# and moving to the beginning of file...
574	fd.seek( 0, 2) # Move to end of file...
575	while lines_hit < numlines:
576	if fd.read(1) == linesep[-1]: # This moves filedescriptor
577	# one step forward...
578	lines_hit += 1
579	try:
580	fd.seek( -2, 1) # Go two bytes back from current pos...
581	except IOError:
582	# We cannot go back two bytes. Maybe the file is too small...
583	break
584	fd.seek(2,1)
585
586	# Read all lines from current position...
587	result = fd.readlines()
588	# Remove line endings...
589	result = [x.strip() for x in result]
590	fd.close()
591	return result
592	###)
593
594	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
595	def getCallbacksFromLog(self,filename):
596	"""fix Online Payment Transactions from Z2.log entries"""
597	import transaction
598	import random
599	from cgi import parse_qs
600	from urlparse import urlparse
601	#from pdb import set_trace
602	wftool = self.portal_workflow
603	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
604	students_folder = self.portal_url.getPortalObject().campus.students
605	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
606	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
607	s += r'(?P<intersw>.)" "(?P<agent>.)"'
608	data = re.compile(s)
609	start = True
610	tr_count = 1
611	total = 0
612	#name = 'pume_results'
613	#name = 'epaymentsuccessful_z2log2'
614	name = filename
615	no_import = []
616	imported = []
617	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
618	try:
619	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
620	except:
621	logger.error('Error reading %s' % name)
622	return
623	tas = []
624	for line in transactions:
625	dict = {}
626	items = data.search(line)
627	dict['idict'] = idict = items.groupdict()
628	#print idict
629	#from pdb import set_trace;set_trace()
630	urlparsed = urlparse(idict['get'][4:])
631	#print urlparsed
632	path = urlparsed[2].split('/')
633	dict['student_id'] = student_id = path[8]
634	dict['payment_id'] = payment_id = path[10]
635	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
636	tas.append(dict)
637	tr_count += 1
638	return tas
639	###)
640
641	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
642	def importOnlinePaymentTransactions(self):
643	"""load Online Payment Transactions from CSV values"""
644	import transaction
645	import random
646	#from pdb import set_trace
647	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
648	opt = self.online_payments_import
649	students_folder = self.portal_url.getPortalObject().campus.students
650	start = True
651	tr_count = 1
652	total = 0
653	#name = 'pume_results'
654	name = 'OnlineTransactions'
655	no_import = []
656	imported = []
657	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
658	try:
659	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
660	except:
661	logger.error('Error reading %s.csv' % name)
662	return
663	for pay_transaction in transactions:
664	if start:
665	start = False
666	logger.info('Start loading from %s.csv' % name)
667	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
668	no_import.append('%s,"Error"' % s)
669	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
670	format_error = format + ',"%(Error)s"'
671	data = {}
672
673	# format of the first file sent by Tayo
674	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
675	#data['student_id'] = student_id = pay_transaction['Payer ID']
676	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
677	#data['response_code'] = response_code = pay_transaction['Resp Code']
678	#data['amount'] = amount = pay_transaction['Amount']
679
680	# format of the second file sent by Tayo
681	#data['datetime'] = date = 0
682	#data['student_id'] = student_id = pay_transaction['Payer ID']
683	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
684	#data['response_code'] = response_code = '00'
685	#data['amount'] = amount = pay_transaction['Amount']
686
687	# format of the third file sent by Kehinde
688	data['datetime'] = date = 0
689	data['student_id'] = student_id = pay_transaction['customer_id']
690	data['order_id'] = order_id = pay_transaction['merchant_reference']
691	data['response_code'] = response_code = '00'
692	data['amount'] = amount = pay_transaction['Amount']
693
694	dup = False
695	if response_code == "12":
696	continue
697	try:
698	opt.addRecord(**data)
699	except ValueError:
700	dup = True
701	#from pdb import set_trace;set_trace()
702	if dup:
703	if response_code == "00":
704	try:
705	opt.modifyRecord(**data)
706	except:
707	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
708	continue
709	else:
710	pay_transaction['Error'] = "Duplicate order_id"
711	no_import.append( format_error % pay_transaction)
712	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
713	continue
714	tr_count += 1
715	if tr_count > 1000:
716	if len(no_import) > 0:
717	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
718	'\n'.join(no_import) + '\n')
719	no_import = []
720	em = '%d transactions committed\n' % (tr_count)
721	transaction.commit()
722	regs = []
723	logger.info(em)
724	total += tr_count
725	tr_count = 0
726	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
727	'\n'.join(no_import))
728	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
729	###)
730
731
732	# all workflow transitions refer to students in state returning
733	student_field2types = { ###(
734	'StudentApplication':
735	{'id': 'application',
736	'title': 'Application Data',
737	'wf_transition': 'close',
738	'fields':
739	('jamb_reg_no',
740	'entry_mode',
741	'entry_session',
742	'jamb_score',
743	'app_email',
744	)
745	},
746	#'StudentPume':
747	# {'id': 'pume',
748	# 'title': 'Pume Data',
749	# 'wf_transition': 'close',
750	# 'fields':
751	# ('pume_score',
752	# )
753	# },
754	'StudentClearance':
755	{'id': 'clearance',
756	'title': 'Clearance Data',
757	'wf_transition': 'close',
758	'fields':
759	('matric_no',
760	'nationality',
761	'lga',
762	'birthday',
763	)
764	},
765	'StudentPersonal':
766	{'id': 'personal',
767	'title': 'Personal Data',
768	'wf_transition': 'open',
769	'fields':
770	('firstname',
771	'middlename',
772	'lastname',
773	'sex',
774	'email',
775	'phone',
776	'perm_address',
777	)
778	},
779	'StudentStudyCourse':
780	{'id': 'study_course',
781	'title': 'Study Course',
782	'wf_transition': 'open',
783	'fields':
784	('study_course',
785	'current_level',
786	'current_session',
787	'current_mode',
788	'current_verdict',
789	)
790	},
791	#only temporarily changed to import admitted students
792	#'PaymentsFolder':
793	# {'id': 'payments',
794	# 'title': 'Payments',
795	# 'wf_transition': 'open',
796	# 'fields':
797	# ()
798	# },
799	}
800	###)
801
802
803	security.declareProtected(ModifyPortalContent,'importStudent') ###(
804	def importStudent(self,mapping):
805	"create a students record due import"
806	logger = logging.getLogger('WAeUPTool.importStudent')
807	students_folder = self.portal_url.getPortalObject().campus.students
808	jamb_reg_no = mapping.get('jamb_reg_no',None)
809	if jamb_reg_no:
810	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
811	if res:
812	return '',"jamb_reg_no exists"
813	matric_no = mapping.get('matric_no',None)
814	if matric_no:
815	res = self.students_catalog(matric_no = matric_no)
816	if res:
817	return '',"matric_no exists"
818	sid = self.waeup_tool.generateStudentId('?')
819	students_folder.invokeFactory('Student', sid)
820	student_obj = getattr(students_folder,sid)
821	f2t = self.student_field2types
822	d = {}
823	d['jamb_sex'] = 'M'
824	if mapping.get('sex'):
825	d['jamb_sex'] = 'F'
826	for pt in f2t.keys():
827	student_obj.invokeFactory(pt,f2t[pt]['id'])
828	sub_obj = getattr(student_obj,f2t[pt]['id'])
829	sub_doc = sub_obj.getContent()
830	d['Title'] = f2t[pt]['title']
831	for field in f2t[pt]['fields']:
832	d[field] = mapping.get(field,'')
833	if pt == "StudyCourse":
834	for von,zu in (('entry_mode','current_mode'),
835	('entry_session','current_session')):
836	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
837	d[zu] = mapping[von]
838	sub_doc.edit(mapping = d)
839	new_state = f2t[pt]['wf_transition']
840
841	#only temporarily changed to import admitted students
842	#if new_state != "remain":
843	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
844	wfaction = 'admit'
845
846
847	self.portal_workflow.doActionFor(student_obj,wfaction)
848	student_obj.manage_setLocalRoles(sid, ['Owner',])
849	return sid,''
850	###)
851
852	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
853	def importEditStudent(self,mapping):
854	"edit a students record due import"
855	logger = logging.getLogger('WAeUPTool.importEditStudent')
856	students_folder = self.portal_url.getPortalObject().campus.students
857	sid = mapping.get('id',None)
858	jamb_reg_no = mapping.get('jamb_reg_no',None)
859	matric_no = mapping.get('matric_no',None)
860	editable_keys = mapping.keys()
861	if sid:
862	res = self.students_catalog(id = sid)
863	if not res:
864	return '',"no student with id %s" % sid
865	elif matric_no and res[0].matric_no and\
866	matric_no != res[0].matric_no:
867	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
868	elif jamb_reg_no and res[0].jamb_reg_no and\
869	jamb_reg_no != res[0].jamb_reg_no:
870	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
871	elif jamb_reg_no:
872	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
873	if not res:
874	return '',"no student with jamb_reg_no %s" % jamb_reg_no
875	editable_keys.remove('jamb_reg_no')
876	elif matric_no:
877	res = self.students_catalog(matric_no = matric_no)
878	if not res:
879	return '',"no student with matric_no %s" % matric_no
880	editable_keys.remove('matric_no')
881	sid = res[0].id
882	student_obj = getattr(students_folder,sid)
883	f2t = self.student_field2types
884	d = {}
885	#import pdb;pdb.set_trace()
886	any_change = False
887	for pt in f2t.keys():
888	if pt == "student_application":
889	d['jamb_sex'] = 'M'
890	if mapping.get('sex'):
891	d['jamb_sex'] = 'F'
892	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
893	if intersect:
894	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
895	if sub_obj is None:
896	try:
897	student_obj.invokeFactory(pt,f2t[pt]['id'])
898	except:
899	continue
900	sub_obj = getattr(student_obj,f2t[pt]['id'])
901	d['Title'] = f2t[pt]['title']
902	sub_doc = sub_obj.getContent()
903	for field in intersect:
904	changed = False
905	if getattr(sub_doc,field) != mapping.get(field,''):
906	any_change = True
907	changed = True
908	d[field] = mapping.get(field,'')
909	if changed:
910	sub_doc.edit(mapping = d)
911	if any_change:
912	return sid,''
913	else:
914	return sid,'not modified'
915	###)
916
917	security.declareProtected(ModifyPortalContent,"importData")###(
918	def importData(self,filename,name,edit=False):
919	"""load data from CSV values"""
920	import transaction
921	import random
922
923	pm = self.portal_membership
924	member = pm.getAuthenticatedMember()
925
926	logger = logging.getLogger('WAeUPTool.importData')
927	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
928	students_folder = self.portal_url.getPortalObject().campus.students
929	start = True
930	tr_count = 1
931	total_imported = 0
932	total_not_imported = 0
933	total = 0
934	iname = "import_%s" % name
935	stool = getToolByName(self, 'portal_schemas')
936	ltool = getToolByName(self, 'portal_layouts')
937	schema = stool._getOb(iname)
938	if schema is None:
939	em = 'No such schema %s' % iname
940	logger.error('No such schema %s' % iname)
941	return em
942	layout = ltool._getOb(iname)
943	if layout is None:
944	em = 'No such layout %s' % iname
945	logger.error(em)
946	return em
947	validators = {}
948	for widget in layout.keys():
949	validators[widget] = layout[widget].validate
950	if edit:
951	importer_name = "importEdit%s" % name.capitalize()
952	else:
953	importer_name = "import%s" % name.capitalize()
954	importer = getattr(self, '%s' % importer_name,None)
955	if importer is None:
956	em = 'No importer function %s' % importer_name
957	logger.error(em)
958	return em
959	not_imported = []
960	imported = []
961	try:
962	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
963	except:
964	em = 'Error reading %s.csv' % filename
965	logger.error(em)
966	return em
967	for item in items:
968	if start:
969	start = False
970	adapters = [MappingStorageAdapter(schema, item)]
971	dm = DataModel(item, adapters,context=self)
972	logger.info('%s starts import from %s.csv' % (member,filename))
973	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
974	diff2schema = set(import_keys).difference(set(schema.keys()))
975	diff2layout = set(import_keys).difference(set(layout.keys()))
976	if diff2schema:
977	em = "not ignorable key(s) %s found in heading" % diff2schema
978	return em
979	s = ','.join(['"%s"' % fn for fn in import_keys])
980	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
981	s = '"id",' + s
982	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
983	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
984	format_error = format + ',"%(Error)s"'
985	format = '"%(id)s",'+ format
986	ds = DataStructure(data=item,datamodel=dm)
987	error_string = ""
988	for k in import_keys:
989	if not validators[k](ds):
990	error_string += " %s : %s" % (k,ds.getError(k))
991	if not error_string:
992	item.update(dm)
993	item['id'],error = importer(item)
994	if error:
995	error_string += error
996	if error_string:
997	item['Error'] = error_string
998	not_imported.append(format_error % item)
999	total_not_imported += 1
1000	else:
1001	em = format % item
1002	imported.append(em)
1003	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1004	tr_count += 1
1005	total_imported += 1
1006	total += 1
1007	if total_imported and not total_imported % 100:
1008	transaction.commit()
1009	if len(not_imported) > 0:
1010	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1011	'\n'.join(not_imported) + '\n')
1012	not_imported = []
1013	if len(imported) > 0:
1014	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1015	'\n'.join(imported) + '\n')
1016	imported = []
1017	em = '%d transactions committed\n' % (tr_count)
1018	regs = []
1019	logger.info(em)
1020	tr_count = 0
1021	if len(imported) > 0:
1022	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1023	'\n'.join(imported))
1024	if len(not_imported) > 0:
1025	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1026	'\n'.join(not_imported))
1027	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1028	logger.info(em)
1029	return em
1030	###)
1031
1032	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: