Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 2046

Last change on this file since 2046 was 2009, checked in by Henrik Bettermann, 17 years ago
ticket #307
Property svn:keywords set to `Id`
File size: 43.3 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 2009 2007-07-11 04:59:50Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app = student.application
215	app_doc = app.getContent()
216	#clear = student.clearance
217	#clear_doc = clear.getContent()
218	#matric_no = clear_doc.matric_no.upper()
219	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
220	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
221	#import pdb;pdb.set_trace()
222	if os.path.exists(picture1):
223	file = open(picture1)
224	elif os.path.exists(picture2):
225	file = open(picture2)
226	else:
227	return "passport picture not found %s" % picture1
228	reopened = False
229	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
230	self.portal_workflow.doActionFor(app,'open')
231	reopened = True
232	outfile = file.read()
233	app_doc.manage_addFile('passport',
234	file=outfile,
235	title="%s.jpg" % filename)
236	if reopened:
237	self.portal_workflow.doActionFor(app,'close')
238	return "successfully loaded passport picture"
239	###)
240
241	security.declareProtected(ModifyPortalContent,'createOne') ###(
242	def createOne(self,students_folder,student_brain,letter,commit=False):
243	sid = self.waeup_tool.generateStudentId(letter)
244	students_folder.invokeFactory('Student', sid)
245	student = getattr(students_folder,sid)
246	self.portal_workflow.doActionFor(student,'return')
247	student.manage_setLocalRoles(sid, ['Owner',])
248	matric_no = student_brain.matric_no
249	jamb_reg_no = student_brain.Entryregno
250	self.students_catalog.addRecord(id = sid,
251	matric_no = matric_no,
252	jamb_reg_no = jamb_reg_no,
253	sex = student_brain.Sex == "F",
254	name = "%s %s %s" % (student_brain.Firstname,
255	student_brain.Middlename,
256	student_brain.Lastname)
257	)
258	if commit:
259	transaction.commit()
260	return sid,jamb_reg_no
261	###)
262
263	security.declareProtected(ModifyPortalContent,'addStudent') ###(
264	def addStudent(self,dict):
265	students_folder = self.portal_url.getPortalObject().campus.students
266	sid = self.waeup_tool.generateStudentId('?')
267	students_folder.invokeFactory('Student', sid)
268	student_obj = getattr(students_folder,sid)
269	f2t = self.student_field2types
270	#from pdb import set_trace; set_trace()
271	d = {}
272	d['jamb_sex'] = 'M'
273	if dict.get('sex'):
274	d['jamb_sex'] = 'F'
275
276	entry_session = dict.get('entry_session')
277	if entry_session == self.getSessionId()[-2:]:
278	wfaction = 'admit'
279	wft = 'wf_transition_admit'
280	password = None
281	else:
282	wfaction = 'return'
283	wft = 'wf_transition_return'
284	password = self.generatePassword()
285	self.makeStudentMember(sid,password)
286
287	for pt in f2t.keys():
288	student_obj.invokeFactory(pt,f2t[pt]['id'])
289	sub_obj = getattr(student_obj,f2t[pt]['id'])
290	sub_doc = sub_obj.getContent()
291	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
292	d['Title'] = f2t[pt]['title']
293	for field in f2t[pt]['fields']:
294	d[field] = dict.get(field,'')
295	sub_doc.edit(mapping = d)
296	new_state = f2t[pt][wft]
297	if new_state != "remain":
298	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
299	self.portal_workflow.doActionFor(student_obj,wfaction)
300	student_obj.manage_setLocalRoles(sid, ['Owner',])
301	return sid,password
302	###)
303
304	security.declarePublic('getCertificateBrain') ###(
305	def getCertificateBrain(self,cert_id):
306	"do it"
307	res = ZCatalog.searchResults(self.portal_catalog_real,
308	{'portal_type':"Certificate",
309	'id': cert_id})
310	if res:
311	return res[0]
312	return None
313	###)
314
315	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
316	def get_csv_filenames(self):
317	"do it"
318	files = [file for file in os.listdir("%s/import/" % (i_home))
319	if file.endswith('.csv') and file.find('imported') == -1]
320	return files
321	###)
322
323	security.declarePublic('findStudentByMatricelNo') ###(
324	def findStudentByMatricelNo(self,matric_no):
325	"do it"
326	res = ZCatalog.searchResults(self.portal_catalog_real,
327	{'portal_type':"StudentClearance",
328	'SearchableText': matric_no})
329	if res:
330	return res[0]
331	return None
332	###)
333
334	security.declarePublic('makeStudentMember') ###(
335	def makeStudentMember(self,sid,password='uNsEt'):
336	"""make the student a member"""
337	membership = self.portal_membership
338	membership.addMember(sid,
339	password ,
340	roles=('Member',
341	'Student',
342	),
343	domains='',
344	properties = {'memberareaCreationFlag': False,
345	'homeless': True},)
346	member = membership.getMemberById(sid)
347	self.portal_registration.afterAdd(member, sid, password, None)
348	#self.manage_setLocalRoles(sid, ['Owner',])
349	###)
350
351	security.declarePublic('makeStudentData') ###(
352	def makeStudentData(self,student_id,email=None,phone_nr=None):
353	"create Datastructure for a returning Student"
354	#import pdb;pdb.set_trace()
355	logger = logging.getLogger('WAeUPTool.makeStudentData')
356	students_folder = self.portal_url.getPortalObject().campus.students
357	#res = self.students_catalog(id=student_id)
358	#if res:
359	# st = res[0]
360	#res = self.returning_import(matric_no = st.matric_no)
361	res = self.returning_import(id = student_id)
362	if res:
363	student = res[0]
364	else:
365	logger.info('Id %s not found in returning_import' % student_id)
366	return
367	logger.info('%s creates data structure' % student_id)
368	s_results = self.results_import(matric_no = student.matric_no)
369	if s_results:
370	lnr = self.getLevelFromResultsCosCode(s_results)
371	level = "%d00" % lnr
372	verdict,eligible = self.getVerdict(s_results[0].Verdict)
373	if eligible:
374	level = "%d00" % (lnr + 1)
375	else:
376	logger.info('matric_no %s not found in results_import' % student.matric_no)
377	level = 0
378	verdict = 'N/A'
379	#student should not be allowed to perform this transition
380	#wftool = self.portal_workflow
381	#wftool.doActionFor(student,'return')
382	certcode_org = student.Coursemajorcode
383	certcode = makeCertificateCode(certcode_org)
384	certificate_brain = self.getCertificateBrain(certcode)
385	if not certificate_brain:
386	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
387	logger.info(em)
388	matric_no = student.matric_no
389	sid = student_id
390	student_obj = getattr(students_folder,sid)
391	student_obj.invokeFactory('StudentApplication','application')
392	application = student_obj.application
393	self.portal_workflow.doActionFor(application,'open',dest_container=application)
394	da = {'Title': 'Application Data'}
395	student_obj.invokeFactory('StudentPersonal','personal')
396	da['jamb_reg_no'] = student.Entryregno
397	em = self.getEntryMode(student.Entryregno)
398	## em = student.Mode_of_Entry
399	## if em in ('DIRECT', 'DIRECT ENTRY',):
400	## em = 'DE'
401	## elif em in ('U.M.E', 'UNE',):
402	## em = 'UME'
403	## elif not em:
404	## em = "unknown"
405	da['entry_mode'] = em
406	personal = student_obj.personal
407	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
408	dp = {'Title': 'Personal Data'}
409	student_obj.invokeFactory('StudentClearance','clearance')
410	clearance = student_obj.clearance
411	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
412	dc = {'Title': 'Clearance/Eligibility Record'}
413	dc['matric_no'] = matric_no
414	state = student.State
415	lga = student.LGA
416	if state and lga:
417	lga = state + ' / ' + lga
418	else:
419	lga = "None"
420	da['jamb_lga'] = dc['lga'] = lga
421	da['app_email'] = dp['email'] = email
422	da['app_mobile'] = dp['phone'] = phone_nr
423	dp['firstname'] = student.Firstname
424	dp['middlename'] = student.Middlename
425	dp['lastname'] = student.Lastname
426	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
427	da['jamb_sex'] = student.Sex
428	dp['sex'] = student.Sex == 'F'
429	dp['perm_address'] = student.Permanent_Address
430	application.getContent().edit(mapping=da)
431	self.portal_workflow.doActionFor(application,'close',dest_container=application)
432	personal.getContent().edit(mapping=dp)
433	clearance.getContent().edit(mapping=dc)
434	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
435	## catd = {}
436	## catd['id'] = sid
437	## catd['entry_mode']= da['entry_mode']
438	## catd['matric_no'] = matric_no
439	## catd['jamb_reg_no'] = da['jamb_reg_no']
440	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
441	## catd['sex'] = dp['sex']
442	## catd['level'] = level
443	## catd['verdict'] = verdict
444	## if certificate_brain:
445	## cpath = certificate_brain.getPath().split('/')
446	## catd['faculty'] = cpath[-4]
447	## catd['department'] = cpath[-3]
448	## catd['course'] = certcode
449	## self.students_catalog.modifyRecord(**catd)
450	#
451	# Study Course
452	#
453	student_obj.invokeFactory('StudentStudyCourse','study_course')
454	studycourse = student_obj.study_course
455	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
456	dsc = {}
457	dsc['study_course'] = certcode
458	dsc['current_level'] = level
459	dsc['current_verdict'] = verdict
460	dsc['current_mode'] = em
461	dsc['current_session'] = '06'
462	studycourse.getContent().edit(mapping=dsc)
463	#
464	# Level
465	#
466	## l = getattr(studycourse,level,None)
467	## if l is None:
468	## studycourse.invokeFactory('StudentStudyLevel', level)
469	## l = getattr(studycourse, level)
470	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
471	## l.getContent().edit(mapping={'Title': "Level %s" % level})
472	###)
473
474	security.declarePublic('makeStudentLevel') ###(
475	def makeStudentLevel(self,student_id):
476	"create the StudyLevel for a returning Student"
477	#import pdb;pdb.set_trace()
478	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
479	students_folder = self.portal_url.getPortalObject().campus.students
480	res = self.students_catalog(id=student_id)
481	if res:
482	st = res[0]
483	course = st.course
484	matric_no = st.matric_no
485	level = st.level
486	res = self.results_import(matric_no = matric_no)
487	if res:
488	results = res
489	logger.info('%s creating Level %s' % (student_id,level))
490	#
491	# Level
492	#
493	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
494	studycourse = getattr(student_obj,"study_course",None)
495	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
496	l = getattr(studycourse,level,None)
497	if l is None:
498	studycourse.invokeFactory('StudentStudyLevel', level)
499	l = getattr(studycourse, level)
500	self.portal_workflow.doActionFor(l,'open',dest_container=l)
501	l.getContent().edit(mapping={'Title': "Level %s" % level})
502	###)
503
504	security.declarePublic('getHallInfo') ###(
505	def getHallInfo(self,bed):
506	"""return Hall Info"""
507	info = {}
508	hall,block,room,letter = bed.split('_')
509	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
510	if res and len(res) == 1:
511	hall_brain = res[0]
512	hall_doc = hall_brain.getObject().getContent()
513	else:
514	return info
515	info['hall_title'] = hall_brain.Title
516	info['maintenance_code'] = hall_doc.maintenance_code
517	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
518	batch_doc = None
519	for brain in res:
520	if brain.id.startswith(info['maintenance_code']):
521	batch_doc = brain.getObject().getContent()
522	break
523	if batch_doc is None:
524	info['maintenance_fee'] = None
525	else:
526	info['maintenance_fee'] = batch_doc.cost
527	return info
528	###)
529
530	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
531	def deleteAllCourses(self,department="All"):
532	''' delete the courses'''
533	pm = self.portal_membership
534	member = pm.getAuthenticatedMember()
535
536	if str(member) not in ("henrik","joachim"):
537	return "not possible"
538	if department == "All":
539	res = self.portal_catalog({'meta_type': 'Department'})
540	if len(res) < 1:
541	return "No Departments found"
542
543	deleted = []
544	for dep in res:
545	cf = dep.getObject().courses
546	if cf:
547	cf.manage_delObjects(ids=cf.objectIds())
548	deleted.append("deleted Courses in %s" % dep.getId)
549	return "\r".join(deleted)
550	###)
551
552	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
553	def getLogfileLines(self,filename="event.log",numlines=20):
554	"""Get last NUMLINES lines of logfile FILENAME.
555
556	Return last lines' of a file in the instances logfile directory as
557	a list. The number of returned lines equals `numlines' or less. If
558	less than `numlines' lines are available, the whole file ist
559	returned. If the file can not be opened or some other error
560	occurs, empty list is returend.
561	"""
562	result = []
563	lines_hit = 0
564
565	# We only handle files in instances' log directory...
566	logpath = os.path.join(i_home, "log")
567	filename = str(os.path.abspath( os.path.join( logpath, filename )))
568	if not filename.startswith( logpath ):
569	# Attempt to access file outside log-dir...
570	return []
571
572	try:
573	fd = file( filename, "rb" )
574	except IOError:
575	return []
576	if not fd:
577	return []
578
579	if os.linesep == None:
580	linesep = '\n'
581	else:
582	linesep = os.linesep
583
584	# Try to find 'numlines' times a lineseparator, searching from end
585	# and moving to the beginning of file...
586	fd.seek( 0, 2) # Move to end of file...
587	while lines_hit < numlines:
588	if fd.read(1) == linesep[-1]: # This moves filedescriptor
589	# one step forward...
590	lines_hit += 1
591	try:
592	fd.seek( -2, 1) # Go two bytes back from current pos...
593	except IOError:
594	# We cannot go back two bytes. Maybe the file is too small...
595	break
596	fd.seek(2,1)
597
598	# Read all lines from current position...
599	result = fd.readlines()
600	# Remove line endings...
601	result = [x.strip() for x in result]
602	fd.close()
603	return result
604	###)
605
606	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
607	def getCallbacksFromLog(self,filename):
608	"""fix Online Payment Transactions from Z2.log entries"""
609	import transaction
610	import random
611	from cgi import parse_qs
612	from urlparse import urlparse
613	#from pdb import set_trace
614	wftool = self.portal_workflow
615	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
616	students_folder = self.portal_url.getPortalObject().campus.students
617	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
618	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
619	s += r'(?P<intersw>.)" "(?P<agent>.)"'
620	data = re.compile(s)
621	start = True
622	tr_count = 1
623	total = 0
624	#name = 'pume_results'
625	#name = 'epaymentsuccessful_z2log2'
626	name = filename
627	no_import = []
628	imported = []
629	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
630	try:
631	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
632	except:
633	logger.error('Error reading %s' % name)
634	return
635	tas = []
636	for line in transactions:
637	dict = {}
638	items = data.search(line)
639	dict['idict'] = idict = items.groupdict()
640	#print idict
641	#from pdb import set_trace;set_trace()
642	urlparsed = urlparse(idict['get'][4:])
643	#print urlparsed
644	path = urlparsed[2].split('/')
645	dict['student_id'] = student_id = path[8]
646	dict['payment_id'] = payment_id = path[10]
647	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
648	tas.append(dict)
649	tr_count += 1
650	return tas
651	###)
652
653	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
654	def importOnlinePaymentTransactions(self):
655	"""load Online Payment Transactions from CSV values"""
656	import transaction
657	import random
658	#from pdb import set_trace
659	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
660	opt = self.online_payments_import
661	students_folder = self.portal_url.getPortalObject().campus.students
662	start = True
663	tr_count = 1
664	total = 0
665	#name = 'pume_results'
666	name = 'OnlineTransactions'
667	no_import = []
668	imported = []
669	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
670	try:
671	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
672	except:
673	logger.error('Error reading %s.csv' % name)
674	return
675	for pay_transaction in transactions:
676	if start:
677	start = False
678	logger.info('Start loading from %s.csv' % name)
679	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
680	no_import.append('%s,"Error"' % s)
681	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
682	format_error = format + ',"%(Error)s"'
683	data = {}
684
685	# format of the first file sent by Tayo
686	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
687	#data['student_id'] = student_id = pay_transaction['Payer ID']
688	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
689	#data['response_code'] = response_code = pay_transaction['Resp Code']
690	#data['amount'] = amount = pay_transaction['Amount']
691
692	# format of the second file sent by Tayo
693	#data['datetime'] = date = 0
694	#data['student_id'] = student_id = pay_transaction['Payer ID']
695	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
696	#data['response_code'] = response_code = '00'
697	#data['amount'] = amount = pay_transaction['Amount']
698
699	# format of the third file sent by Kehinde
700	data['datetime'] = date = 0
701	data['student_id'] = student_id = pay_transaction['customer_id']
702	data['order_id'] = order_id = pay_transaction['merchant_reference']
703	data['response_code'] = response_code = '00'
704	data['amount'] = amount = pay_transaction['Amount']
705
706	dup = False
707	if response_code == "12":
708	continue
709	try:
710	opt.addRecord(**data)
711	except ValueError:
712	dup = True
713	#from pdb import set_trace;set_trace()
714	if dup:
715	if response_code == "00":
716	try:
717	opt.modifyRecord(**data)
718	except:
719	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
720	continue
721	else:
722	pay_transaction['Error'] = "Duplicate order_id"
723	no_import.append( format_error % pay_transaction)
724	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
725	continue
726	tr_count += 1
727	if tr_count > 1000:
728	if len(no_import) > 0:
729	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
730	'\n'.join(no_import) + '\n')
731	no_import = []
732	em = '%d transactions committed\n' % (tr_count)
733	transaction.commit()
734	regs = []
735	logger.info(em)
736	total += tr_count
737	tr_count = 0
738	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
739	'\n'.join(no_import))
740	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
741	###)
742
743
744	student_field2types = { ###(
745	'StudentApplication':
746	{'id': 'application',
747	'title': 'Application Data',
748	'wf_transition_return': 'close',
749	'wf_transition_admit': 'remain',
750	'fields':
751	('jamb_reg_no',
752	'entry_mode',
753	'entry_session',
754	'jamb_score',
755	'app_email',
756	'jamb_age',
757	'jamb_state',
758	'jamb_lga',
759	)
760	},
761	#'StudentPume':
762	# {'id': 'pume',
763	# 'title': 'Pume Data',
764	# 'wf_transition_return': 'close',
765	# 'wf_transition_admit': 'close',
766	# 'fields':
767	# ('pume_score',
768	# )
769	# },
770	'StudentClearance':
771	{'id': 'clearance',
772	'title': 'Clearance/Eligibility Record',
773	'wf_transition_return': 'close',
774	'wf_transition_admit': 'remain',
775	'fields':
776	('matric_no',
777	'nationality',
778	'lga',
779	'birthday',
780	)
781	},
782	'StudentPersonal':
783	{'id': 'personal',
784	'title': 'Personal Data',
785	'wf_transition_return': 'open',
786	'wf_transition_admit': 'remain',
787	'fields':
788	('firstname',
789	'middlename',
790	'lastname',
791	'sex',
792	'email',
793	'phone',
794	'perm_address',
795	)
796	},
797	'StudentStudyCourse':
798	{'id': 'study_course',
799	'title': 'Study Course',
800	'wf_transition_return': 'open',
801	'wf_transition_admit': 'remain',
802	'fields':
803	('study_course',
804	'current_level',
805	'current_session',
806	'current_mode',
807	'current_verdict',
808	)
809	},
810
811	'PaymentsFolder':
812	{'id': 'payments',
813	'title': 'Payments',
814	'wf_transition_return': 'open',
815	'wf_transition_admit': 'open',
816	'fields':
817	()
818	},
819	}
820	###)
821
822
823	security.declareProtected(ModifyPortalContent,'importStudent') ###(
824	def importStudent(self,mapping):
825	"create a students record due import"
826	logger = logging.getLogger('WAeUPTool.importStudent')
827	students_folder = self.portal_url.getPortalObject().campus.students
828	jamb_reg_no = mapping.get('jamb_reg_no',None)
829	if jamb_reg_no:
830	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
831	if res:
832	return '',"jamb_reg_no exists"
833	matric_no = mapping.get('matric_no',None)
834	if matric_no:
835	res = self.students_catalog(matric_no = matric_no)
836	if res:
837	return '',"matric_no exists"
838	sid = self.waeup_tool.generateStudentId('?')
839	students_folder.invokeFactory('Student', sid)
840	student_obj = getattr(students_folder,sid)
841	f2t = self.student_field2types
842	d = {}
843	d['jamb_sex'] = 'M'
844	if mapping.get('sex'):
845	d['jamb_sex'] = 'F'
846	transition = mapping.get('reg_transition','admit')
847	if transition not in ('admit','return'):
848	return '',"no valid transition provided"
849	for pt in f2t.keys():
850	student_obj.invokeFactory(pt,f2t[pt]['id'])
851	sub_obj = getattr(student_obj,f2t[pt]['id'])
852	sub_doc = sub_obj.getContent()
853	d['Title'] = f2t[pt]['title']
854	for field in f2t[pt]['fields']:
855	d[field] = mapping.get(field,'')
856	if pt == "StudyCourse":
857	for von,zu in (('entry_mode','current_mode'),
858	('entry_session','current_session')):
859	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
860	d[zu] = mapping[von]
861	sub_doc.edit(mapping = d)
862
863	#import pdb;pdb.set_trace()
864	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
865	if new_state != "remain":
866	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
867	self.portal_workflow.doActionFor(student_obj,transition)
868	student_obj.manage_setLocalRoles(sid, ['Owner',])
869	return sid,''
870	###)
871
872	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
873	def importEditStudent(self,mapping):
874	"edit a students record due import"
875	logger = logging.getLogger('WAeUPTool.importEditStudent')
876	students_folder = self.portal_url.getPortalObject().campus.students
877	sid = mapping.get('id',None)
878	jamb_reg_no = mapping.get('jamb_reg_no',None)
879	matric_no = mapping.get('matric_no',None)
880	editable_keys = mapping.keys()
881	if sid:
882	res = self.students_catalog(id = sid)
883	if not res:
884	return '',"no student with id %s" % sid
885	elif matric_no and res[0].matric_no and\
886	matric_no != res[0].matric_no:
887	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
888	elif jamb_reg_no and res[0].jamb_reg_no and\
889	jamb_reg_no != res[0].jamb_reg_no:
890	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
891	elif jamb_reg_no:
892	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
893	if not res:
894	return '',"no student with jamb_reg_no %s" % jamb_reg_no
895	editable_keys.remove('jamb_reg_no')
896	elif matric_no:
897	res = self.students_catalog(matric_no = matric_no)
898	if not res:
899	return '',"no student with matric_no %s" % matric_no
900	editable_keys.remove('matric_no')
901
902	# included only to change wf state from admitted to returning
903	if res[0].review_state != 'admitted':
904	return '%s' % res[0].id ,"student is not in state admitted"
905	# end inclusion
906
907	sid = res[0].id
908	student_obj = getattr(students_folder,sid)
909	f2t = self.student_field2types
910	d = {}
911	#import pdb;pdb.set_trace()
912	any_change = False
913	for pt in f2t.keys():
914	if pt == "student_application":
915	d['jamb_sex'] = 'M'
916	if mapping.get('sex'):
917	d['jamb_sex'] = 'F'
918	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
919	if intersect:
920	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
921	if sub_obj is None:
922	try:
923	student_obj.invokeFactory(pt,f2t[pt]['id'])
924	except:
925	continue
926	sub_obj = getattr(student_obj,f2t[pt]['id'])
927	d['Title'] = f2t[pt]['title']
928	sub_doc = sub_obj.getContent()
929	for field in intersect:
930	changed = False
931	if getattr(sub_doc,field) != mapping.get(field,''):
932	any_change = True
933	changed = True
934	d[field] = mapping.get(field,'')
935	if changed:
936	sub_doc.edit(mapping = d)
937
938
939	# included only to change wf state from admitted to returning
940	if res[0].review_state == 'admitted':
941	new_state = f2t[pt]['wf_transition_return']
942	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
943	if sub_obj and new_state != "remain":
944	try:
945	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
946	except:
947	#logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
948	pass
949	if res[0].review_state == 'admitted':
950	wfaction = 'return'
951	try:
952	self.portal_workflow.doActionFor(student_obj,wfaction)
953	logger.info('%s, wf state changed' % sid)
954	any_change = True
955	except:
956	logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
957	pass
958	# end inclusion
959
960
961	if any_change:
962	return sid,''
963	else:
964	return sid,'not modified'
965	###)
966
967
968	security.declareProtected(ModifyPortalContent,"importData")###(
969	def importData(self,filename,name,edit=False):
970	"""load data from CSV values"""
971	import transaction
972	import random
973
974	pm = self.portal_membership
975	member = pm.getAuthenticatedMember()
976
977	logger = logging.getLogger('WAeUPTool.importData')
978	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
979	students_folder = self.portal_url.getPortalObject().campus.students
980	start = True
981	tr_count = 1
982	total_imported = 0
983	total_not_imported = 0
984	total = 0
985	iname = "import_%s" % name
986	stool = getToolByName(self, 'portal_schemas')
987	ltool = getToolByName(self, 'portal_layouts')
988	schema = stool._getOb(iname)
989	if schema is None:
990	em = 'No such schema %s' % iname
991	logger.error('No such schema %s' % iname)
992	return em
993	layout = ltool._getOb(iname)
994	if layout is None:
995	em = 'No such layout %s' % iname
996	logger.error(em)
997	return em
998	validators = {}
999	for widget in layout.keys():
1000	validators[widget] = layout[widget].validate
1001	if edit:
1002	importer_name = "importEdit%s" % name.capitalize()
1003	else:
1004	importer_name = "import%s" % name.capitalize()
1005	importer = getattr(self, '%s' % importer_name,None)
1006	if importer is None:
1007	em = 'No importer function %s' % importer_name
1008	logger.error(em)
1009	return em
1010	not_imported = []
1011	imported = []
1012	try:
1013	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1014	except:
1015	em = 'Error reading %s.csv' % filename
1016	logger.error(em)
1017	return em
1018	for item in items:
1019	if start:
1020	start = False
1021	adapters = [MappingStorageAdapter(schema, item)]
1022	dm = DataModel(item, adapters,context=self)
1023	logger.info('%s starts import from %s.csv' % (member,filename))
1024	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1025	diff2schema = set(import_keys).difference(set(schema.keys()))
1026	diff2layout = set(import_keys).difference(set(layout.keys()))
1027	if diff2schema:
1028	em = "not ignorable key(s) %s found in heading" % diff2schema
1029	return em
1030	s = ','.join(['"%s"' % fn for fn in import_keys])
1031	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1032	s = '"id",' + s
1033	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1034	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1035	format_error = format + ',"%(Error)s"'
1036	format = '"%(id)s",'+ format
1037	ds = DataStructure(data=item,datamodel=dm)
1038	error_string = ""
1039	for k in import_keys:
1040	if not validators[k](ds):
1041	error_string += " %s : %s" % (k,ds.getError(k))
1042	if not error_string:
1043	item.update(dm)
1044	item['id'],error = importer(item)
1045	if error:
1046	error_string += error
1047	if error_string:
1048	item['Error'] = error_string
1049	not_imported.append(format_error % item)
1050	total_not_imported += 1
1051	else:
1052	em = format % item
1053	imported.append(em)
1054	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1055	tr_count += 1
1056	total_imported += 1
1057	total += 1
1058	if total_imported and not total_imported % 100:
1059	transaction.commit()
1060	if len(not_imported) > 0:
1061	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1062	'\n'.join(not_imported) + '\n')
1063	not_imported = []
1064	if len(imported) > 0:
1065	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1066	'\n'.join(imported) + '\n')
1067	imported = []
1068	em = '%d transactions committed\n' % (tr_count)
1069	regs = []
1070	logger.info(em)
1071	tr_count = 0
1072	if len(imported) > 0:
1073	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1074	'\n'.join(imported))
1075	if len(not_imported) > 0:
1076	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1077	'\n'.join(not_imported))
1078	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1079	logger.info(em)
1080	return em
1081	###)
1082
1083	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: