Context navigation

WAeUPTool.py @ 17421

Last change on this file since 17421 was 1988, checked in by joachim, 17 years ago
merged r1979:r1987 from trunk
Property svn:keywords set to `Id`
File size: 43.3 KB

Line
1	3#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1988 2007-07-05 11:14:12Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app = student.application
215	app_doc = app.getContent()
216	#clear = student.clearance
217	#clear_doc = clear.getContent()
218	#matric_no = clear_doc.matric_no.upper()
219	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
220	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
221	#import pdb;pdb.set_trace()
222	if os.path.exists(picture1):
223	file = open(picture1)
224	elif os.path.exists(picture2):
225	file = open(picture2)
226	else:
227	return "passport picture not found %s" % picture1
228	reopened = False
229	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
230	self.portal_workflow.doActionFor(app,'open')
231	reopened = True
232	outfile = file.read()
233	app_doc.manage_addFile('passport',
234	file=outfile,
235	title="%s.jpg" % filename)
236	if reopened:
237	self.portal_workflow.doActionFor(app,'close')
238	return "successfully loaded passport picture"
239	###)
240
241	security.declareProtected(ModifyPortalContent,'createOne') ###(
242	def createOne(self,students_folder,student_brain,letter,commit=False):
243	sid = self.waeup_tool.generateStudentId(letter)
244	students_folder.invokeFactory('Student', sid)
245	student = getattr(students_folder,sid)
246	self.portal_workflow.doActionFor(student,'return')
247	student.manage_setLocalRoles(sid, ['Owner',])
248	matric_no = student_brain.matric_no
249	jamb_reg_no = student_brain.Entryregno
250	self.students_catalog.addRecord(id = sid,
251	matric_no = matric_no,
252	jamb_reg_no = jamb_reg_no,
253	sex = student_brain.Sex == "F",
254	name = "%s %s %s" % (student_brain.Firstname,
255	student_brain.Middlename,
256	student_brain.Lastname)
257	)
258	if commit:
259	transaction.commit()
260	return sid,jamb_reg_no
261	###)
262
263	security.declareProtected(ModifyPortalContent,'addStudent') ###(
264	def addStudent(self,dict):
265	students_folder = self.portal_url.getPortalObject().campus.students
266	sid = self.waeup_tool.generateStudentId('?')
267	students_folder.invokeFactory('Student', sid)
268	student_obj = getattr(students_folder,sid)
269	f2t = self.student_field2types
270	#from pdb import set_trace; set_trace()
271	d = {}
272	d['jamb_sex'] = 'M'
273	if dict.get('sex'):
274	d['jamb_sex'] = 'F'
275
276	entry_session = dict.get('entry_session')
277	if entry_session == self.getSessionId()[-2:]:
278	wfaction = 'admit'
279	wft = 'wf_transition_admitted'
280	password = None
281	else:
282	wfaction = 'return'
283	wft = 'wf_transition_returning'
284	password = self.generatePassword()
285	self.makeStudentMember(sid,password)
286
287	for pt in f2t.keys():
288	student_obj.invokeFactory(pt,f2t[pt]['id'])
289	sub_obj = getattr(student_obj,f2t[pt]['id'])
290	sub_doc = sub_obj.getContent()
291	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
292	d['Title'] = f2t[pt]['title']
293	for field in f2t[pt]['fields']:
294	d[field] = dict.get(field,'')
295	sub_doc.edit(mapping = d)
296	new_state = f2t[pt][wft]
297	if new_state != "remain":
298	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
299	self.portal_workflow.doActionFor(student_obj,wfaction)
300	student_obj.manage_setLocalRoles(sid, ['Owner',])
301	return sid,password
302	###)
303
304	security.declarePublic('getCertificateBrain') ###(
305	def getCertificateBrain(self,cert_id):
306	"do it"
307	res = ZCatalog.searchResults(self.portal_catalog_real,
308	{'portal_type':"Certificate",
309	'id': cert_id})
310	if res:
311	return res[0]
312	return None
313	###)
314
315	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
316	def get_csv_filenames(self):
317	"do it"
318	files = [file for file in os.listdir("%s/import/" % (i_home))
319	if file.endswith('.csv') and file.find('imported') == -1]
320	return files
321	###)
322
323	security.declarePublic('findStudentByMatricelNo') ###(
324	def findStudentByMatricelNo(self,matric_no):
325	"do it"
326	res = ZCatalog.searchResults(self.portal_catalog_real,
327	{'portal_type':"StudentClearance",
328	'SearchableText': matric_no})
329	if res:
330	return res[0]
331	return None
332	###)
333
334	security.declarePublic('makeStudentMember') ###(
335	def makeStudentMember(self,sid,password='uNsEt'):
336	"""make the student a member"""
337	membership = self.portal_membership
338	membership.addMember(sid,
339	password ,
340	roles=('Member',
341	'Student',
342	),
343	domains='',
344	properties = {'memberareaCreationFlag': False,
345	'homeless': True},)
346	member = membership.getMemberById(sid)
347	self.portal_registration.afterAdd(member, sid, password, None)
348	#self.manage_setLocalRoles(sid, ['Owner',])
349	###)
350
351	security.declarePublic('makeStudentData') ###(
352	def makeStudentData(self,student_id,email=None,phone_nr=None):
353	"create Datastructure for a returning Student"
354	#import pdb;pdb.set_trace()
355	logger = logging.getLogger('WAeUPTool.makeStudentData')
356	students_folder = self.portal_url.getPortalObject().campus.students
357	#res = self.students_catalog(id=student_id)
358	#if res:
359	# st = res[0]
360	#res = self.returning_import(matric_no = st.matric_no)
361	res = self.returning_import(id = student_id)
362	if res:
363	student = res[0]
364	else:
365	logger.info('Id %s not found in returning_import' % student_id)
366	return
367	logger.info('%s creates data structure' % student_id)
368	s_results = self.results_import(matric_no = student.matric_no)
369	if s_results:
370	lnr = self.getLevelFromResultsCosCode(s_results)
371	level = "%d00" % lnr
372	verdict,eligible = self.getVerdict(s_results[0].Verdict)
373	if eligible:
374	level = "%d00" % (lnr + 1)
375	else:
376	logger.info('matric_no %s not found in results_import' % student.matric_no)
377	level = 0
378	verdict = 'N/A'
379	#student should not be allowed to perform this transition
380	#wftool = self.portal_workflow
381	#wftool.doActionFor(student,'return')
382	certcode_org = student.Coursemajorcode
383	certcode = makeCertificateCode(certcode_org)
384	certificate_brain = self.getCertificateBrain(certcode)
385	if not certificate_brain:
386	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
387	logger.info(em)
388	matric_no = student.matric_no
389	sid = student_id
390	student_obj = getattr(students_folder,sid)
391	student_obj.invokeFactory('StudentApplication','application')
392	application = student_obj.application
393	self.portal_workflow.doActionFor(application,'open',dest_container=application)
394	da = {'Title': 'Application Data'}
395	student_obj.invokeFactory('StudentPersonal','personal')
396	da['jamb_reg_no'] = student.Entryregno
397	em = self.getEntryMode(student.Entryregno)
398	## em = student.Mode_of_Entry
399	## if em in ('DIRECT', 'DIRECT ENTRY',):
400	## em = 'DE'
401	## elif em in ('U.M.E', 'UNE',):
402	## em = 'UME'
403	## elif not em:
404	## em = "unknown"
405	da['entry_mode'] = em
406	personal = student_obj.personal
407	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
408	dp = {'Title': 'Personal Data'}
409	student_obj.invokeFactory('StudentClearance','clearance')
410	clearance = student_obj.clearance
411	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
412	dc = {'Title': 'Clearance/Eligibility Record'}
413	dc['matric_no'] = matric_no
414	state = student.State
415	lga = student.LGA
416	if state and lga:
417	lga = state + ' / ' + lga
418	else:
419	lga = "None"
420	da['jamb_lga'] = dc['lga'] = lga
421	da['app_email'] = dp['email'] = email
422	da['app_mobile'] = dp['phone'] = phone_nr
423	dp['firstname'] = student.Firstname
424	dp['middlename'] = student.Middlename
425	dp['lastname'] = student.Lastname
426	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
427	da['jamb_sex'] = student.Sex
428	dp['sex'] = student.Sex == 'F'
429	dp['perm_address'] = student.Permanent_Address
430	application.getContent().edit(mapping=da)
431	self.portal_workflow.doActionFor(application,'close',dest_container=application)
432	personal.getContent().edit(mapping=dp)
433	clearance.getContent().edit(mapping=dc)
434	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
435	## catd = {}
436	## catd['id'] = sid
437	## catd['entry_mode']= da['entry_mode']
438	## catd['matric_no'] = matric_no
439	## catd['jamb_reg_no'] = da['jamb_reg_no']
440	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
441	## catd['sex'] = dp['sex']
442	## catd['level'] = level
443	## catd['verdict'] = verdict
444	## if certificate_brain:
445	## cpath = certificate_brain.getPath().split('/')
446	## catd['faculty'] = cpath[-4]
447	## catd['department'] = cpath[-3]
448	## catd['course'] = certcode
449	## self.students_catalog.modifyRecord(**catd)
450	#
451	# Study Course
452	#
453	student_obj.invokeFactory('StudentStudyCourse','study_course')
454	studycourse = student_obj.study_course
455	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
456	dsc = {}
457	dsc['study_course'] = certcode
458	dsc['current_level'] = level
459	dsc['current_verdict'] = verdict
460	dsc['current_mode'] = em
461	dsc['current_session'] = '06'
462	studycourse.getContent().edit(mapping=dsc)
463	#
464	# Level
465	#
466	## l = getattr(studycourse,level,None)
467	## if l is None:
468	## studycourse.invokeFactory('StudentStudyLevel', level)
469	## l = getattr(studycourse, level)
470	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
471	## l.getContent().edit(mapping={'Title': "Level %s" % level})
472	###)
473
474	security.declarePublic('makeStudentLevel') ###(
475	def makeStudentLevel(self,student_id):
476	"create the StudyLevel for a returning Student"
477	#import pdb;pdb.set_trace()
478	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
479	students_folder = self.portal_url.getPortalObject().campus.students
480	res = self.students_catalog(id=student_id)
481	if res:
482	st = res[0]
483	course = st.course
484	matric_no = st.matric_no
485	level = st.level
486	res = self.results_import(matric_no = matric_no)
487	if res:
488	results = res
489	logger.info('%s creating Level %s' % (student_id,level))
490	#
491	# Level
492	#
493	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
494	studycourse = getattr(student_obj,"study_course",None)
495	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
496	l = getattr(studycourse,level,None)
497	if l is None:
498	studycourse.invokeFactory('StudentStudyLevel', level)
499	l = getattr(studycourse, level)
500	self.portal_workflow.doActionFor(l,'open',dest_container=l)
501	l.getContent().edit(mapping={'Title': "Level %s" % level})
502	###)
503
504	security.declarePublic('getAccommodationInfo') ###(
505	def getAccommodationInfo(self,bed):
506	"""return Accommodation Info"""
507	info = {}
508	hall,block,room,letter = bed.split('_')
509	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
510	if res and len(res) == 1:
511	hall_brain = res[0]
512	hall_doc = hall_brain.getObject().getContent()
513	else:
514	return info
515	info['hall_title'] = hall_brain.Title
516	info['maintenance_code'] = hall_doc.maintenance_code
517	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
518	batch_doc = None
519	for brain in res:
520	if brain.id.startswith(info['maintenance_code']):
521	batch_doc = brain.getObject().getContent()
522	break
523	if batch_doc is None:
524	info['maintenance_fee'] = None
525	else:
526	info['maintenance_fee'] = batch_doc.cost
527	return info
528	###)
529
530	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
531	def deleteAllCourses(self,department="All"):
532	''' delete the courses'''
533	pm = self.portal_membership
534	member = pm.getAuthenticatedMember()
535
536	if str(member) not in ("henrik","joachim"):
537	return "not possible"
538	if department == "All":
539	res = self.portal_catalog({'meta_type': 'Department'})
540	if len(res) < 1:
541	return "No Departments found"
542
543	deleted = []
544	for dep in res:
545	cf = dep.getObject().courses
546	if cf:
547	cf.manage_delObjects(ids=cf.objectIds())
548	deleted.append("deleted Courses in %s" % dep.getId)
549	return "\r".join(deleted)
550	###)
551
552	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
553	def getLogfileLines(self,filename="event.log",numlines=20):
554	"""Get last NUMLINES lines of logfile FILENAME.
555
556	Return last lines' of a file in the instances logfile directory as
557	a list. The number of returned lines equals `numlines' or less. If
558	less than `numlines' lines are available, the whole file ist
559	returned. If the file can not be opened or some other error
560	occurs, empty list is returend.
561	"""
562	result = []
563	lines_hit = 0
564
565	# We only handle files in instances' log directory...
566	logpath = os.path.join(i_home, "log")
567	filename = str(os.path.abspath( os.path.join( logpath, filename )))
568	if not filename.startswith( logpath ):
569	# Attempt to access file outside log-dir...
570	return []
571
572	try:
573	fd = file( filename, "rb" )
574	except IOError:
575	return []
576	if not fd:
577	return []
578
579	if os.linesep == None:
580	linesep = '\n'
581	else:
582	linesep = os.linesep
583
584	# Try to find 'numlines' times a lineseparator, searching from end
585	# and moving to the beginning of file...
586	fd.seek( 0, 2) # Move to end of file...
587	while lines_hit < numlines:
588	if fd.read(1) == linesep[-1]: # This moves filedescriptor
589	# one step forward...
590	lines_hit += 1
591	try:
592	fd.seek( -2, 1) # Go two bytes back from current pos...
593	except IOError:
594	# We cannot go back two bytes. Maybe the file is too small...
595	break
596	fd.seek(2,1)
597
598	# Read all lines from current position...
599	result = fd.readlines()
600	# Remove line endings...
601	result = [x.strip() for x in result]
602	fd.close()
603	return result
604	###)
605
606	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
607	def getCallbacksFromLog(self,filename):
608	"""fix Online Payment Transactions from Z2.log entries"""
609	import transaction
610	import random
611	from cgi import parse_qs
612	from urlparse import urlparse
613	#from pdb import set_trace
614	wftool = self.portal_workflow
615	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
616	students_folder = self.portal_url.getPortalObject().campus.students
617	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
618	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
619	s += r'(?P<intersw>.)" "(?P<agent>.)"'
620	data = re.compile(s)
621	start = True
622	tr_count = 1
623	total = 0
624	#name = 'pume_results'
625	#name = 'epaymentsuccessful_z2log2'
626	name = filename
627	no_import = []
628	imported = []
629	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
630	try:
631	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
632	except:
633	logger.error('Error reading %s' % name)
634	return
635	tas = []
636	for line in transactions:
637	dict = {}
638	items = data.search(line)
639	dict['idict'] = idict = items.groupdict()
640	#print idict
641	#from pdb import set_trace;set_trace()
642	urlparsed = urlparse(idict['get'][4:])
643	#print urlparsed
644	path = urlparsed[2].split('/')
645	dict['student_id'] = student_id = path[8]
646	dict['payment_id'] = payment_id = path[10]
647	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
648	tas.append(dict)
649	tr_count += 1
650	return tas
651	###)
652
653	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
654	def importOnlinePaymentTransactions(self):
655	"""load Online Payment Transactions from CSV values"""
656	import transaction
657	import random
658	#from pdb import set_trace
659	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
660	opt = self.online_payments_import
661	students_folder = self.portal_url.getPortalObject().campus.students
662	start = True
663	tr_count = 1
664	total = 0
665	#name = 'pume_results'
666	name = 'OnlineTransactions'
667	no_import = []
668	imported = []
669	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
670	try:
671	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
672	except:
673	logger.error('Error reading %s.csv' % name)
674	return
675	for pay_transaction in transactions:
676	if start:
677	start = False
678	logger.info('Start loading from %s.csv' % name)
679	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
680	no_import.append('%s,"Error"' % s)
681	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
682	format_error = format + ',"%(Error)s"'
683	data = {}
684
685	# format of the first file sent by Tayo
686	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
687	#data['student_id'] = student_id = pay_transaction['Payer ID']
688	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
689	#data['response_code'] = response_code = pay_transaction['Resp Code']
690	#data['amount'] = amount = pay_transaction['Amount']
691
692	# format of the second file sent by Tayo
693	#data['datetime'] = date = 0
694	#data['student_id'] = student_id = pay_transaction['Payer ID']
695	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
696	#data['response_code'] = response_code = '00'
697	#data['amount'] = amount = pay_transaction['Amount']
698
699	# format of the third file sent by Kehinde
700	data['datetime'] = date = 0
701	data['student_id'] = student_id = pay_transaction['customer_id']
702	data['order_id'] = order_id = pay_transaction['merchant_reference']
703	data['response_code'] = response_code = '00'
704	data['amount'] = amount = pay_transaction['Amount']
705
706	dup = False
707	if response_code == "12":
708	continue
709	try:
710	opt.addRecord(**data)
711	except ValueError:
712	dup = True
713	#from pdb import set_trace;set_trace()
714	if dup:
715	if response_code == "00":
716	try:
717	opt.modifyRecord(**data)
718	except:
719	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
720	continue
721	else:
722	pay_transaction['Error'] = "Duplicate order_id"
723	no_import.append( format_error % pay_transaction)
724	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
725	continue
726	tr_count += 1
727	if tr_count > 1000:
728	if len(no_import) > 0:
729	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
730	'\n'.join(no_import) + '\n')
731	no_import = []
732	em = '%d transactions committed\n' % (tr_count)
733	transaction.commit()
734	regs = []
735	logger.info(em)
736	total += tr_count
737	tr_count = 0
738	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
739	'\n'.join(no_import))
740	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
741	###)
742
743
744	# all workflow transitions refer to students in state returning
745	student_field2types = { ###(
746	'StudentApplication':
747	{'id': 'application',
748	'title': 'Application Data',
749	'wf_transition_returning': 'close',
750	'wf_transition_admitted': 'remain',
751	'fields':
752	('jamb_reg_no',
753	'entry_mode',
754	'entry_session',
755	'jamb_score',
756	'app_email',
757	)
758	},
759	#'StudentPume':
760	# {'id': 'pume',
761	# 'title': 'Pume Data',
762	# 'wf_transition_returning': 'close',
763	# 'wf_transition_admitted': 'close',
764	# 'fields':
765	# ('pume_score',
766	# )
767	# },
768	'StudentClearance':
769	{'id': 'clearance',
770	'title': 'Clearance Data',
771	'wf_transition_returning': 'close',
772	'wf_transition_admitted': 'remain',
773	'fields':
774	('matric_no',
775	'nationality',
776	'lga',
777	'birthday',
778	)
779	},
780	'StudentPersonal':
781	{'id': 'personal',
782	'title': 'Personal Data',
783	'wf_transition_returning': 'open',
784	'wf_transition_admitted': 'remain',
785	'fields':
786	('firstname',
787	'middlename',
788	'lastname',
789	'sex',
790	'email',
791	'phone',
792	'perm_address',
793	)
794	},
795	'StudentStudyCourse':
796	{'id': 'study_course',
797	'title': 'Study Course',
798	'wf_transition_returning': 'open',
799	'wf_transition_admitted': 'remain',
800	'fields':
801	('study_course',
802	'current_level',
803	'current_session',
804	'current_mode',
805	'current_verdict',
806	)
807	},
808
809	'PaymentsFolder':
810	{'id': 'payments',
811	'title': 'Payments',
812	'wf_transition_returning': 'open',
813	'wf_transition_admitted': 'open',
814	'fields':
815	()
816	},
817	}
818	###)
819
820
821	security.declareProtected(ModifyPortalContent,'importStudent') ###(
822	def importStudent(self,mapping):
823	"create a students record due import"
824	logger = logging.getLogger('WAeUPTool.importStudent')
825	students_folder = self.portal_url.getPortalObject().campus.students
826	jamb_reg_no = mapping.get('jamb_reg_no',None)
827	if jamb_reg_no:
828	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
829	if res:
830	return '',"jamb_reg_no exists"
831	matric_no = mapping.get('matric_no',None)
832	if matric_no:
833	res = self.students_catalog(matric_no = matric_no)
834	if res:
835	return '',"matric_no exists"
836	sid = self.waeup_tool.generateStudentId('?')
837	students_folder.invokeFactory('Student', sid)
838	student_obj = getattr(students_folder,sid)
839	f2t = self.student_field2types
840	d = {}
841	d['jamb_sex'] = 'M'
842	if mapping.get('sex'):
843	d['jamb_sex'] = 'F'
844	for pt in f2t.keys():
845	student_obj.invokeFactory(pt,f2t[pt]['id'])
846	sub_obj = getattr(student_obj,f2t[pt]['id'])
847	sub_doc = sub_obj.getContent()
848	d['Title'] = f2t[pt]['title']
849	for field in f2t[pt]['fields']:
850	d[field] = mapping.get(field,'')
851	if pt == "StudyCourse":
852	for von,zu in (('entry_mode','current_mode'),
853	('entry_session','current_session')):
854	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
855	d[zu] = mapping[von]
856	sub_doc.edit(mapping = d)
857	transition = mapping.get('reg_transition','admitted')
858	if transition not in ('admitted','returning'):
859	transition = 'admitted'
860	#import pdb;pdb.set_trace()
861	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
862	if new_state != "remain":
863	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
864	wfaction = 'admit'
865	self.portal_workflow.doActionFor(student_obj,wfaction)
866	student_obj.manage_setLocalRoles(sid, ['Owner',])
867	return sid,''
868	###)
869
870	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
871	def importEditStudent(self,mapping):
872	"edit a students record due import"
873	logger = logging.getLogger('WAeUPTool.importEditStudent')
874	students_folder = self.portal_url.getPortalObject().campus.students
875	sid = mapping.get('id',None)
876	jamb_reg_no = mapping.get('jamb_reg_no',None)
877	matric_no = mapping.get('matric_no',None)
878	editable_keys = mapping.keys()
879	if sid:
880	res = self.students_catalog(id = sid)
881	if not res:
882	return '',"no student with id %s" % sid
883	elif matric_no and res[0].matric_no and\
884	matric_no != res[0].matric_no:
885	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
886	elif jamb_reg_no and res[0].jamb_reg_no and\
887	jamb_reg_no != res[0].jamb_reg_no:
888	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
889	elif jamb_reg_no:
890	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
891	if not res:
892	return '',"no student with jamb_reg_no %s" % jamb_reg_no
893	editable_keys.remove('jamb_reg_no')
894	elif matric_no:
895	res = self.students_catalog(matric_no = matric_no)
896	if not res:
897	return '',"no student with matric_no %s" % matric_no
898	editable_keys.remove('matric_no')
899
900	# included only to change wf state from admitted to returning
901	if res[0].review_state != 'admitted':
902	return '%s' % res[0].id ,"student is not in state admitted"
903	# end inclusion
904
905	sid = res[0].id
906	student_obj = getattr(students_folder,sid)
907	f2t = self.student_field2types
908	d = {}
909	#import pdb;pdb.set_trace()
910	any_change = False
911	for pt in f2t.keys():
912	if pt == "student_application":
913	d['jamb_sex'] = 'M'
914	if mapping.get('sex'):
915	d['jamb_sex'] = 'F'
916	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
917	if intersect:
918	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
919	if sub_obj is None:
920	try:
921	student_obj.invokeFactory(pt,f2t[pt]['id'])
922	except:
923	continue
924	sub_obj = getattr(student_obj,f2t[pt]['id'])
925	d['Title'] = f2t[pt]['title']
926	sub_doc = sub_obj.getContent()
927	for field in intersect:
928	changed = False
929	if getattr(sub_doc,field) != mapping.get(field,''):
930	any_change = True
931	changed = True
932	d[field] = mapping.get(field,'')
933	if changed:
934	sub_doc.edit(mapping = d)
935
936
937	# included only to change wf state from admitted to returning
938	if res[0].review_state == 'admitted':
939	new_state = f2t[pt]['wf_transition_returning']
940	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
941	if sub_obj and new_state != "remain":
942	try:
943	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
944	except:
945	#logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
946	pass
947	if res[0].review_state == 'admitted':
948	wfaction = 'return'
949	try:
950	self.portal_workflow.doActionFor(student_obj,wfaction)
951	logger.info('%s, wf state changed' % sid)
952	any_change = True
953	except:
954	logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
955	pass
956	# end inclusion
957
958
959	if any_change:
960	return sid,''
961	else:
962	return sid,'not modified'
963	###)
964
965
966	security.declareProtected(ModifyPortalContent,"importData")###(
967	def importData(self,filename,name,edit=False):
968	"""load data from CSV values"""
969	import transaction
970	import random
971
972	pm = self.portal_membership
973	member = pm.getAuthenticatedMember()
974
975	logger = logging.getLogger('WAeUPTool.importData')
976	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
977	students_folder = self.portal_url.getPortalObject().campus.students
978	start = True
979	tr_count = 1
980	total_imported = 0
981	total_not_imported = 0
982	total = 0
983	iname = "import_%s" % name
984	stool = getToolByName(self, 'portal_schemas')
985	ltool = getToolByName(self, 'portal_layouts')
986	schema = stool._getOb(iname)
987	if schema is None:
988	em = 'No such schema %s' % iname
989	logger.error('No such schema %s' % iname)
990	return em
991	layout = ltool._getOb(iname)
992	if layout is None:
993	em = 'No such layout %s' % iname
994	logger.error(em)
995	return em
996	validators = {}
997	for widget in layout.keys():
998	validators[widget] = layout[widget].validate
999	if edit:
1000	importer_name = "importEdit%s" % name.capitalize()
1001	else:
1002	importer_name = "import%s" % name.capitalize()
1003	importer = getattr(self, '%s' % importer_name,None)
1004	if importer is None:
1005	em = 'No importer function %s' % importer_name
1006	logger.error(em)
1007	return em
1008	not_imported = []
1009	imported = []
1010	try:
1011	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1012	except:
1013	em = 'Error reading %s.csv' % filename
1014	logger.error(em)
1015	return em
1016	for item in items:
1017	if start:
1018	start = False
1019	adapters = [MappingStorageAdapter(schema, item)]
1020	dm = DataModel(item, adapters,context=self)
1021	logger.info('%s starts import from %s.csv' % (member,filename))
1022	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1023	diff2schema = set(import_keys).difference(set(schema.keys()))
1024	diff2layout = set(import_keys).difference(set(layout.keys()))
1025	if diff2schema:
1026	em = "not ignorable key(s) %s found in heading" % diff2schema
1027	return em
1028	s = ','.join(['"%s"' % fn for fn in import_keys])
1029	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1030	s = '"id",' + s
1031	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1032	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1033	format_error = format + ',"%(Error)s"'
1034	format = '"%(id)s",'+ format
1035	ds = DataStructure(data=item,datamodel=dm)
1036	error_string = ""
1037	for k in import_keys:
1038	if not validators[k](ds):
1039	error_string += " %s : %s" % (k,ds.getError(k))
1040	if not error_string:
1041	item.update(dm)
1042	item['id'],error = importer(item)
1043	if error:
1044	error_string += error
1045	if error_string:
1046	item['Error'] = error_string
1047	not_imported.append(format_error % item)
1048	total_not_imported += 1
1049	else:
1050	em = format % item
1051	imported.append(em)
1052	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1053	tr_count += 1
1054	total_imported += 1
1055	total += 1
1056	if total_imported and not total_imported % 100:
1057	transaction.commit()
1058	if len(not_imported) > 0:
1059	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1060	'\n'.join(not_imported) + '\n')
1061	not_imported = []
1062	if len(imported) > 0:
1063	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1064	'\n'.join(imported) + '\n')
1065	imported = []
1066	em = '%d transactions committed\n' % (tr_count)
1067	regs = []
1068	logger.info(em)
1069	tr_count = 0
1070	if len(imported) > 0:
1071	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1072	'\n'.join(imported))
1073	if len(not_imported) > 0:
1074	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1075	'\n'.join(not_imported))
1076	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1077	logger.info(em)
1078	return em
1079	###)
1080
1081	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Context navigation

source: WAeUP_SRP/branches/joachim-azax-branch/WAeUPTool.py @ 17421

Download in other formats: