Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1958

Last change on this file since 1958 was 1921, checked in by joachim, 18 years ago
reopen application object for loading foto
Property svn:keywords set to `Id`
File size: 42.7 KB

Line
1	3#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1921 2007-06-18 09:28:04Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app = student.application
215	app_doc = app.getContent()
216	#clear = student.clearance
217	#clear_doc = clear.getContent()
218	#matric_no = clear_doc.matric_no.upper()
219	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
220	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
221	#import pdb;pdb.set_trace()
222	if os.path.exists(picture1):
223	file = open(picture1)
224	elif os.path.exists(picture2):
225	file = open(picture2)
226	else:
227	return "passport picture not found %s" % picture1
228	reopened = False
229	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
230	self.portal_workflow.doActionFor(app,'open')
231	reopened = True
232	outfile = file.read()
233	app_doc.manage_addFile('passport',
234	file=outfile,
235	title="%s.jpg" % filename)
236	if reopened:
237	self.portal_workflow.doActionFor(app,'close')
238	return "successfully loaded passport picture"
239	###)
240
241	security.declareProtected(ModifyPortalContent,'createOne') ###(
242	def createOne(self,students_folder,student_brain,letter,commit=False):
243	sid = self.waeup_tool.generateStudentId(letter)
244	students_folder.invokeFactory('Student', sid)
245	student = getattr(students_folder,sid)
246	self.portal_workflow.doActionFor(student,'return')
247	student.manage_setLocalRoles(sid, ['Owner',])
248	matric_no = student_brain.matric_no
249	jamb_reg_no = student_brain.Entryregno
250	self.students_catalog.addRecord(id = sid,
251	matric_no = matric_no,
252	jamb_reg_no = jamb_reg_no,
253	sex = student_brain.Sex == "F",
254	name = "%s %s %s" % (student_brain.Firstname,
255	student_brain.Middlename,
256	student_brain.Lastname)
257	)
258	if commit:
259	transaction.commit()
260	return sid,jamb_reg_no
261	###)
262
263	security.declareProtected(ModifyPortalContent,'addStudent') ###(
264	def addStudent(self,dict):
265	students_folder = self.portal_url.getPortalObject().campus.students
266	sid = self.waeup_tool.generateStudentId('?')
267	students_folder.invokeFactory('Student', sid)
268	student_obj = getattr(students_folder,sid)
269	f2t = self.student_field2types
270	#from pdb import set_trace; set_trace()
271	d = {}
272	d['jamb_sex'] = 'M'
273	if dict.get('sex'):
274	d['jamb_sex'] = 'F'
275
276	entry_session = dict.get('entry_session')
277	if entry_session == self.getSessionId()[-2:]:
278	wfaction = 'admit'
279	wft = 'wf_transition_admitted'
280	password = None
281	else:
282	wfaction = 'return'
283	wft = 'wf_transition_returning'
284	password = self.generatePassword()
285	self.makeStudentMember(sid,password)
286
287	for pt in f2t.keys():
288	student_obj.invokeFactory(pt,f2t[pt]['id'])
289	sub_obj = getattr(student_obj,f2t[pt]['id'])
290	sub_doc = sub_obj.getContent()
291	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
292	d['Title'] = f2t[pt]['title']
293	for field in f2t[pt]['fields']:
294	d[field] = dict.get(field,'')
295	sub_doc.edit(mapping = d)
296	new_state = f2t[pt][wft]
297	if new_state != "remain":
298	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
299	self.portal_workflow.doActionFor(student_obj,wfaction)
300	student_obj.manage_setLocalRoles(sid, ['Owner',])
301	return sid,password
302	###)
303
304	security.declarePublic('getCertificateBrain') ###(
305	def getCertificateBrain(self,cert_id):
306	"do it"
307	res = ZCatalog.searchResults(self.portal_catalog_real,
308	{'portal_type':"Certificate",
309	'id': cert_id})
310	if res:
311	return res[0]
312	return None
313	###)
314
315	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
316	def get_csv_filenames(self):
317	"do it"
318	files = [file for file in os.listdir("%s/import/" % (i_home))
319	if file.endswith('.csv') and file.find('imported') == -1]
320	return files
321	###)
322
323	security.declarePublic('findStudentByMatricelNo') ###(
324	def findStudentByMatricelNo(self,matric_no):
325	"do it"
326	res = ZCatalog.searchResults(self.portal_catalog_real,
327	{'portal_type':"StudentClearance",
328	'SearchableText': matric_no})
329	if res:
330	return res[0]
331	return None
332	###)
333
334	security.declarePublic('makeStudentMember') ###(
335	def makeStudentMember(self,sid,password='uNsEt'):
336	"""make the student a member"""
337	membership = self.portal_membership
338	membership.addMember(sid,
339	password ,
340	roles=('Member',
341	'Student',
342	),
343	domains='',
344	properties = {'memberareaCreationFlag': False,
345	'homeless': True},)
346	member = membership.getMemberById(sid)
347	self.portal_registration.afterAdd(member, sid, password, None)
348	#self.manage_setLocalRoles(sid, ['Owner',])
349	###)
350
351	security.declarePublic('makeStudentData') ###(
352	def makeStudentData(self,student_id,email=None,phone_nr=None):
353	"create Datastructure for a returning Student"
354	#import pdb;pdb.set_trace()
355	logger = logging.getLogger('WAeUPTool.makeStudentData')
356	students_folder = self.portal_url.getPortalObject().campus.students
357	#res = self.students_catalog(id=student_id)
358	#if res:
359	# st = res[0]
360	#res = self.returning_import(matric_no = st.matric_no)
361	res = self.returning_import(id = student_id)
362	if res:
363	student = res[0]
364	else:
365	logger.info('Id %s not found in returning_import' % student_id)
366	return
367	logger.info('%s creates data structure' % student_id)
368	s_results = self.results_import(matric_no = student.matric_no)
369	if s_results:
370	lnr = self.getLevelFromResultsCosCode(s_results)
371	level = "%d00" % lnr
372	verdict,eligible = self.getVerdict(s_results[0].Verdict)
373	if eligible:
374	level = "%d00" % (lnr + 1)
375	else:
376	logger.info('matric_no %s not found in results_import' % student.matric_no)
377	level = 0
378	verdict = 'N/A'
379	#student should not be allowed to perform this transition
380	#wftool = self.portal_workflow
381	#wftool.doActionFor(student,'return')
382	certcode_org = student.Coursemajorcode
383	certcode = makeCertificateCode(certcode_org)
384	certificate_brain = self.getCertificateBrain(certcode)
385	if not certificate_brain:
386	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
387	logger.info(em)
388	matric_no = student.matric_no
389	sid = student_id
390	student_obj = getattr(students_folder,sid)
391	student_obj.invokeFactory('StudentApplication','application')
392	application = student_obj.application
393	self.portal_workflow.doActionFor(application,'open',dest_container=application)
394	da = {'Title': 'Application Data'}
395	student_obj.invokeFactory('StudentPersonal','personal')
396	da['jamb_reg_no'] = student.Entryregno
397	em = self.getEntryMode(student.Entryregno)
398	## em = student.Mode_of_Entry
399	## if em in ('DIRECT', 'DIRECT ENTRY',):
400	## em = 'DE'
401	## elif em in ('U.M.E', 'UNE',):
402	## em = 'UME'
403	## elif not em:
404	## em = "unknown"
405	da['entry_mode'] = em
406	personal = student_obj.personal
407	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
408	dp = {'Title': 'Personal Data'}
409	student_obj.invokeFactory('StudentClearance','clearance')
410	clearance = student_obj.clearance
411	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
412	dc = {'Title': 'Clearance/Eligibility Record'}
413	dc['matric_no'] = matric_no
414	state = student.State
415	lga = student.LGA
416	if state and lga:
417	lga = state + ' / ' + lga
418	else:
419	lga = "None"
420	da['jamb_lga'] = dc['lga'] = lga
421	da['app_email'] = dp['email'] = email
422	da['app_mobile'] = dp['phone'] = phone_nr
423	dp['firstname'] = student.Firstname
424	dp['middlename'] = student.Middlename
425	dp['lastname'] = student.Lastname
426	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
427	da['jamb_sex'] = student.Sex
428	dp['sex'] = student.Sex == 'F'
429	dp['perm_address'] = student.Permanent_Address
430	application.getContent().edit(mapping=da)
431	self.portal_workflow.doActionFor(application,'close',dest_container=application)
432	personal.getContent().edit(mapping=dp)
433	clearance.getContent().edit(mapping=dc)
434	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
435	## catd = {}
436	## catd['id'] = sid
437	## catd['entry_mode']= da['entry_mode']
438	## catd['matric_no'] = matric_no
439	## catd['jamb_reg_no'] = da['jamb_reg_no']
440	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
441	## catd['sex'] = dp['sex']
442	## catd['level'] = level
443	## catd['verdict'] = verdict
444	## if certificate_brain:
445	## cpath = certificate_brain.getPath().split('/')
446	## catd['faculty'] = cpath[-4]
447	## catd['department'] = cpath[-3]
448	## catd['course'] = certcode
449	## self.students_catalog.modifyRecord(**catd)
450	#
451	# Study Course
452	#
453	student_obj.invokeFactory('StudentStudyCourse','study_course')
454	studycourse = student_obj.study_course
455	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
456	dsc = {}
457	dsc['study_course'] = certcode
458	dsc['current_level'] = level
459	dsc['current_verdict'] = verdict
460	dsc['current_mode'] = em
461	dsc['current_session'] = '06'
462	studycourse.getContent().edit(mapping=dsc)
463	#
464	# Level
465	#
466	## l = getattr(studycourse,level,None)
467	## if l is None:
468	## studycourse.invokeFactory('StudentStudyLevel', level)
469	## l = getattr(studycourse, level)
470	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
471	## l.getContent().edit(mapping={'Title': "Level %s" % level})
472	###)
473
474	security.declarePublic('makeStudentLevel') ###(
475	def makeStudentLevel(self,student_id):
476	"create the StudyLevel for a returning Student"
477	#import pdb;pdb.set_trace()
478	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
479	students_folder = self.portal_url.getPortalObject().campus.students
480	res = self.students_catalog(id=student_id)
481	if res:
482	st = res[0]
483	course = st.course
484	matric_no = st.matric_no
485	level = st.level
486	res = self.results_import(matric_no = matric_no)
487	if res:
488	results = res
489	logger.info('%s creating Level %s' % (student_id,level))
490	#
491	# Level
492	#
493	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
494	studycourse = getattr(student_obj,"study_course",None)
495	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
496	l = getattr(studycourse,level,None)
497	if l is None:
498	studycourse.invokeFactory('StudentStudyLevel', level)
499	l = getattr(studycourse, level)
500	self.portal_workflow.doActionFor(l,'open',dest_container=l)
501	l.getContent().edit(mapping={'Title': "Level %s" % level})
502	###)
503
504	security.declarePublic('getAccommodationInfo') ###(
505	def getAccommodationInfo(self,bed):
506	"""return Accommodation Info"""
507	info = {}
508	hall,block,room,letter = bed.split('_')
509	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
510	if res and len(res) == 1:
511	hall_brain = res[0]
512	hall_doc = hall_brain.getObject().getContent()
513	else:
514	return info
515	info['hall_title'] = hall_brain.Title
516	info['maintenance_code'] = hall_doc.maintenance_code
517	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
518	batch_doc = None
519	for brain in res:
520	if brain.id.startswith(info['maintenance_code']):
521	batch_doc = brain.getObject().getContent()
522	break
523	if batch_doc is None:
524	info['maintenance_fee'] = None
525	else:
526	info['maintenance_fee'] = batch_doc.cost
527	return info
528	###)
529
530	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
531	def deleteAllCourses(self,department="All"):
532	''' delete the courses'''
533	pm = self.portal_membership
534	member = pm.getAuthenticatedMember()
535
536	if str(member) not in ("henrik","joachim"):
537	return "not possible"
538	if department == "All":
539	res = self.portal_catalog({'meta_type': 'Department'})
540	if len(res) < 1:
541	return "No Departments found"
542
543	deleted = []
544	for dep in res:
545	cf = dep.getObject().courses
546	if cf:
547	cf.manage_delObjects(ids=cf.objectIds())
548	deleted.append("deleted Courses in %s" % dep.getId)
549	return "\r".join(deleted)
550	###)
551
552	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
553	def getLogfileLines(self,filename="event.log",numlines=20):
554	"""Get last NUMLINES lines of logfile FILENAME.
555
556	Return last lines' of a file in the instances logfile directory as
557	a list. The number of returned lines equals `numlines' or less. If
558	less than `numlines' lines are available, the whole file ist
559	returned. If the file can not be opened or some other error
560	occurs, empty list is returend.
561	"""
562	result = []
563	lines_hit = 0
564
565	# We only handle files in instances' log directory...
566	logpath = os.path.join(i_home, "log")
567	filename = str(os.path.abspath( os.path.join( logpath, filename )))
568	if not filename.startswith( logpath ):
569	# Attempt to access file outside log-dir...
570	return []
571
572	try:
573	fd = file( filename, "rb" )
574	except IOError:
575	return []
576	if not fd:
577	return []
578
579	if os.linesep == None:
580	linesep = '\n'
581	else:
582	linesep = os.linesep
583
584	# Try to find 'numlines' times a lineseparator, searching from end
585	# and moving to the beginning of file...
586	fd.seek( 0, 2) # Move to end of file...
587	while lines_hit < numlines:
588	if fd.read(1) == linesep[-1]: # This moves filedescriptor
589	# one step forward...
590	lines_hit += 1
591	try:
592	fd.seek( -2, 1) # Go two bytes back from current pos...
593	except IOError:
594	# We cannot go back two bytes. Maybe the file is too small...
595	break
596	fd.seek(2,1)
597
598	# Read all lines from current position...
599	result = fd.readlines()
600	# Remove line endings...
601	result = [x.strip() for x in result]
602	fd.close()
603	return result
604	###)
605
606	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
607	def getCallbacksFromLog(self,filename):
608	"""fix Online Payment Transactions from Z2.log entries"""
609	import transaction
610	import random
611	from cgi import parse_qs
612	from urlparse import urlparse
613	#from pdb import set_trace
614	wftool = self.portal_workflow
615	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
616	students_folder = self.portal_url.getPortalObject().campus.students
617	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
618	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
619	s += r'(?P<intersw>.)" "(?P<agent>.)"'
620	data = re.compile(s)
621	start = True
622	tr_count = 1
623	total = 0
624	#name = 'pume_results'
625	#name = 'epaymentsuccessful_z2log2'
626	name = filename
627	no_import = []
628	imported = []
629	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
630	try:
631	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
632	except:
633	logger.error('Error reading %s' % name)
634	return
635	tas = []
636	for line in transactions:
637	dict = {}
638	items = data.search(line)
639	dict['idict'] = idict = items.groupdict()
640	#print idict
641	#from pdb import set_trace;set_trace()
642	urlparsed = urlparse(idict['get'][4:])
643	#print urlparsed
644	path = urlparsed[2].split('/')
645	dict['student_id'] = student_id = path[8]
646	dict['payment_id'] = payment_id = path[10]
647	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
648	tas.append(dict)
649	tr_count += 1
650	return tas
651	###)
652
653	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
654	def importOnlinePaymentTransactions(self):
655	"""load Online Payment Transactions from CSV values"""
656	import transaction
657	import random
658	#from pdb import set_trace
659	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
660	opt = self.online_payments_import
661	students_folder = self.portal_url.getPortalObject().campus.students
662	start = True
663	tr_count = 1
664	total = 0
665	#name = 'pume_results'
666	name = 'OnlineTransactions'
667	no_import = []
668	imported = []
669	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
670	try:
671	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
672	except:
673	logger.error('Error reading %s.csv' % name)
674	return
675	for pay_transaction in transactions:
676	if start:
677	start = False
678	logger.info('Start loading from %s.csv' % name)
679	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
680	no_import.append('%s,"Error"' % s)
681	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
682	format_error = format + ',"%(Error)s"'
683	data = {}
684
685	# format of the first file sent by Tayo
686	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
687	#data['student_id'] = student_id = pay_transaction['Payer ID']
688	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
689	#data['response_code'] = response_code = pay_transaction['Resp Code']
690	#data['amount'] = amount = pay_transaction['Amount']
691
692	# format of the second file sent by Tayo
693	#data['datetime'] = date = 0
694	#data['student_id'] = student_id = pay_transaction['Payer ID']
695	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
696	#data['response_code'] = response_code = '00'
697	#data['amount'] = amount = pay_transaction['Amount']
698
699	# format of the third file sent by Kehinde
700	data['datetime'] = date = 0
701	data['student_id'] = student_id = pay_transaction['customer_id']
702	data['order_id'] = order_id = pay_transaction['merchant_reference']
703	data['response_code'] = response_code = '00'
704	data['amount'] = amount = pay_transaction['Amount']
705
706	dup = False
707	if response_code == "12":
708	continue
709	try:
710	opt.addRecord(**data)
711	except ValueError:
712	dup = True
713	#from pdb import set_trace;set_trace()
714	if dup:
715	if response_code == "00":
716	try:
717	opt.modifyRecord(**data)
718	except:
719	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
720	continue
721	else:
722	pay_transaction['Error'] = "Duplicate order_id"
723	no_import.append( format_error % pay_transaction)
724	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
725	continue
726	tr_count += 1
727	if tr_count > 1000:
728	if len(no_import) > 0:
729	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
730	'\n'.join(no_import) + '\n')
731	no_import = []
732	em = '%d transactions committed\n' % (tr_count)
733	transaction.commit()
734	regs = []
735	logger.info(em)
736	total += tr_count
737	tr_count = 0
738	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
739	'\n'.join(no_import))
740	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
741	###)
742
743
744	# all workflow transitions refer to students in state returning
745	student_field2types = { ###(
746	'StudentApplication':
747	{'id': 'application',
748	'title': 'Application Data',
749	'wf_transition_returning': 'close',
750	'wf_transition_admitted': 'remain',
751	'fields':
752	('jamb_reg_no',
753	'entry_mode',
754	'entry_session',
755	'jamb_score',
756	'app_email',
757	)
758	},
759	#'StudentPume':
760	# {'id': 'pume',
761	# 'title': 'Pume Data',
762	# 'wf_transition_returning': 'close',
763	# 'wf_transition_admitted': 'close',
764	# 'fields':
765	# ('pume_score',
766	# )
767	# },
768	'StudentClearance':
769	{'id': 'clearance',
770	'title': 'Clearance Data',
771	'wf_transition_returning': 'close',
772	'wf_transition_admitted': 'remain',
773	'fields':
774	('matric_no',
775	'nationality',
776	'lga',
777	'birthday',
778	)
779	},
780	'StudentPersonal':
781	{'id': 'personal',
782	'title': 'Personal Data',
783	'wf_transition_returning': 'open',
784	'wf_transition_admitted': 'remain',
785	'fields':
786	('firstname',
787	'middlename',
788	'lastname',
789	'sex',
790	'email',
791	'phone',
792	'perm_address',
793	)
794	},
795	'StudentStudyCourse':
796	{'id': 'study_course',
797	'title': 'Study Course',
798	'wf_transition_returning': 'open',
799	'wf_transition_admitted': 'remain',
800	'fields':
801	('study_course',
802	'current_level',
803	'current_session',
804	'current_mode',
805	'current_verdict',
806	)
807	},
808
809	'PaymentsFolder':
810	{'id': 'payments',
811	'title': 'Payments',
812	'wf_transition_returning': 'open',
813	'wf_transition_admitted': 'open',
814	'fields':
815	()
816	},
817	}
818	###)
819
820
821	security.declareProtected(ModifyPortalContent,'importStudent') ###(
822	def importStudent(self,mapping):
823	"create a students record due import"
824	logger = logging.getLogger('WAeUPTool.importStudent')
825	students_folder = self.portal_url.getPortalObject().campus.students
826	jamb_reg_no = mapping.get('jamb_reg_no',None)
827	if jamb_reg_no:
828	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
829	if res:
830	return '',"jamb_reg_no exists"
831	matric_no = mapping.get('matric_no',None)
832	if matric_no:
833	res = self.students_catalog(matric_no = matric_no)
834	if res:
835	return '',"matric_no exists"
836	sid = self.waeup_tool.generateStudentId('?')
837	students_folder.invokeFactory('Student', sid)
838	student_obj = getattr(students_folder,sid)
839	f2t = self.student_field2types
840	d = {}
841	d['jamb_sex'] = 'M'
842	if mapping.get('sex'):
843	d['jamb_sex'] = 'F'
844	for pt in f2t.keys():
845	student_obj.invokeFactory(pt,f2t[pt]['id'])
846	sub_obj = getattr(student_obj,f2t[pt]['id'])
847	sub_doc = sub_obj.getContent()
848	d['Title'] = f2t[pt]['title']
849	for field in f2t[pt]['fields']:
850	d[field] = mapping.get(field,'')
851	if pt == "StudyCourse":
852	for von,zu in (('entry_mode','current_mode'),
853	('entry_session','current_session')):
854	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
855	d[zu] = mapping[von]
856	sub_doc.edit(mapping = d)
857
858	new_state = f2t[pt]['wf_transition_admitted']
859	if new_state != "remain":
860	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
861	wfaction = 'admit'
862	self.portal_workflow.doActionFor(student_obj,wfaction)
863	student_obj.manage_setLocalRoles(sid, ['Owner',])
864	return sid,''
865	###)
866
867	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
868	def importEditStudent(self,mapping):
869	"edit a students record due import"
870	logger = logging.getLogger('WAeUPTool.importEditStudent')
871	students_folder = self.portal_url.getPortalObject().campus.students
872	sid = mapping.get('id',None)
873	jamb_reg_no = mapping.get('jamb_reg_no',None)
874	matric_no = mapping.get('matric_no',None)
875	editable_keys = mapping.keys()
876	if sid:
877	res = self.students_catalog(id = sid)
878	if not res:
879	return '',"no student with id %s" % sid
880	elif matric_no and res[0].matric_no and\
881	matric_no != res[0].matric_no:
882	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
883	elif jamb_reg_no and res[0].jamb_reg_no and\
884	jamb_reg_no != res[0].jamb_reg_no:
885	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
886	elif jamb_reg_no:
887	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
888	if not res:
889	return '',"no student with jamb_reg_no %s" % jamb_reg_no
890	editable_keys.remove('jamb_reg_no')
891	elif matric_no:
892	res = self.students_catalog(matric_no = matric_no)
893	if not res:
894	return '',"no student with matric_no %s" % matric_no
895	editable_keys.remove('matric_no')
896	sid = res[0].id
897	student_obj = getattr(students_folder,sid)
898	f2t = self.student_field2types
899	d = {}
900	#import pdb;pdb.set_trace()
901	any_change = False
902	for pt in f2t.keys():
903	if pt == "student_application":
904	d['jamb_sex'] = 'M'
905	if mapping.get('sex'):
906	d['jamb_sex'] = 'F'
907	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
908	if intersect:
909	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
910	if sub_obj is None:
911	try:
912	student_obj.invokeFactory(pt,f2t[pt]['id'])
913	except:
914	continue
915	sub_obj = getattr(student_obj,f2t[pt]['id'])
916	d['Title'] = f2t[pt]['title']
917	sub_doc = sub_obj.getContent()
918	for field in intersect:
919	changed = False
920	if getattr(sub_doc,field) != mapping.get(field,''):
921	any_change = True
922	changed = True
923	d[field] = mapping.get(field,'')
924	if changed:
925	sub_doc.edit(mapping = d)
926
927
928	# included only to change wf state from admitted to returning
929	new_state = f2t[pt]['wf_transition_returning']
930	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
931	if sub_obj and new_state != "remain":
932	try:
933	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
934	except:
935	#logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
936	pass
937	wfaction = 'return'
938	try:
939	self.portal_workflow.doActionFor(student_obj,wfaction)
940	logger.info('%s, wf state changed' % sid)
941	any_change = True
942	except:
943	logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
944	pass
945	# end inclusion
946
947
948	if any_change:
949	return sid,''
950	else:
951	return sid,'not modified'
952	###)
953
954
955	security.declareProtected(ModifyPortalContent,"importData")###(
956	def importData(self,filename,name,edit=False):
957	"""load data from CSV values"""
958	import transaction
959	import random
960
961	pm = self.portal_membership
962	member = pm.getAuthenticatedMember()
963
964	logger = logging.getLogger('WAeUPTool.importData')
965	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
966	students_folder = self.portal_url.getPortalObject().campus.students
967	start = True
968	tr_count = 1
969	total_imported = 0
970	total_not_imported = 0
971	total = 0
972	iname = "import_%s" % name
973	stool = getToolByName(self, 'portal_schemas')
974	ltool = getToolByName(self, 'portal_layouts')
975	schema = stool._getOb(iname)
976	if schema is None:
977	em = 'No such schema %s' % iname
978	logger.error('No such schema %s' % iname)
979	return em
980	layout = ltool._getOb(iname)
981	if layout is None:
982	em = 'No such layout %s' % iname
983	logger.error(em)
984	return em
985	validators = {}
986	for widget in layout.keys():
987	validators[widget] = layout[widget].validate
988	if edit:
989	importer_name = "importEdit%s" % name.capitalize()
990	else:
991	importer_name = "import%s" % name.capitalize()
992	importer = getattr(self, '%s' % importer_name,None)
993	if importer is None:
994	em = 'No importer function %s' % importer_name
995	logger.error(em)
996	return em
997	not_imported = []
998	imported = []
999	try:
1000	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1001	except:
1002	em = 'Error reading %s.csv' % filename
1003	logger.error(em)
1004	return em
1005	for item in items:
1006	if start:
1007	start = False
1008	adapters = [MappingStorageAdapter(schema, item)]
1009	dm = DataModel(item, adapters,context=self)
1010	logger.info('%s starts import from %s.csv' % (member,filename))
1011	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1012	diff2schema = set(import_keys).difference(set(schema.keys()))
1013	diff2layout = set(import_keys).difference(set(layout.keys()))
1014	if diff2schema:
1015	em = "not ignorable key(s) %s found in heading" % diff2schema
1016	return em
1017	s = ','.join(['"%s"' % fn for fn in import_keys])
1018	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1019	s = '"id",' + s
1020	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1021	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1022	format_error = format + ',"%(Error)s"'
1023	format = '"%(id)s",'+ format
1024	ds = DataStructure(data=item,datamodel=dm)
1025	error_string = ""
1026	for k in import_keys:
1027	if not validators[k](ds):
1028	error_string += " %s : %s" % (k,ds.getError(k))
1029	if not error_string:
1030	item.update(dm)
1031	item['id'],error = importer(item)
1032	if error:
1033	error_string += error
1034	if error_string:
1035	item['Error'] = error_string
1036	not_imported.append(format_error % item)
1037	total_not_imported += 1
1038	else:
1039	em = format % item
1040	imported.append(em)
1041	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1042	tr_count += 1
1043	total_imported += 1
1044	total += 1
1045	if total_imported and not total_imported % 100:
1046	transaction.commit()
1047	if len(not_imported) > 0:
1048	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1049	'\n'.join(not_imported) + '\n')
1050	not_imported = []
1051	if len(imported) > 0:
1052	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1053	'\n'.join(imported) + '\n')
1054	imported = []
1055	em = '%d transactions committed\n' % (tr_count)
1056	regs = []
1057	logger.info(em)
1058	tr_count = 0
1059	if len(imported) > 0:
1060	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1061	'\n'.join(imported))
1062	if len(not_imported) > 0:
1063	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1064	'\n'.join(not_imported))
1065	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1066	logger.info(em)
1067	return em
1068	###)
1069
1070	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: