Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 2003

Last change on this file since 2003 was 2000, checked in by Henrik Bettermann, 17 years ago
ticket #245
Property svn:keywords set to `Id`
File size: 43.1 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 2000 2007-07-09 07:04:27Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	idlist = repository.objectIds()
149	for id in idlist:
150	docid, rev = repository._splitId(id)
151	if docid is None:
152	logger.info("invalid doc_id %s" % docid)
153	continue
154	nb_revs += 1
155	docids_d[docid] = None
156	if not pxtool_infos.has_key(docid):
157	unused_docids_d[docid] = None
158	ids_unused_revs_docids.append(id)
159	ids_unused_revs.append(id)
160	elif not pxtool_infos[docid].has_key(rev):
161	ids_unused_revs.append(id)
162	if len(ids_unused_revs) >= max:
163	repository.manage_delObjects(ids_unused_revs)
164	#import pdb;pdb.set_trace()
165	transaction.commit()
166	total += max
167	logger.info('removed %d total %d unused docids ' % (max,total))
168	anz = len(ids_unused_revs)
169	if anz > 0:
170	repository.manage_delObjects(ids_unused_revs)
171	transaction.commit()
172	total += anz
173	logger.info('finished removing %d unused docids ' % (total))
174
175
176	###)
177
178	security.declareProtected(ModifyPortalContent,'getCredential') ###(
179	def getCredential(self,student_id):
180	"return a student password"
181	student_entry = getattr(self.portal_directories.students,student_id,None)
182	if student_entry is None:
183	return None
184	return getattr(student_entry,"password","not set")
185	###)
186
187	security.declarePublic('checkPassword') ###(
188	def checkPassword(self,student_id,password):
189	"return a student password"
190	student_entry = getattr(self.portal_directories.students,student_id,None)
191	if student_entry is None:
192	return False
193	return getattr(student_entry,"password","not set") == password
194	###)
195
196	security.declarePublic('editPassword') ###(
197	def editPassword(self,student_id,password):
198	"edit a student password"
199	student_entry = getattr(self.portal_directories.students,student_id,None)
200	if student_entry is None:
201	return
202	setattr(student_entry,'password',password)
203	###)
204
205	security.declareProtected(View,'doCommit') ###(
206	def doCommit(self,logger=None):
207	"commit some transactions"
208	transaction.commit()
209	###)
210
211	security.declarePublic('loadStudentFoto') ###(
212	def loadStudentFoto(self,student,filename,folder):
213	"return a student passport picture"
214	app = student.application
215	app_doc = app.getContent()
216	#clear = student.clearance
217	#clear_doc = clear.getContent()
218	#matric_no = clear_doc.matric_no.upper()
219	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
220	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
221	#import pdb;pdb.set_trace()
222	if os.path.exists(picture1):
223	file = open(picture1)
224	elif os.path.exists(picture2):
225	file = open(picture2)
226	else:
227	return "passport picture not found %s" % picture1
228	reopened = False
229	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
230	self.portal_workflow.doActionFor(app,'open')
231	reopened = True
232	outfile = file.read()
233	app_doc.manage_addFile('passport',
234	file=outfile,
235	title="%s.jpg" % filename)
236	if reopened:
237	self.portal_workflow.doActionFor(app,'close')
238	return "successfully loaded passport picture"
239	###)
240
241	security.declareProtected(ModifyPortalContent,'createOne') ###(
242	def createOne(self,students_folder,student_brain,letter,commit=False):
243	sid = self.waeup_tool.generateStudentId(letter)
244	students_folder.invokeFactory('Student', sid)
245	student = getattr(students_folder,sid)
246	self.portal_workflow.doActionFor(student,'return')
247	student.manage_setLocalRoles(sid, ['Owner',])
248	matric_no = student_brain.matric_no
249	jamb_reg_no = student_brain.Entryregno
250	self.students_catalog.addRecord(id = sid,
251	matric_no = matric_no,
252	jamb_reg_no = jamb_reg_no,
253	sex = student_brain.Sex == "F",
254	name = "%s %s %s" % (student_brain.Firstname,
255	student_brain.Middlename,
256	student_brain.Lastname)
257	)
258	if commit:
259	transaction.commit()
260	return sid,jamb_reg_no
261	###)
262
263	security.declareProtected(ModifyPortalContent,'addStudent') ###(
264	def addStudent(self,dict):
265	students_folder = self.portal_url.getPortalObject().campus.students
266	sid = self.waeup_tool.generateStudentId('?')
267	students_folder.invokeFactory('Student', sid)
268	student_obj = getattr(students_folder,sid)
269	f2t = self.student_field2types
270	#from pdb import set_trace; set_trace()
271	d = {}
272	d['jamb_sex'] = 'M'
273	if dict.get('sex'):
274	d['jamb_sex'] = 'F'
275
276	entry_session = dict.get('entry_session')
277	if entry_session == self.getSessionId()[-2:]:
278	wfaction = 'admit'
279	wft = 'wf_transition_admit'
280	password = None
281	else:
282	wfaction = 'return'
283	wft = 'wf_transition_return'
284	password = self.generatePassword()
285	self.makeStudentMember(sid,password)
286
287	for pt in f2t.keys():
288	student_obj.invokeFactory(pt,f2t[pt]['id'])
289	sub_obj = getattr(student_obj,f2t[pt]['id'])
290	sub_doc = sub_obj.getContent()
291	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
292	d['Title'] = f2t[pt]['title']
293	for field in f2t[pt]['fields']:
294	d[field] = dict.get(field,'')
295	sub_doc.edit(mapping = d)
296	new_state = f2t[pt][wft]
297	if new_state != "remain":
298	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
299	self.portal_workflow.doActionFor(student_obj,wfaction)
300	student_obj.manage_setLocalRoles(sid, ['Owner',])
301	return sid,password
302	###)
303
304	security.declarePublic('getCertificateBrain') ###(
305	def getCertificateBrain(self,cert_id):
306	"do it"
307	res = ZCatalog.searchResults(self.portal_catalog_real,
308	{'portal_type':"Certificate",
309	'id': cert_id})
310	if res:
311	return res[0]
312	return None
313	###)
314
315	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
316	def get_csv_filenames(self):
317	"do it"
318	files = [file for file in os.listdir("%s/import/" % (i_home))
319	if file.endswith('.csv') and file.find('imported') == -1]
320	return files
321	###)
322
323	security.declarePublic('findStudentByMatricelNo') ###(
324	def findStudentByMatricelNo(self,matric_no):
325	"do it"
326	res = ZCatalog.searchResults(self.portal_catalog_real,
327	{'portal_type':"StudentClearance",
328	'SearchableText': matric_no})
329	if res:
330	return res[0]
331	return None
332	###)
333
334	security.declarePublic('makeStudentMember') ###(
335	def makeStudentMember(self,sid,password='uNsEt'):
336	"""make the student a member"""
337	membership = self.portal_membership
338	membership.addMember(sid,
339	password ,
340	roles=('Member',
341	'Student',
342	),
343	domains='',
344	properties = {'memberareaCreationFlag': False,
345	'homeless': True},)
346	member = membership.getMemberById(sid)
347	self.portal_registration.afterAdd(member, sid, password, None)
348	#self.manage_setLocalRoles(sid, ['Owner',])
349	###)
350
351	security.declarePublic('makeStudentData') ###(
352	def makeStudentData(self,student_id,email=None,phone_nr=None):
353	"create Datastructure for a returning Student"
354	#import pdb;pdb.set_trace()
355	logger = logging.getLogger('WAeUPTool.makeStudentData')
356	students_folder = self.portal_url.getPortalObject().campus.students
357	#res = self.students_catalog(id=student_id)
358	#if res:
359	# st = res[0]
360	#res = self.returning_import(matric_no = st.matric_no)
361	res = self.returning_import(id = student_id)
362	if res:
363	student = res[0]
364	else:
365	logger.info('Id %s not found in returning_import' % student_id)
366	return
367	logger.info('%s creates data structure' % student_id)
368	s_results = self.results_import(matric_no = student.matric_no)
369	if s_results:
370	lnr = self.getLevelFromResultsCosCode(s_results)
371	level = "%d00" % lnr
372	verdict,eligible = self.getVerdict(s_results[0].Verdict)
373	if eligible:
374	level = "%d00" % (lnr + 1)
375	else:
376	logger.info('matric_no %s not found in results_import' % student.matric_no)
377	level = 0
378	verdict = 'N/A'
379	#student should not be allowed to perform this transition
380	#wftool = self.portal_workflow
381	#wftool.doActionFor(student,'return')
382	certcode_org = student.Coursemajorcode
383	certcode = makeCertificateCode(certcode_org)
384	certificate_brain = self.getCertificateBrain(certcode)
385	if not certificate_brain:
386	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
387	logger.info(em)
388	matric_no = student.matric_no
389	sid = student_id
390	student_obj = getattr(students_folder,sid)
391	student_obj.invokeFactory('StudentApplication','application')
392	application = student_obj.application
393	self.portal_workflow.doActionFor(application,'open',dest_container=application)
394	da = {'Title': 'Application Data'}
395	student_obj.invokeFactory('StudentPersonal','personal')
396	da['jamb_reg_no'] = student.Entryregno
397	em = self.getEntryMode(student.Entryregno)
398	## em = student.Mode_of_Entry
399	## if em in ('DIRECT', 'DIRECT ENTRY',):
400	## em = 'DE'
401	## elif em in ('U.M.E', 'UNE',):
402	## em = 'UME'
403	## elif not em:
404	## em = "unknown"
405	da['entry_mode'] = em
406	personal = student_obj.personal
407	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
408	dp = {'Title': 'Personal Data'}
409	student_obj.invokeFactory('StudentClearance','clearance')
410	clearance = student_obj.clearance
411	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
412	dc = {'Title': 'Clearance/Eligibility Record'}
413	dc['matric_no'] = matric_no
414	state = student.State
415	lga = student.LGA
416	if state and lga:
417	lga = state + ' / ' + lga
418	else:
419	lga = "None"
420	da['jamb_lga'] = dc['lga'] = lga
421	da['app_email'] = dp['email'] = email
422	da['app_mobile'] = dp['phone'] = phone_nr
423	dp['firstname'] = student.Firstname
424	dp['middlename'] = student.Middlename
425	dp['lastname'] = student.Lastname
426	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
427	da['jamb_sex'] = student.Sex
428	dp['sex'] = student.Sex == 'F'
429	dp['perm_address'] = student.Permanent_Address
430	application.getContent().edit(mapping=da)
431	self.portal_workflow.doActionFor(application,'close',dest_container=application)
432	personal.getContent().edit(mapping=dp)
433	clearance.getContent().edit(mapping=dc)
434	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
435	## catd = {}
436	## catd['id'] = sid
437	## catd['entry_mode']= da['entry_mode']
438	## catd['matric_no'] = matric_no
439	## catd['jamb_reg_no'] = da['jamb_reg_no']
440	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
441	## catd['sex'] = dp['sex']
442	## catd['level'] = level
443	## catd['verdict'] = verdict
444	## if certificate_brain:
445	## cpath = certificate_brain.getPath().split('/')
446	## catd['faculty'] = cpath[-4]
447	## catd['department'] = cpath[-3]
448	## catd['course'] = certcode
449	## self.students_catalog.modifyRecord(**catd)
450	#
451	# Study Course
452	#
453	student_obj.invokeFactory('StudentStudyCourse','study_course')
454	studycourse = student_obj.study_course
455	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
456	dsc = {}
457	dsc['study_course'] = certcode
458	dsc['current_level'] = level
459	dsc['current_verdict'] = verdict
460	dsc['current_mode'] = em
461	dsc['current_session'] = '06'
462	studycourse.getContent().edit(mapping=dsc)
463	#
464	# Level
465	#
466	## l = getattr(studycourse,level,None)
467	## if l is None:
468	## studycourse.invokeFactory('StudentStudyLevel', level)
469	## l = getattr(studycourse, level)
470	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
471	## l.getContent().edit(mapping={'Title': "Level %s" % level})
472	###)
473
474	security.declarePublic('makeStudentLevel') ###(
475	def makeStudentLevel(self,student_id):
476	"create the StudyLevel for a returning Student"
477	#import pdb;pdb.set_trace()
478	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
479	students_folder = self.portal_url.getPortalObject().campus.students
480	res = self.students_catalog(id=student_id)
481	if res:
482	st = res[0]
483	course = st.course
484	matric_no = st.matric_no
485	level = st.level
486	res = self.results_import(matric_no = matric_no)
487	if res:
488	results = res
489	logger.info('%s creating Level %s' % (student_id,level))
490	#
491	# Level
492	#
493	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
494	studycourse = getattr(student_obj,"study_course",None)
495	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
496	l = getattr(studycourse,level,None)
497	if l is None:
498	studycourse.invokeFactory('StudentStudyLevel', level)
499	l = getattr(studycourse, level)
500	self.portal_workflow.doActionFor(l,'open',dest_container=l)
501	l.getContent().edit(mapping={'Title': "Level %s" % level})
502	###)
503
504	security.declarePublic('getHallInfo') ###(
505	def getHallInfo(self,bed):
506	"""return Hall Info"""
507	info = {}
508	hall,block,room,letter = bed.split('_')
509	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
510	if res and len(res) == 1:
511	hall_brain = res[0]
512	hall_doc = hall_brain.getObject().getContent()
513	else:
514	return info
515	info['hall_title'] = hall_brain.Title
516	info['maintenance_code'] = hall_doc.maintenance_code
517	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
518	batch_doc = None
519	for brain in res:
520	if brain.id.startswith(info['maintenance_code']):
521	batch_doc = brain.getObject().getContent()
522	break
523	if batch_doc is None:
524	info['maintenance_fee'] = None
525	else:
526	info['maintenance_fee'] = batch_doc.cost
527	return info
528	###)
529
530	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
531	def deleteAllCourses(self,department="All"):
532	''' delete the courses'''
533	pm = self.portal_membership
534	member = pm.getAuthenticatedMember()
535
536	if str(member) not in ("henrik","joachim"):
537	return "not possible"
538	if department == "All":
539	res = self.portal_catalog({'meta_type': 'Department'})
540	if len(res) < 1:
541	return "No Departments found"
542
543	deleted = []
544	for dep in res:
545	cf = dep.getObject().courses
546	if cf:
547	cf.manage_delObjects(ids=cf.objectIds())
548	deleted.append("deleted Courses in %s" % dep.getId)
549	return "\r".join(deleted)
550	###)
551
552	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
553	def getLogfileLines(self,filename="event.log",numlines=20):
554	"""Get last NUMLINES lines of logfile FILENAME.
555
556	Return last lines' of a file in the instances logfile directory as
557	a list. The number of returned lines equals `numlines' or less. If
558	less than `numlines' lines are available, the whole file ist
559	returned. If the file can not be opened or some other error
560	occurs, empty list is returend.
561	"""
562	result = []
563	lines_hit = 0
564
565	# We only handle files in instances' log directory...
566	logpath = os.path.join(i_home, "log")
567	filename = str(os.path.abspath( os.path.join( logpath, filename )))
568	if not filename.startswith( logpath ):
569	# Attempt to access file outside log-dir...
570	return []
571
572	try:
573	fd = file( filename, "rb" )
574	except IOError:
575	return []
576	if not fd:
577	return []
578
579	if os.linesep == None:
580	linesep = '\n'
581	else:
582	linesep = os.linesep
583
584	# Try to find 'numlines' times a lineseparator, searching from end
585	# and moving to the beginning of file...
586	fd.seek( 0, 2) # Move to end of file...
587	while lines_hit < numlines:
588	if fd.read(1) == linesep[-1]: # This moves filedescriptor
589	# one step forward...
590	lines_hit += 1
591	try:
592	fd.seek( -2, 1) # Go two bytes back from current pos...
593	except IOError:
594	# We cannot go back two bytes. Maybe the file is too small...
595	break
596	fd.seek(2,1)
597
598	# Read all lines from current position...
599	result = fd.readlines()
600	# Remove line endings...
601	result = [x.strip() for x in result]
602	fd.close()
603	return result
604	###)
605
606	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
607	def getCallbacksFromLog(self,filename):
608	"""fix Online Payment Transactions from Z2.log entries"""
609	import transaction
610	import random
611	from cgi import parse_qs
612	from urlparse import urlparse
613	#from pdb import set_trace
614	wftool = self.portal_workflow
615	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
616	students_folder = self.portal_url.getPortalObject().campus.students
617	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
618	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
619	s += r'(?P<intersw>.)" "(?P<agent>.)"'
620	data = re.compile(s)
621	start = True
622	tr_count = 1
623	total = 0
624	#name = 'pume_results'
625	#name = 'epaymentsuccessful_z2log2'
626	name = filename
627	no_import = []
628	imported = []
629	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
630	try:
631	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
632	except:
633	logger.error('Error reading %s' % name)
634	return
635	tas = []
636	for line in transactions:
637	dict = {}
638	items = data.search(line)
639	dict['idict'] = idict = items.groupdict()
640	#print idict
641	#from pdb import set_trace;set_trace()
642	urlparsed = urlparse(idict['get'][4:])
643	#print urlparsed
644	path = urlparsed[2].split('/')
645	dict['student_id'] = student_id = path[8]
646	dict['payment_id'] = payment_id = path[10]
647	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
648	tas.append(dict)
649	tr_count += 1
650	return tas
651	###)
652
653	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
654	def importOnlinePaymentTransactions(self):
655	"""load Online Payment Transactions from CSV values"""
656	import transaction
657	import random
658	#from pdb import set_trace
659	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
660	opt = self.online_payments_import
661	students_folder = self.portal_url.getPortalObject().campus.students
662	start = True
663	tr_count = 1
664	total = 0
665	#name = 'pume_results'
666	name = 'OnlineTransactions'
667	no_import = []
668	imported = []
669	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
670	try:
671	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
672	except:
673	logger.error('Error reading %s.csv' % name)
674	return
675	for pay_transaction in transactions:
676	if start:
677	start = False
678	logger.info('Start loading from %s.csv' % name)
679	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
680	no_import.append('%s,"Error"' % s)
681	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
682	format_error = format + ',"%(Error)s"'
683	data = {}
684
685	# format of the first file sent by Tayo
686	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
687	#data['student_id'] = student_id = pay_transaction['Payer ID']
688	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
689	#data['response_code'] = response_code = pay_transaction['Resp Code']
690	#data['amount'] = amount = pay_transaction['Amount']
691
692	# format of the second file sent by Tayo
693	#data['datetime'] = date = 0
694	#data['student_id'] = student_id = pay_transaction['Payer ID']
695	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
696	#data['response_code'] = response_code = '00'
697	#data['amount'] = amount = pay_transaction['Amount']
698
699	# format of the third file sent by Kehinde
700	data['datetime'] = date = 0
701	data['student_id'] = student_id = pay_transaction['customer_id']
702	data['order_id'] = order_id = pay_transaction['merchant_reference']
703	data['response_code'] = response_code = '00'
704	data['amount'] = amount = pay_transaction['Amount']
705
706	dup = False
707	if response_code == "12":
708	continue
709	try:
710	opt.addRecord(**data)
711	except ValueError:
712	dup = True
713	#from pdb import set_trace;set_trace()
714	if dup:
715	if response_code == "00":
716	try:
717	opt.modifyRecord(**data)
718	except:
719	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
720	continue
721	else:
722	pay_transaction['Error'] = "Duplicate order_id"
723	no_import.append( format_error % pay_transaction)
724	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
725	continue
726	tr_count += 1
727	if tr_count > 1000:
728	if len(no_import) > 0:
729	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
730	'\n'.join(no_import) + '\n')
731	no_import = []
732	em = '%d transactions committed\n' % (tr_count)
733	transaction.commit()
734	regs = []
735	logger.info(em)
736	total += tr_count
737	tr_count = 0
738	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
739	'\n'.join(no_import))
740	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
741	###)
742
743
744	student_field2types = { ###(
745	'StudentApplication':
746	{'id': 'application',
747	'title': 'Application Data',
748	'wf_transition_return': 'close',
749	'wf_transition_admit': 'remain',
750	'fields':
751	('jamb_reg_no',
752	'entry_mode',
753	'entry_session',
754	'jamb_score',
755	'app_email',
756	)
757	},
758	#'StudentPume':
759	# {'id': 'pume',
760	# 'title': 'Pume Data',
761	# 'wf_transition_return': 'close',
762	# 'wf_transition_admit': 'close',
763	# 'fields':
764	# ('pume_score',
765	# )
766	# },
767	'StudentClearance':
768	{'id': 'clearance',
769	'title': 'Clearance Data',
770	'wf_transition_return': 'close',
771	'wf_transition_admit': 'remain',
772	'fields':
773	('matric_no',
774	'nationality',
775	'lga',
776	'birthday',
777	)
778	},
779	'StudentPersonal':
780	{'id': 'personal',
781	'title': 'Personal Data',
782	'wf_transition_return': 'open',
783	'wf_transition_admit': 'remain',
784	'fields':
785	('firstname',
786	'middlename',
787	'lastname',
788	'sex',
789	'email',
790	'phone',
791	'perm_address',
792	)
793	},
794	'StudentStudyCourse':
795	{'id': 'study_course',
796	'title': 'Study Course',
797	'wf_transition_return': 'open',
798	'wf_transition_admit': 'remain',
799	'fields':
800	('study_course',
801	'current_level',
802	'current_session',
803	'current_mode',
804	'current_verdict',
805	)
806	},
807
808	'PaymentsFolder':
809	{'id': 'payments',
810	'title': 'Payments',
811	'wf_transition_return': 'open',
812	'wf_transition_admit': 'open',
813	'fields':
814	()
815	},
816	}
817	###)
818
819
820	security.declareProtected(ModifyPortalContent,'importStudent') ###(
821	def importStudent(self,mapping):
822	"create a students record due import"
823	logger = logging.getLogger('WAeUPTool.importStudent')
824	students_folder = self.portal_url.getPortalObject().campus.students
825	jamb_reg_no = mapping.get('jamb_reg_no',None)
826	if jamb_reg_no:
827	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
828	if res:
829	return '',"jamb_reg_no exists"
830	matric_no = mapping.get('matric_no',None)
831	if matric_no:
832	res = self.students_catalog(matric_no = matric_no)
833	if res:
834	return '',"matric_no exists"
835	sid = self.waeup_tool.generateStudentId('?')
836	students_folder.invokeFactory('Student', sid)
837	student_obj = getattr(students_folder,sid)
838	f2t = self.student_field2types
839	d = {}
840	d['jamb_sex'] = 'M'
841	if mapping.get('sex'):
842	d['jamb_sex'] = 'F'
843	transition = mapping.get('reg_transition','admit')
844	if transition not in ('admit','return'):
845	transition = 'admit'
846	for pt in f2t.keys():
847	student_obj.invokeFactory(pt,f2t[pt]['id'])
848	sub_obj = getattr(student_obj,f2t[pt]['id'])
849	sub_doc = sub_obj.getContent()
850	d['Title'] = f2t[pt]['title']
851	for field in f2t[pt]['fields']:
852	d[field] = mapping.get(field,'')
853	if pt == "StudyCourse":
854	for von,zu in (('entry_mode','current_mode'),
855	('entry_session','current_session')):
856	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
857	d[zu] = mapping[von]
858	sub_doc.edit(mapping = d)
859
860	#import pdb;pdb.set_trace()
861	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
862	if new_state != "remain":
863	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
864	self.portal_workflow.doActionFor(student_obj,transition)
865	student_obj.manage_setLocalRoles(sid, ['Owner',])
866	return sid,''
867	###)
868
869	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
870	def importEditStudent(self,mapping):
871	"edit a students record due import"
872	logger = logging.getLogger('WAeUPTool.importEditStudent')
873	students_folder = self.portal_url.getPortalObject().campus.students
874	sid = mapping.get('id',None)
875	jamb_reg_no = mapping.get('jamb_reg_no',None)
876	matric_no = mapping.get('matric_no',None)
877	editable_keys = mapping.keys()
878	if sid:
879	res = self.students_catalog(id = sid)
880	if not res:
881	return '',"no student with id %s" % sid
882	elif matric_no and res[0].matric_no and\
883	matric_no != res[0].matric_no:
884	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
885	elif jamb_reg_no and res[0].jamb_reg_no and\
886	jamb_reg_no != res[0].jamb_reg_no:
887	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
888	elif jamb_reg_no:
889	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
890	if not res:
891	return '',"no student with jamb_reg_no %s" % jamb_reg_no
892	editable_keys.remove('jamb_reg_no')
893	elif matric_no:
894	res = self.students_catalog(matric_no = matric_no)
895	if not res:
896	return '',"no student with matric_no %s" % matric_no
897	editable_keys.remove('matric_no')
898
899	# included only to change wf state from admitted to returning
900	if res[0].review_state != 'admitted':
901	return '%s' % res[0].id ,"student is not in state admitted"
902	# end inclusion
903
904	sid = res[0].id
905	student_obj = getattr(students_folder,sid)
906	f2t = self.student_field2types
907	d = {}
908	#import pdb;pdb.set_trace()
909	any_change = False
910	for pt in f2t.keys():
911	if pt == "student_application":
912	d['jamb_sex'] = 'M'
913	if mapping.get('sex'):
914	d['jamb_sex'] = 'F'
915	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
916	if intersect:
917	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
918	if sub_obj is None:
919	try:
920	student_obj.invokeFactory(pt,f2t[pt]['id'])
921	except:
922	continue
923	sub_obj = getattr(student_obj,f2t[pt]['id'])
924	d['Title'] = f2t[pt]['title']
925	sub_doc = sub_obj.getContent()
926	for field in intersect:
927	changed = False
928	if getattr(sub_doc,field) != mapping.get(field,''):
929	any_change = True
930	changed = True
931	d[field] = mapping.get(field,'')
932	if changed:
933	sub_doc.edit(mapping = d)
934
935
936	# included only to change wf state from admitted to returning
937	if res[0].review_state == 'admitted':
938	new_state = f2t[pt]['wf_transition_return']
939	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
940	if sub_obj and new_state != "remain":
941	try:
942	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
943	except:
944	#logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
945	pass
946	if res[0].review_state == 'admitted':
947	wfaction = 'return'
948	try:
949	self.portal_workflow.doActionFor(student_obj,wfaction)
950	logger.info('%s, wf state changed' % sid)
951	any_change = True
952	except:
953	logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
954	pass
955	# end inclusion
956
957
958	if any_change:
959	return sid,''
960	else:
961	return sid,'not modified'
962	###)
963
964
965	security.declareProtected(ModifyPortalContent,"importData")###(
966	def importData(self,filename,name,edit=False):
967	"""load data from CSV values"""
968	import transaction
969	import random
970
971	pm = self.portal_membership
972	member = pm.getAuthenticatedMember()
973
974	logger = logging.getLogger('WAeUPTool.importData')
975	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
976	students_folder = self.portal_url.getPortalObject().campus.students
977	start = True
978	tr_count = 1
979	total_imported = 0
980	total_not_imported = 0
981	total = 0
982	iname = "import_%s" % name
983	stool = getToolByName(self, 'portal_schemas')
984	ltool = getToolByName(self, 'portal_layouts')
985	schema = stool._getOb(iname)
986	if schema is None:
987	em = 'No such schema %s' % iname
988	logger.error('No such schema %s' % iname)
989	return em
990	layout = ltool._getOb(iname)
991	if layout is None:
992	em = 'No such layout %s' % iname
993	logger.error(em)
994	return em
995	validators = {}
996	for widget in layout.keys():
997	validators[widget] = layout[widget].validate
998	if edit:
999	importer_name = "importEdit%s" % name.capitalize()
1000	else:
1001	importer_name = "import%s" % name.capitalize()
1002	importer = getattr(self, '%s' % importer_name,None)
1003	if importer is None:
1004	em = 'No importer function %s' % importer_name
1005	logger.error(em)
1006	return em
1007	not_imported = []
1008	imported = []
1009	try:
1010	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1011	except:
1012	em = 'Error reading %s.csv' % filename
1013	logger.error(em)
1014	return em
1015	for item in items:
1016	if start:
1017	start = False
1018	adapters = [MappingStorageAdapter(schema, item)]
1019	dm = DataModel(item, adapters,context=self)
1020	logger.info('%s starts import from %s.csv' % (member,filename))
1021	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1022	diff2schema = set(import_keys).difference(set(schema.keys()))
1023	diff2layout = set(import_keys).difference(set(layout.keys()))
1024	if diff2schema:
1025	em = "not ignorable key(s) %s found in heading" % diff2schema
1026	return em
1027	s = ','.join(['"%s"' % fn for fn in import_keys])
1028	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1029	s = '"id",' + s
1030	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1031	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1032	format_error = format + ',"%(Error)s"'
1033	format = '"%(id)s",'+ format
1034	ds = DataStructure(data=item,datamodel=dm)
1035	error_string = ""
1036	for k in import_keys:
1037	if not validators[k](ds):
1038	error_string += " %s : %s" % (k,ds.getError(k))
1039	if not error_string:
1040	item.update(dm)
1041	item['id'],error = importer(item)
1042	if error:
1043	error_string += error
1044	if error_string:
1045	item['Error'] = error_string
1046	not_imported.append(format_error % item)
1047	total_not_imported += 1
1048	else:
1049	em = format % item
1050	imported.append(em)
1051	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1052	tr_count += 1
1053	total_imported += 1
1054	total += 1
1055	if total_imported and not total_imported % 100:
1056	transaction.commit()
1057	if len(not_imported) > 0:
1058	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1059	'\n'.join(not_imported) + '\n')
1060	not_imported = []
1061	if len(imported) > 0:
1062	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1063	'\n'.join(imported) + '\n')
1064	imported = []
1065	em = '%d transactions committed\n' % (tr_count)
1066	regs = []
1067	logger.info(em)
1068	tr_count = 0
1069	if len(imported) > 0:
1070	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1071	'\n'.join(imported))
1072	if len(not_imported) > 0:
1073	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1074	'\n'.join(not_imported))
1075	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1076	logger.info(em)
1077	return em
1078	###)
1079
1080	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: