Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 1892

Last change on this file since 1892 was 1890, checked in by joachim, 18 years ago
new function removeDeletedDocIds
Property svn:keywords set to `Id`
File size: 41.2 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 1890 2007-06-13 10:40:51Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime
47	import logging
48	import transaction
49	import csv,re,os
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66
67	manage_options = ( ActionProviderBase.manage_options
68	+ SimpleItem.manage_options
69	)
70
71	def rwrite(self,s): ###(
72	response = self.REQUEST.RESPONSE
73	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74	response.write("%s<br />\r\n" % s)
75	###)
76
77	def sleep(self,secs): ###(
78	"sleep"
79	import time
80	time.sleep(secs)
81	return
82
83	###)
84
85	security.declareProtected(ModifyPortalContent,'openLog') ###(
86	def openLog(self,name):
87	"""open a log file"""
88	version = 1
89	path = "%s/log/%s_%d.log" % (i_home,name,version)
90	while os.path.exists(path):
91	version += 1
92	path = "%s/log/%s_%d.log" % (i_home,name,version)
93	log = open(path,"w")
94	return log
95
96	###)
97
98	security.declareProtected(ModifyPortalContent,'writeLog') ###(
99	def writeLog(self,logfile,s):
100	"""write to the log file"""
101	logfile.write(s)
102
103	###)
104
105	def generateStudentId(self,letter): ###(
106	import random
107	r = random
108	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109	if letter == '?':
110	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111	sid = "%c%d" % (letter,r.randint(99999,1000000))
112	students = self.portal_url.getPortalObject().campus.students
113	## while hasattr(students, sid):
114	## sid = "%c%d" % (letter,r.randint(99999,1000000))
115	while self.students_catalog(id = sid):
116	sid = "%c%d" % (letter,r.randint(99999,1000000))
117	return sid
118	###)
119
120	def generatePassword(self,s=None): ###(
121	import random
122	r = random
123	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124	if s is None:
125	s = 'abcdefghklmnpqrstuvwxy23456789'
126	pw = ''
127	while len(pw) < 6:
128	pw += r.choice(s)
129	return pw
130	###)
131
132	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133	def removeDeletedDocIds(self, max=1000):
134	"""
135	remove deleted docids from repository commit after max
136	"""
137	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138	repository = getToolByName(self, 'portal_repository')
139	pxtool = getToolByName(self, 'portal_proxies')
140	pxtool_infos = pxtool.getRevisionsUsed()
141
142	nb_revs = 0
143	docids_d = {} # all docids
144	unused_docids_d = {} # all docids that are unused
145	ids_unused_revs_docids = [] # ids for revs of unused docids
146	ids_unused_revs = [] # ids for unused revs
147	total = 0
148	for id in repository.objectIds():
149	docid, rev = repository._splitId(id)
150	if docid is None:
151	logger.info("invalid doc_id %s" % docid)
152	continue
153	nb_revs += 1
154	docids_d[docid] = None
155	if not pxtool_infos.has_key(docid):
156	unused_docids_d[docid] = None
157	ids_unused_revs_docids.append(id)
158	ids_unused_revs.append(id)
159	elif not pxtool_infos[docid].has_key(rev):
160	ids_unused_revs.append(id)
161	if len(ids_unused_revs) >= max:
162	repository.manage_delObjects(ids_unused_revs)
163	#import pdb;pdb.set_trace()
164	transaction.commit()
165	total += max
166	logger.info('removed %d total %d unused docids ' % (max,total))
167	anz = len(ids_unused_revs)
168	if anz > 0:
169	repository.manage_delObjects(ids_unused_revs)
170	transaction.commit()
171	total += anz
172	logger.info('finished removing %d unused docids ' % (total))
173
174
175	###)
176
177	security.declareProtected(ModifyPortalContent,'getCredential') ###(
178	def getCredential(self,student_id):
179	"return a student password"
180	student_entry = getattr(self.portal_directories.students,student_id,None)
181	if student_entry is None:
182	return None
183	return getattr(student_entry,"password","not set")
184	###)
185
186	security.declarePublic('checkPassword') ###(
187	def checkPassword(self,student_id,password):
188	"return a student password"
189	student_entry = getattr(self.portal_directories.students,student_id,None)
190	if student_entry is None:
191	return False
192	return getattr(student_entry,"password","not set") == password
193	###)
194
195	security.declarePublic('editPassword') ###(
196	def editPassword(self,student_id,password):
197	"edit a student password"
198	student_entry = getattr(self.portal_directories.students,student_id,None)
199	if student_entry is None:
200	return
201	setattr(student_entry,'password',password)
202	###)
203
204	security.declareProtected(View,'doCommit') ###(
205	def doCommit(self,logger=None):
206	"commit some transactions"
207	transaction.commit()
208	###)
209
210	security.declarePublic('loadStudentFoto') ###(
211	def loadStudentFoto(self,student,filename,folder):
212	"return a student passport picture"
213	app_doc = student.application.getContent()
214	#clear = student.clearance
215	#clear_doc = clear.getContent()
216	#matric_no = clear_doc.matric_no.upper()
217	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
218	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
219	#import pdb;pdb.set_trace()
220	if os.path.exists(picture1):
221	file = open(picture1)
222	elif os.path.exists(picture2):
223	file = open(picture2)
224	else:
225	return "passport picture not found %s" % picture1
226
227	outfile = file.read()
228	app_doc.manage_addFile('passport',
229	file=outfile,
230	title="%s.jpg" % filename)
231	return "successfully loaded passport picture"
232	###)
233
234	security.declareProtected(ModifyPortalContent,'createOne') ###(
235	def createOne(self,students_folder,student_brain,letter,commit=False):
236	sid = self.waeup_tool.generateStudentId(letter)
237	students_folder.invokeFactory('Student', sid)
238	student = getattr(students_folder,sid)
239	self.portal_workflow.doActionFor(student,'return')
240	student.manage_setLocalRoles(sid, ['Owner',])
241	matric_no = student_brain.matric_no
242	jamb_reg_no = student_brain.Entryregno
243	self.students_catalog.addRecord(id = sid,
244	matric_no = matric_no,
245	jamb_reg_no = jamb_reg_no,
246	sex = student_brain.Sex == "F",
247	name = "%s %s %s" % (student_brain.Firstname,
248	student_brain.Middlename,
249	student_brain.Lastname)
250	)
251	if commit:
252	transaction.commit()
253	return sid,jamb_reg_no
254	###)
255
256	security.declareProtected(ModifyPortalContent,'addStudent') ###(
257	def addStudent(self,dict):
258	students_folder = self.portal_url.getPortalObject().campus.students
259	sid = self.waeup_tool.generateStudentId('?')
260	students_folder.invokeFactory('Student', sid)
261	student_obj = getattr(students_folder,sid)
262	password = self.generatePassword()
263	self.makeStudentMember(sid,password)
264	f2t = self.student_field2types
265	#from pdb import set_trace; set_trace()
266	d = {}
267	d['jamb_sex'] = 'M'
268	if dict.get('sex'):
269	d['jamb_sex'] = 'F'
270	for pt in f2t.keys():
271	student_obj.invokeFactory(pt,f2t[pt]['id'])
272	sub_obj = getattr(student_obj,f2t[pt]['id'])
273	sub_doc = sub_obj.getContent()
274	self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
275	d['Title'] = f2t[pt]['title']
276	for field in f2t[pt]['fields']:
277	d[field] = dict.get(field,'')
278	sub_doc.edit(mapping = d)
279	new_state = f2t[pt]['wf_transition']
280	if new_state != "remain":
281	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
282	entry_session = dict.get('entry_session')
283	wfaction = 'return'
284	if entry_session == self.getSessionId():
285	wfaction = 'admit'
286	self.portal_workflow.doActionFor(student_obj,wfaction)
287	student_obj.manage_setLocalRoles(sid, ['Owner',])
288	return sid,password
289	###)
290
291	security.declarePublic('getCertificateBrain') ###(
292	def getCertificateBrain(self,cert_id):
293	"do it"
294	res = ZCatalog.searchResults(self.portal_catalog_real,
295	{'portal_type':"Certificate",
296	'id': cert_id})
297	if res:
298	return res[0]
299	return None
300	###)
301
302	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
303	def get_csv_filenames(self):
304	"do it"
305	files = [file for file in os.listdir("%s/import/" % (i_home))
306	if file.endswith('.csv') and file.find('imported') == -1]
307	return files
308	###)
309
310	security.declarePublic('findStudentByMatricelNo') ###(
311	def findStudentByMatricelNo(self,matric_no):
312	"do it"
313	res = ZCatalog.searchResults(self.portal_catalog_real,
314	{'portal_type':"StudentClearance",
315	'SearchableText': matric_no})
316	if res:
317	return res[0]
318	return None
319	###)
320
321	security.declarePublic('makeStudentMember') ###(
322	def makeStudentMember(self,sid,password='uNsEt'):
323	"""make the student a member"""
324	membership = self.portal_membership
325	membership.addMember(sid,
326	password ,
327	roles=('Member',
328	'Student',
329	),
330	domains='',
331	properties = {'memberareaCreationFlag': False,
332	'homeless': True},)
333	member = membership.getMemberById(sid)
334	self.portal_registration.afterAdd(member, sid, password, None)
335	#self.manage_setLocalRoles(sid, ['Owner',])
336	###)
337
338	security.declarePublic('makeStudentData') ###(
339	def makeStudentData(self,student_id,email=None,phone_nr=None):
340	"create Datastructure for a returning Student"
341	#import pdb;pdb.set_trace()
342	logger = logging.getLogger('WAeUPTool.makeStudentData')
343	students_folder = self.portal_url.getPortalObject().campus.students
344	#res = self.students_catalog(id=student_id)
345	#if res:
346	# st = res[0]
347	#res = self.returning_import(matric_no = st.matric_no)
348	res = self.returning_import(id = student_id)
349	if res:
350	student = res[0]
351	else:
352	logger.info('Id %s not found in returning_import' % student_id)
353	return
354	logger.info('%s creates data structure' % student_id)
355	s_results = self.results_import(matric_no = student.matric_no)
356	if s_results:
357	lnr = self.getLevelFromResultsCosCode(s_results)
358	level = "%d00" % lnr
359	verdict,eligible = self.getVerdict(s_results[0].Verdict)
360	if eligible:
361	level = "%d00" % (lnr + 1)
362	else:
363	logger.info('matric_no %s not found in results_import' % student.matric_no)
364	level = 0
365	verdict = 'N/A'
366	#student should not be allowed to perform this transition
367	#wftool = self.portal_workflow
368	#wftool.doActionFor(student,'return')
369	certcode_org = student.Coursemajorcode
370	certcode = makeCertificateCode(certcode_org)
371	certificate_brain = self.getCertificateBrain(certcode)
372	if not certificate_brain:
373	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
374	logger.info(em)
375	matric_no = student.matric_no
376	sid = student_id
377	student_obj = getattr(students_folder,sid)
378	student_obj.invokeFactory('StudentApplication','application')
379	application = student_obj.application
380	self.portal_workflow.doActionFor(application,'open',dest_container=application)
381	da = {'Title': 'Application Data'}
382	student_obj.invokeFactory('StudentPersonal','personal')
383	da['jamb_reg_no'] = student.Entryregno
384	em = self.getEntryMode(student.Entryregno)
385	## em = student.Mode_of_Entry
386	## if em in ('DIRECT', 'DIRECT ENTRY',):
387	## em = 'DE'
388	## elif em in ('U.M.E', 'UNE',):
389	## em = 'UME'
390	## elif not em:
391	## em = "unknown"
392	da['entry_mode'] = em
393	personal = student_obj.personal
394	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
395	dp = {'Title': 'Personal Data'}
396	student_obj.invokeFactory('StudentClearance','clearance')
397	clearance = student_obj.clearance
398	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
399	dc = {'Title': 'Clearance/Eligibility Record'}
400	dc['matric_no'] = matric_no
401	state = student.State
402	lga = student.LGA
403	if state and lga:
404	lga = state + ' / ' + lga
405	else:
406	lga = "None"
407	da['jamb_lga'] = dc['lga'] = lga
408	da['app_email'] = dp['email'] = email
409	da['app_mobile'] = dp['phone'] = phone_nr
410	dp['firstname'] = student.Firstname
411	dp['middlename'] = student.Middlename
412	dp['lastname'] = student.Lastname
413	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
414	da['jamb_sex'] = student.Sex
415	dp['sex'] = student.Sex == 'F'
416	dp['perm_address'] = student.Permanent_Address
417	application.getContent().edit(mapping=da)
418	self.portal_workflow.doActionFor(application,'close',dest_container=application)
419	personal.getContent().edit(mapping=dp)
420	clearance.getContent().edit(mapping=dc)
421	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
422	## catd = {}
423	## catd['id'] = sid
424	## catd['entry_mode']= da['entry_mode']
425	## catd['matric_no'] = matric_no
426	## catd['jamb_reg_no'] = da['jamb_reg_no']
427	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
428	## catd['sex'] = dp['sex']
429	## catd['level'] = level
430	## catd['verdict'] = verdict
431	## if certificate_brain:
432	## cpath = certificate_brain.getPath().split('/')
433	## catd['faculty'] = cpath[-4]
434	## catd['department'] = cpath[-3]
435	## catd['course'] = certcode
436	## self.students_catalog.modifyRecord(**catd)
437	#
438	# Study Course
439	#
440	student_obj.invokeFactory('StudentStudyCourse','study_course')
441	studycourse = student_obj.study_course
442	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
443	dsc = {}
444	dsc['study_course'] = certcode
445	dsc['current_level'] = level
446	dsc['current_verdict'] = verdict
447	dsc['current_mode'] = em
448	dsc['current_session'] = '06'
449	studycourse.getContent().edit(mapping=dsc)
450	#
451	# Level
452	#
453	## l = getattr(studycourse,level,None)
454	## if l is None:
455	## studycourse.invokeFactory('StudentStudyLevel', level)
456	## l = getattr(studycourse, level)
457	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
458	## l.getContent().edit(mapping={'Title': "Level %s" % level})
459	###)
460
461	security.declarePublic('makeStudentLevel') ###(
462	def makeStudentLevel(self,student_id):
463	"create the StudyLevel for a returning Student"
464	#import pdb;pdb.set_trace()
465	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
466	students_folder = self.portal_url.getPortalObject().campus.students
467	res = self.students_catalog(id=student_id)
468	if res:
469	st = res[0]
470	course = st.course
471	matric_no = st.matric_no
472	level = st.level
473	res = self.results_import(matric_no = matric_no)
474	if res:
475	results = res
476	logger.info('%s creating Level %s' % (student_id,level))
477	#
478	# Level
479	#
480	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
481	studycourse = getattr(student_obj,"study_course",None)
482	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
483	l = getattr(studycourse,level,None)
484	if l is None:
485	studycourse.invokeFactory('StudentStudyLevel', level)
486	l = getattr(studycourse, level)
487	self.portal_workflow.doActionFor(l,'open',dest_container=l)
488	l.getContent().edit(mapping={'Title': "Level %s" % level})
489	###)
490
491	security.declarePublic('getAccommodationInfo') ###(
492	def getAccommodationInfo(self,bed):
493	"""return Accommodation Info"""
494	info = {}
495	hall,block,room,letter = bed.split('_')
496	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
497	if res and len(res) == 1:
498	hall_brain = res[0]
499	hall_doc = hall_brain.getObject().getContent()
500	else:
501	return info
502	info['hall_title'] = hall_brain.Title
503	info['maintenance_code'] = hall_doc.maintenance_code
504	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
505	batch_doc = None
506	for brain in res:
507	if brain.id.startswith(info['maintenance_code']):
508	batch_doc = brain.getObject().getContent()
509	break
510	if batch_doc is None:
511	info['maintenance_fee'] = None
512	else:
513	info['maintenance_fee'] = batch_doc.cost
514	return info
515	###)
516
517	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
518	def deleteAllCourses(self,department="All"):
519	''' delete the courses'''
520	pm = self.portal_membership
521	member = pm.getAuthenticatedMember()
522
523	if str(member) not in ("henrik","joachim"):
524	return "not possible"
525	if department == "All":
526	res = self.portal_catalog({'meta_type': 'Department'})
527	if len(res) < 1:
528	return "No Departments found"
529
530	deleted = []
531	for dep in res:
532	cf = dep.getObject().courses
533	if cf:
534	cf.manage_delObjects(ids=cf.objectIds())
535	deleted.append("deleted Courses in %s" % dep.getId)
536	return "\r".join(deleted)
537	###)
538
539	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
540	def getLogfileLines(self,filename="event.log",numlines=20):
541	"""Get last NUMLINES lines of logfile FILENAME.
542
543	Return last lines' of a file in the instances logfile directory as
544	a list. The number of returned lines equals `numlines' or less. If
545	less than `numlines' lines are available, the whole file ist
546	returned. If the file can not be opened or some other error
547	occurs, empty list is returend.
548	"""
549	result = []
550	lines_hit = 0
551
552	# We only handle files in instances' log directory...
553	logpath = os.path.join(i_home, "log")
554	filename = str(os.path.abspath( os.path.join( logpath, filename )))
555	if not filename.startswith( logpath ):
556	# Attempt to access file outside log-dir...
557	return []
558
559	try:
560	fd = file( filename, "rb" )
561	except IOError:
562	return []
563	if not fd:
564	return []
565
566	if os.linesep == None:
567	linesep = '\n'
568	else:
569	linesep = os.linesep
570
571	# Try to find 'numlines' times a lineseparator, searching from end
572	# and moving to the beginning of file...
573	fd.seek( 0, 2) # Move to end of file...
574	while lines_hit < numlines:
575	if fd.read(1) == linesep[-1]: # This moves filedescriptor
576	# one step forward...
577	lines_hit += 1
578	try:
579	fd.seek( -2, 1) # Go two bytes back from current pos...
580	except IOError:
581	# We cannot go back two bytes. Maybe the file is too small...
582	break
583	fd.seek(2,1)
584
585	# Read all lines from current position...
586	result = fd.readlines()
587	# Remove line endings...
588	result = [x.strip() for x in result]
589	fd.close()
590	return result
591	###)
592
593	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
594	def getCallbacksFromLog(self,filename):
595	"""fix Online Payment Transactions from Z2.log entries"""
596	import transaction
597	import random
598	from cgi import parse_qs
599	from urlparse import urlparse
600	#from pdb import set_trace
601	wftool = self.portal_workflow
602	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
603	students_folder = self.portal_url.getPortalObject().campus.students
604	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
605	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
606	s += r'(?P<intersw>.)" "(?P<agent>.)"'
607	data = re.compile(s)
608	start = True
609	tr_count = 1
610	total = 0
611	#name = 'pume_results'
612	#name = 'epaymentsuccessful_z2log2'
613	name = filename
614	no_import = []
615	imported = []
616	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
617	try:
618	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
619	except:
620	logger.error('Error reading %s' % name)
621	return
622	tas = []
623	for line in transactions:
624	dict = {}
625	items = data.search(line)
626	dict['idict'] = idict = items.groupdict()
627	#print idict
628	#from pdb import set_trace;set_trace()
629	urlparsed = urlparse(idict['get'][4:])
630	#print urlparsed
631	path = urlparsed[2].split('/')
632	dict['student_id'] = student_id = path[8]
633	dict['payment_id'] = payment_id = path[10]
634	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
635	tas.append(dict)
636	tr_count += 1
637	return tas
638	###)
639
640	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
641	def importOnlinePaymentTransactions(self):
642	"""load Online Payment Transactions from CSV values"""
643	import transaction
644	import random
645	#from pdb import set_trace
646	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
647	opt = self.online_payments_import
648	students_folder = self.portal_url.getPortalObject().campus.students
649	start = True
650	tr_count = 1
651	total = 0
652	#name = 'pume_results'
653	name = 'OnlineTransactions'
654	no_import = []
655	imported = []
656	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
657	try:
658	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
659	except:
660	logger.error('Error reading %s.csv' % name)
661	return
662	for pay_transaction in transactions:
663	if start:
664	start = False
665	logger.info('Start loading from %s.csv' % name)
666	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
667	no_import.append('%s,"Error"' % s)
668	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
669	format_error = format + ',"%(Error)s"'
670	data = {}
671
672	# format of the first file sent by Tayo
673	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
674	#data['student_id'] = student_id = pay_transaction['Payer ID']
675	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
676	#data['response_code'] = response_code = pay_transaction['Resp Code']
677	#data['amount'] = amount = pay_transaction['Amount']
678
679	# format of the second file sent by Tayo
680	#data['datetime'] = date = 0
681	#data['student_id'] = student_id = pay_transaction['Payer ID']
682	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
683	#data['response_code'] = response_code = '00'
684	#data['amount'] = amount = pay_transaction['Amount']
685
686	# format of the third file sent by Kehinde
687	data['datetime'] = date = 0
688	data['student_id'] = student_id = pay_transaction['customer_id']
689	data['order_id'] = order_id = pay_transaction['merchant_reference']
690	data['response_code'] = response_code = '00'
691	data['amount'] = amount = pay_transaction['Amount']
692
693	dup = False
694	if response_code == "12":
695	continue
696	try:
697	opt.addRecord(**data)
698	except ValueError:
699	dup = True
700	#from pdb import set_trace;set_trace()
701	if dup:
702	if response_code == "00":
703	try:
704	opt.modifyRecord(**data)
705	except:
706	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
707	continue
708	else:
709	pay_transaction['Error'] = "Duplicate order_id"
710	no_import.append( format_error % pay_transaction)
711	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
712	continue
713	tr_count += 1
714	if tr_count > 1000:
715	if len(no_import) > 0:
716	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
717	'\n'.join(no_import) + '\n')
718	no_import = []
719	em = '%d transactions committed\n' % (tr_count)
720	transaction.commit()
721	regs = []
722	logger.info(em)
723	total += tr_count
724	tr_count = 0
725	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
726	'\n'.join(no_import))
727	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
728	###)
729
730
731	# all workflow transitions refer to students in state returning
732	student_field2types = { ###(
733	'StudentApplication':
734	{'id': 'application',
735	'title': 'Application Data',
736	'wf_transition': 'close',
737	'fields':
738	('jamb_reg_no',
739	'entry_mode',
740	'entry_session',
741	'jamb_score',
742	'app_email',
743	)
744	},
745	#'StudentPume':
746	# {'id': 'pume',
747	# 'title': 'Pume Data',
748	# 'wf_transition': 'close',
749	# 'fields':
750	# ('pume_score',
751	# )
752	# },
753	'StudentClearance':
754	{'id': 'clearance',
755	'title': 'Clearance Data',
756	'wf_transition': 'close',
757	'fields':
758	('matric_no',
759	'nationality',
760	'lga',
761	'birthday',
762	)
763	},
764	'StudentPersonal':
765	{'id': 'personal',
766	'title': 'Personal Data',
767	'wf_transition': 'open',
768	'fields':
769	('firstname',
770	'middlename',
771	'lastname',
772	'sex',
773	'email',
774	'phone',
775	'perm_address',
776	)
777	},
778	'StudentStudyCourse':
779	{'id': 'study_course',
780	'title': 'Study Course',
781	'wf_transition': 'open',
782	'fields':
783	('study_course',
784	'current_level',
785	'current_session',
786	'current_mode',
787	'current_verdict',
788	)
789	},
790	#only temporarily changed to import admitted students
791	#'PaymentsFolder':
792	# {'id': 'payments',
793	# 'title': 'Payments',
794	# 'wf_transition': 'open',
795	# 'fields':
796	# ()
797	# },
798	}
799	###)
800
801
802	security.declareProtected(ModifyPortalContent,'importStudent') ###(
803	def importStudent(self,mapping):
804	"create a students record due import"
805	logger = logging.getLogger('WAeUPTool.importStudent')
806	students_folder = self.portal_url.getPortalObject().campus.students
807	jamb_reg_no = mapping.get('jamb_reg_no',None)
808	if jamb_reg_no:
809	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
810	if res:
811	return '',"jamb_reg_no exists"
812	matric_no = mapping.get('matric_no',None)
813	if matric_no:
814	res = self.students_catalog(matric_no = matric_no)
815	if res:
816	return '',"matric_no exists"
817	sid = self.waeup_tool.generateStudentId('?')
818	students_folder.invokeFactory('Student', sid)
819	student_obj = getattr(students_folder,sid)
820	f2t = self.student_field2types
821	d = {}
822	d['jamb_sex'] = 'M'
823	if mapping.get('sex'):
824	d['jamb_sex'] = 'F'
825	for pt in f2t.keys():
826	student_obj.invokeFactory(pt,f2t[pt]['id'])
827	sub_obj = getattr(student_obj,f2t[pt]['id'])
828	sub_doc = sub_obj.getContent()
829	d['Title'] = f2t[pt]['title']
830	for field in f2t[pt]['fields']:
831	d[field] = mapping.get(field,'')
832	if pt == "StudyCourse":
833	for von,zu in (('entry_mode','current_mode'),
834	('entry_session','current_session')):
835	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
836	d[zu] = mapping[von]
837	sub_doc.edit(mapping = d)
838	new_state = f2t[pt]['wf_transition']
839
840	#only temporarily changed to import admitted students
841	#if new_state != "remain":
842	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
843	wfaction = 'admit'
844
845
846	self.portal_workflow.doActionFor(student_obj,wfaction)
847	student_obj.manage_setLocalRoles(sid, ['Owner',])
848	return sid,''
849	###)
850
851	security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
852	def importEditStudent(self,mapping):
853	"edit a students record due import"
854	logger = logging.getLogger('WAeUPTool.importEditStudent')
855	students_folder = self.portal_url.getPortalObject().campus.students
856	sid = mapping.get('id',None)
857	jamb_reg_no = mapping.get('jamb_reg_no',None)
858	matric_no = mapping.get('matric_no',None)
859	editable_keys = mapping.keys()
860	if sid:
861	res = self.students_catalog(id = sid)
862	if not res:
863	return '',"no student with id %s" % sid
864	elif matric_no and res[0].matric_no and\
865	matric_no != res[0].matric_no:
866	return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
867	elif jamb_reg_no and res[0].jamb_reg_no and\
868	jamb_reg_no != res[0].jamb_reg_no:
869	return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
870	elif jamb_reg_no:
871	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
872	if not res:
873	return '',"no student with jamb_reg_no %s" % jamb_reg_no
874	editable_keys.remove('jamb_reg_no')
875	elif matric_no:
876	res = self.students_catalog(matric_no = matric_no)
877	if not res:
878	return '',"no student with matric_no %s" % matric_no
879	editable_keys.remove('matric_no')
880	sid = res[0].id
881	student_obj = getattr(students_folder,sid)
882	f2t = self.student_field2types
883	d = {}
884	#import pdb;pdb.set_trace()
885	any_change = False
886	for pt in f2t.keys():
887	if pt == "student_application":
888	d['jamb_sex'] = 'M'
889	if mapping.get('sex'):
890	d['jamb_sex'] = 'F'
891	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
892	if intersect:
893	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
894	if sub_obj is None:
895	try:
896	student_obj.invokeFactory(pt,f2t[pt]['id'])
897	except:
898	continue
899	sub_obj = getattr(student_obj,f2t[pt]['id'])
900	d['Title'] = f2t[pt]['title']
901	sub_doc = sub_obj.getContent()
902	for field in intersect:
903	changed = False
904	if getattr(sub_doc,field) != mapping.get(field,''):
905	any_change = True
906	changed = True
907	d[field] = mapping.get(field,'')
908	if changed:
909	sub_doc.edit(mapping = d)
910	if any_change:
911	return sid,''
912	else:
913	return sid,'not modified'
914	###)
915
916	security.declareProtected(ModifyPortalContent,"importData")###(
917	def importData(self,filename,name,edit=False):
918	"""load data from CSV values"""
919	import transaction
920	import random
921
922	pm = self.portal_membership
923	member = pm.getAuthenticatedMember()
924
925	logger = logging.getLogger('WAeUPTool.importData')
926	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
927	students_folder = self.portal_url.getPortalObject().campus.students
928	start = True
929	tr_count = 1
930	total_imported = 0
931	total_not_imported = 0
932	total = 0
933	iname = "import_%s" % name
934	stool = getToolByName(self, 'portal_schemas')
935	ltool = getToolByName(self, 'portal_layouts')
936	schema = stool._getOb(iname)
937	if schema is None:
938	em = 'No such schema %s' % iname
939	logger.error('No such schema %s' % iname)
940	return em
941	layout = ltool._getOb(iname)
942	if layout is None:
943	em = 'No such layout %s' % iname
944	logger.error(em)
945	return em
946	validators = {}
947	for widget in layout.keys():
948	validators[widget] = layout[widget].validate
949	if edit:
950	importer_name = "importEdit%s" % name.capitalize()
951	else:
952	importer_name = "import%s" % name.capitalize()
953	importer = getattr(self, '%s' % importer_name,None)
954	if importer is None:
955	em = 'No importer function %s' % importer_name
956	logger.error(em)
957	return em
958	not_imported = []
959	imported = []
960	try:
961	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
962	except:
963	em = 'Error reading %s.csv' % filename
964	logger.error(em)
965	return em
966	for item in items:
967	if start:
968	start = False
969	adapters = [MappingStorageAdapter(schema, item)]
970	dm = DataModel(item, adapters,context=self)
971	logger.info('%s starts import from %s.csv' % (member,filename))
972	import_keys = [k for k in item.keys() if not k.startswith('ignore')]
973	diff2schema = set(import_keys).difference(set(schema.keys()))
974	diff2layout = set(import_keys).difference(set(layout.keys()))
975	if diff2schema:
976	em = "not ignorable key(s) %s found in heading" % diff2schema
977	return em
978	s = ','.join(['"%s"' % fn for fn in import_keys])
979	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
980	s = '"id",' + s
981	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
982	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
983	format_error = format + ',"%(Error)s"'
984	format = '"%(id)s",'+ format
985	ds = DataStructure(data=item,datamodel=dm)
986	error_string = ""
987	for k in import_keys:
988	if not validators[k](ds):
989	error_string += " %s : %s" % (k,ds.getError(k))
990	if not error_string:
991	item.update(dm)
992	item['id'],error = importer(item)
993	if error:
994	error_string += error
995	if error_string:
996	item['Error'] = error_string
997	not_imported.append(format_error % item)
998	total_not_imported += 1
999	else:
1000	em = format % item
1001	imported.append(em)
1002	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1003	tr_count += 1
1004	total_imported += 1
1005	total += 1
1006	if total_imported and not total_imported % 100:
1007	transaction.commit()
1008	if len(not_imported) > 0:
1009	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1010	'\n'.join(not_imported) + '\n')
1011	not_imported = []
1012	if len(imported) > 0:
1013	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1014	'\n'.join(imported) + '\n')
1015	imported = []
1016	em = '%d transactions committed\n' % (tr_count)
1017	regs = []
1018	logger.info(em)
1019	tr_count = 0
1020	if len(imported) > 0:
1021	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1022	'\n'.join(imported))
1023	if len(not_imported) > 0:
1024	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1025	'\n'.join(not_imported))
1026	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1027	logger.info(em)
1028	return em
1029	###)
1030
1031	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: