Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 2292

Last change on this file since 2292 was 2292, checked in by joachim, 17 years ago
write student images to file, takes about 3 Minutes per 1000 Students on my system
Property svn:keywords set to `Id`
File size: 60.9 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 2292 2007-09-28 14:47:14Z joachim $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29	from zExceptions import BadRequest
30
31	from Products.CMFCore.utils import getToolByName
32	from Products.CPSSchemas.DataStructure import DataStructure
33	from Products.CPSSchemas.DataModel import DataModel
34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
36	from Products.CMFCore.permissions import View
37	from Products.ZCatalog.ZCatalog import ZCatalog
38	from Products.CMFCore.permissions import ModifyPortalContent
39	from Products.CMFCore.permissions import ManagePortal
40	from Products.CMFCore.utils import UniqueObject
41	from Products.CMFCore.URLTool import URLTool
42	from Products.CMFCore.utils import getToolByName
43	from Students import makeCertificateCode
44	from Globals import package_home,INSTANCE_HOME
45	p_home = package_home(globals())
46	i_home = INSTANCE_HOME
47	import DateTime,time
48	import logging
49	import transaction
50	import csv,re,os,sys
51	from Products.AdvancedQuery import Eq, Between, Le,In
52
53	def getObject(object,name):
54	if object.hasObject(name):
55	return getattr(object,name)
56	return None
57
58	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
59	"""WAeUP tool"""
60
61	id = 'waeup_tool'
62	meta_type = 'WAeUP Tool'
63	_actions = ()
64	security = ClassSecurityInfo()
65	security.declareObjectProtected(View)
66	manage_options = ( ActionProviderBase.manage_options
67	+ SimpleItem.manage_options
68	)
69
70	def rwrite(self,s): ###(
71	response = self.REQUEST.RESPONSE
72	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
73	response.write("%s<br />\r\n" % s)
74	###)
75
76	def sleep(self,secs): ###(
77	"sleep"
78	import time
79	time.sleep(secs)
80	return
81
82	###)
83
84	security.declareProtected(ModifyPortalContent,'openLog') ###(
85	def openLog(self,name):
86	"""open a log file"""
87	version = 1
88	path = "%s/log/%s_%d.log" % (i_home,name,version)
89	while os.path.exists(path):
90	version += 1
91	path = "%s/log/%s_%d.log" % (i_home,name,version)
92	log = open(path,"w")
93	return log
94	###)
95
96	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
97	def bypassQueueCatalog(self,enable=True):
98	"""bypass the QueueCatalog by setting all indexes to process imediate,
99	if enable is True (default) the old settings are restored
100	"""
101
102	###)
103
104	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
105	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
106	"""measure Object access Time"""
107	import random
108	if hasattr(self,'portal_catalog_real'):
109	aq_portal = self.portal_catalog_real.evalAdvancedQuery
110	else:
111	aq_portal = self.portal_catalog.evalAdvancedQuery
112	nr_pts = int(nr_pts)
113	probe = int(probe)
114	intervall = probe/10
115	objects = ("application","clearance","personal")
116	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
117	#i = random.randrange(num_objects)
118	count = 0
119	found = 0
120	not_found = 0
121	t_found = 0
122	t_not_found = 0
123	time_found = time_not_found = 0.0
124	t_time_found = t_time_not_found = 0.0
125	accessed = []
126	t_min = 1000
127	t_max = 0
128	#import pdb;pdb.set_trace()
129	students = self.portal_catalog(portal_type="Student")
130	num_students = len(students)
131	if method == "d":
132	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
133	res = aq_portal(query)
134	brains = {}
135	for r in res:
136	sid = r.relative_path.split('/')[-2]
137	if brains.has_key(sid):
138	brains[sid][r.portal_type] = r
139	else:
140	brains[sid] = {r.portal_type : r}
141	brains_list = brains.keys()
142	num_objects = len(brains_list)
143	else:
144	num_objects = num_students
145	print "="*40
146	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
147	probe,
148	nr_pts,
149	num_objects)
150	print "nr found/not time found/not min/max"
151	elapse = time.time()
152	i_elapse = time.time()
153	c_elapse = time.clock()
154	for c in range(1,probe + 1):
155	i = random.randrange(num_objects)
156	if method in ('a','b','c'):
157	student_brain = students[i]
158	elif method == "d":
159	#import pdb;pdb.set_trace()
160	student_brain = brains[brains_list[i]]
161	if method == "c":
162	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
163	res = aq_portal(query)
164	this_portal_types = [r.portal_type for r in res]
165	for i in range(nr_pts):
166	oid = objects[i]
167	if method == "a":
168	try:
169	student_path = student_brain.getPath()
170	path = "%s/%s" % (student_path,oid)
171	doc = self.unrestrictedTraverse(path).getContent()
172	found += 1
173	i_time = time.time() - i_elapse
174	time_found += i_time
175	except:
176	not_found += 1
177	i_time = time.time() - i_elapse
178	time_not_found += i_time
179	pass
180	elif method == "b":
181	try:
182	student_object = student_brain.getObject()
183	doc = getattr(student_object,oid).getContent()
184	found += 1
185	i_time = time.time() - i_elapse
186	time_found += i_time
187	except:
188	i_time = time.time() - i_elapse
189	time_not_found += i_time
190	not_found += 1
191	pass
192	elif method == "c":
193	if portal_types[i] in this_portal_types:
194	found += 1
195	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
196	i_time = time.time() - i_elapse
197	time_found += i_time
198	else:
199	not_found += 1
200	i_time = time.time() - i_elapse
201	time_not_found += i_time
202	elif method == "d":
203	if student_brain.has_key(portal_types[i]):
204	found += 1
205	doc = student_brain[portal_types[i]].getObject().getContent()
206	i_time = time.time() - i_elapse
207	time_found += i_time
208	else:
209	not_found += 1
210	i_time = time.time() - i_elapse
211	time_not_found += i_time
212	i_elapse = time.time()
213	if c and (c % intervall == 0):
214	#i_time = time.time() - i_elapse
215	t_per = 0.0
216	if found:
217	t_per = time_found/found
218	if t_per > t_max:
219	t_max = t_per
220	if t_per > 0.0 and t_per < t_min:
221	t_min = t_per
222	itf = 0.0
223	if found:
224	itf = time_found/found
225	itnf = 0.0
226	if not_found :
227	itnf = time_not_found / not_found
228	interval_time = time_found + time_not_found
229	s = "%(c)d: %(found)d/%(not_found)d " % vars()
230	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
231	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
232	print s
233	t_found += found
234	t_not_found += not_found
235	t_time_found += time_found
236	t_time_not_found += time_not_found
237	time_found = time_not_found = 0.0
238	found = not_found = 0
239	# t_found += found
240	# t_not_found += not_found
241	elapse = time.time() - elapse
242	itf = 0.0
243	if t_found:
244	itf = t_time_found/t_found
245	itnf = 0.0
246	if t_not_found:
247	itnf = t_time_not_found / t_not_found
248	#c_elapse = time.clock() - c_elapse
249	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
250	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
251	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
252	print "-"*40
253	print s
254	rel_found = float(t_found)/probe
255	rel_not_found = float(t_not_found)/probe
256	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
257	print estimated_total_time
258	###)
259
260	security.declareProtected(ModifyPortalContent,'writeLog') ###(
261	def writeLog(self,logfile,s):
262	"""write to the log file"""
263	logfile.write(s)
264
265	###)
266
267	def generateStudentId(self,letter): ###(
268	import random
269	r = random
270	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
271	if letter == '?':
272	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
273	sid = "%c%d" % (letter,r.randint(99999,1000000))
274	students = self.portal_url.getPortalObject().campus.students
275	## while hasattr(students, sid):
276	## sid = "%c%d" % (letter,r.randint(99999,1000000))
277	while self.students_catalog(id = sid):
278	sid = "%c%d" % (letter,r.randint(99999,1000000))
279	return sid
280	###)
281
282	def generatePassword(self,s=None): ###(
283	import random
284	r = random
285	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
286	if s is None:
287	s = 'abcdefghklmnpqrstuvwxy23456789'
288	pw = ''
289	while len(pw) < 6:
290	pw += r.choice(s)
291	return pw
292	###)
293
294	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
295	def removeDeletedDocIds(self, max=1000):
296	"""
297	remove deleted docids from repository commit after max
298	"""
299	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
300	repository = getToolByName(self, 'portal_repository')
301	pxtool = getToolByName(self, 'portal_proxies')
302	pxtool_infos = pxtool.getRevisionsUsed()
303
304	nb_revs = 0
305	docids_d = {} # all docids
306	unused_docids_d = {} # all docids that are unused
307	ids_unused_revs_docids = [] # ids for revs of unused docids
308	ids_unused_revs = [] # ids for unused revs
309	total = 0
310	idlist = repository.objectIds()
311	for id in idlist:
312	docid, rev = repository._splitId(id)
313	if docid is None:
314	logger.info("invalid doc_id %s" % docid)
315	continue
316	nb_revs += 1
317	docids_d[docid] = None
318	if not pxtool_infos.has_key(docid):
319	unused_docids_d[docid] = None
320	ids_unused_revs_docids.append(id)
321	ids_unused_revs.append(id)
322	elif not pxtool_infos[docid].has_key(rev):
323	ids_unused_revs.append(id)
324	if len(ids_unused_revs) >= max:
325	repository.manage_delObjects(ids_unused_revs)
326	#import pdb;pdb.set_trace()
327	transaction.commit()
328	total += max
329	logger.info('removed %d total %d unused docids ' % (max,total))
330	anz = len(ids_unused_revs)
331	if anz > 0:
332	repository.manage_delObjects(ids_unused_revs)
333	transaction.commit()
334	total += anz
335	logger.info('finished removing %d unused docids ' % (total))
336
337
338	###)
339
340	security.declareProtected(ModifyPortalContent,'getCredential') ###(
341	def getCredential(self,student_id):
342	"return a student password"
343	student_entry = getattr(self.portal_directories.students,student_id,None)
344	if student_entry is None:
345	return None
346	return getattr(student_entry,"password","not set")
347	###)
348
349	security.declarePublic('checkPassword') ###(
350	def checkPassword(self,student_id,password):
351	"return a student password"
352	student_entry = getattr(self.portal_directories.students,student_id,None)
353	if student_entry is None:
354	return False
355	return getattr(student_entry,"password","not set") == password
356	###)
357
358	security.declarePublic('editPassword') ###(
359	def editPassword(self,student_id,password):
360	"edit a student password"
361	student_entry = getattr(self.portal_directories.students,student_id,None)
362	if student_entry is None:
363	return
364	setattr(student_entry,'password',password)
365	###)
366
367	security.declareProtected(View,'doCommit') ###(
368	def doCommit(self,logger=None):
369	"commit some transactions"
370	transaction.commit()
371	###)
372
373	security.declarePublic('loadStudentFoto') ###(
374	def loadStudentFoto(self,student,filename,folder):
375	"return a student passport picture"
376	app = student.application
377	app_doc = app.getContent()
378	#clear = student.clearance
379	#clear_doc = clear.getContent()
380	#matric_no = clear_doc.matric_no.upper()
381	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
382	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
383	#import pdb;pdb.set_trace()
384	if os.path.exists(picture1):
385	file = open(picture1)
386	elif os.path.exists(picture2):
387	file = open(picture2)
388	else:
389	return "passport picture not found %s" % picture1
390	reopened = False
391	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
392	self.portal_workflow.doActionFor(app,'open')
393	reopened = True
394	outfile = file.read()
395	app_doc.manage_addFile('passport',
396	file=outfile,
397	title="%s.jpg" % filename)
398	if reopened:
399	self.portal_workflow.doActionFor(app,'close')
400	return "successfully loaded passport picture"
401	###)
402
403	security.declareProtected(ModifyPortalContent,'createOne') ###(
404	def createOne(self,students_folder,student_brain,letter,commit=False):
405	sid = self.waeup_tool.generateStudentId(letter)
406	students_folder.invokeFactory('Student', sid)
407	student = getattr(students_folder,sid)
408	self.portal_workflow.doActionFor(student,'return')
409	student.manage_setLocalRoles(sid, ['Owner',])
410	matric_no = student_brain.matric_no
411	jamb_reg_no = student_brain.Entryregno
412	self.students_catalog.addRecord(id = sid,
413	matric_no = matric_no,
414	jamb_reg_no = jamb_reg_no,
415	sex = student_brain.Sex == "F",
416	name = "%s %s %s" % (student_brain.Firstname,
417	student_brain.Middlename,
418	student_brain.Lastname)
419	)
420	if commit:
421	transaction.commit()
422	return sid,jamb_reg_no
423	###)
424
425	security.declareProtected(ModifyPortalContent,'addStudent') ###(
426	def addStudent(self,dict):
427	students_folder = self.portal_url.getPortalObject().campus.students
428	sid = self.waeup_tool.generateStudentId('?')
429	students_folder.invokeFactory('Student', sid)
430	student_obj = getattr(students_folder,sid)
431	f2t = self.student_field2types
432	#from pdb import set_trace; set_trace()
433	d = {}
434	d['jamb_sex'] = 'M'
435	if dict.get('sex'):
436	d['jamb_sex'] = 'F'
437
438	entry_session = dict.get('entry_session')
439	if entry_session == self.getSessionId()[-2:]:
440	wfaction = 'admit'
441	wft = 'wf_transition_admit'
442	password = None
443	else:
444	wfaction = 'return'
445	wft = 'wf_transition_return'
446	password = self.generatePassword()
447	self.makeStudentMember(sid,password)
448
449	for pt in f2t.keys():
450	student_obj.invokeFactory(pt,f2t[pt]['id'])
451	sub_obj = getattr(student_obj,f2t[pt]['id'])
452	sub_doc = sub_obj.getContent()
453	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
454	d['Title'] = f2t[pt]['title']
455	for field in f2t[pt]['fields']:
456	d[field] = dict.get(field,'')
457	sub_doc.edit(mapping = d)
458	new_state = f2t[pt][wft]
459	if new_state != "remain":
460	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
461	self.portal_workflow.doActionFor(student_obj,wfaction)
462	student_obj.manage_setLocalRoles(sid, ['Owner',])
463	return sid,password
464	###)
465
466	security.declarePublic('getCertificateBrain') ###(
467	def getCertificateBrain(self,cert_id):
468	"do it"
469	res = ZCatalog.searchResults(self.portal_catalog_real,
470	{'portal_type':"Certificate",
471	'id': cert_id})
472	if res:
473	return res[0]
474	return None
475	###)
476
477	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
478	def get_csv_filenames(self):
479	"do it"
480	files = [file for file in os.listdir("%s/import/" % (i_home))
481	if file.endswith('.csv') and file.find('imported') == -1]
482	return files
483	###)
484
485	security.declarePublic('findStudentByMatricelNo') ###(
486	def findStudentByMatricelNo(self,matric_no):
487	"do it"
488	res = ZCatalog.searchResults(self.portal_catalog_real,
489	{'portal_type':"StudentClearance",
490	'SearchableText': matric_no})
491	if res:
492	return res[0]
493	return None
494	###)
495
496	security.declarePublic('makeStudentMember') ###(
497	def makeStudentMember(self,sid,password='uNsEt'):
498	"""make the student a member"""
499	membership = self.portal_membership
500	membership.addMember(sid,
501	password ,
502	roles=('Member',
503	'Student',
504	),
505	domains='',
506	properties = {'memberareaCreationFlag': False,
507	'homeless': True},)
508	member = membership.getMemberById(sid)
509	self.portal_registration.afterAdd(member, sid, password, None)
510	#self.manage_setLocalRoles(sid, ['Owner',])
511	###)
512
513	security.declarePublic('makeStudentData') ###(
514	def makeStudentData(self,student_id,email=None,phone_nr=None):
515	"create Datastructure for a returning Student"
516	#import pdb;pdb.set_trace()
517	logger = logging.getLogger('WAeUPTool.makeStudentData')
518	students_folder = self.portal_url.getPortalObject().campus.students
519	#res = self.students_catalog(id=student_id)
520	#if res:
521	# st = res[0]
522	#res = self.returning_import(matric_no = st.matric_no)
523	res = self.returning_import(id = student_id)
524	if res:
525	student = res[0]
526	else:
527	logger.info('Id %s not found in returning_import' % student_id)
528	return
529	logger.info('%s creates data structure' % student_id)
530	s_results = self.results_import(matric_no = student.matric_no)
531	if s_results:
532	lnr = self.getLevelFromResultsCosCode(s_results)
533	level = "%d00" % lnr
534	verdict,eligible = self.getVerdict(s_results[0].Verdict)
535	if eligible:
536	level = "%d00" % (lnr + 1)
537	else:
538	logger.info('matric_no %s not found in results_import' % student.matric_no)
539	level = 0
540	verdict = 'N/A'
541	#student should not be allowed to perform this transition
542	#wftool = self.portal_workflow
543	#wftool.doActionFor(student,'return')
544	certcode_org = student.Coursemajorcode
545	certcode = makeCertificateCode(certcode_org)
546	certificate_brain = self.getCertificateBrain(certcode)
547	if not certificate_brain:
548	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
549	logger.info(em)
550	matric_no = student.matric_no
551	sid = student_id
552	student_obj = getattr(students_folder,sid)
553	student_obj.invokeFactory('StudentApplication','application')
554	application = student_obj.application
555	self.portal_workflow.doActionFor(application,'open',dest_container=application)
556	da = {'Title': 'Application Data'}
557	student_obj.invokeFactory('StudentPersonal','personal')
558	da['jamb_reg_no'] = student.Entryregno
559	em = self.getEntryMode(student.Entryregno)
560	## em = student.Mode_of_Entry
561	## if em in ('DIRECT', 'DIRECT ENTRY',):
562	## em = 'DE'
563	## elif em in ('U.M.E', 'UNE',):
564	## em = 'UME'
565	## elif not em:
566	## em = "unknown"
567	da['entry_mode'] = em
568	personal = student_obj.personal
569	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
570	dp = {'Title': 'Personal Data'}
571	student_obj.invokeFactory('StudentClearance','clearance')
572	clearance = student_obj.clearance
573	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
574	dc = {'Title': 'Clearance/Eligibility Record'}
575	dc['matric_no'] = matric_no
576	state = student.State
577	lga = student.LGA
578	if state and lga:
579	lga = state + ' / ' + lga
580	else:
581	lga = "None"
582	da['jamb_lga'] = dc['lga'] = lga
583	da['app_email'] = dp['email'] = email
584	da['app_mobile'] = dp['phone'] = phone_nr
585	dp['firstname'] = student.Firstname
586	dp['middlename'] = student.Middlename
587	dp['lastname'] = student.Lastname
588	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
589	da['jamb_sex'] = student.Sex
590	dp['sex'] = student.Sex == 'F'
591	dp['perm_address'] = student.Permanent_Address
592	application.getContent().edit(mapping=da)
593	self.portal_workflow.doActionFor(application,'close',dest_container=application)
594	personal.getContent().edit(mapping=dp)
595	clearance.getContent().edit(mapping=dc)
596	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
597	## catd = {}
598	## catd['id'] = sid
599	## catd['entry_mode']= da['entry_mode']
600	## catd['matric_no'] = matric_no
601	## catd['jamb_reg_no'] = da['jamb_reg_no']
602	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
603	## catd['sex'] = dp['sex']
604	## catd['level'] = level
605	## catd['verdict'] = verdict
606	## if certificate_brain:
607	## cpath = certificate_brain.getPath().split('/')
608	## catd['faculty'] = cpath[-4]
609	## catd['department'] = cpath[-3]
610	## catd['course'] = certcode
611	## self.students_catalog.modifyRecord(**catd)
612	#
613	# Study Course
614	#
615	student_obj.invokeFactory('StudentStudyCourse','study_course')
616	studycourse = student_obj.study_course
617	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
618	dsc = {}
619	dsc['study_course'] = certcode
620	dsc['current_level'] = level
621	dsc['current_verdict'] = verdict
622	dsc['current_mode'] = em
623	dsc['current_session'] = '06'
624	studycourse.getContent().edit(mapping=dsc)
625	#
626	# Level
627	#
628	## l = getattr(studycourse,level,None)
629	## if l is None:
630	## studycourse.invokeFactory('StudentStudyLevel', level)
631	## l = getattr(studycourse, level)
632	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
633	## l.getContent().edit(mapping={'Title': "Level %s" % level})
634	###)
635
636	security.declarePublic('makeStudentLevel') ###(
637	def makeStudentLevel(self,student_id):
638	"create the StudyLevel for a returning Student"
639	#import pdb;pdb.set_trace()
640	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
641	students_folder = self.portal_url.getPortalObject().campus.students
642	res = self.students_catalog(id=student_id)
643	if res:
644	st = res[0]
645	course = st.course
646	matric_no = st.matric_no
647	level = st.level
648	res = self.results_import(matric_no = matric_no)
649	if res:
650	results = res
651	logger.info('%s creating Level %s' % (student_id,level))
652	#
653	# Level
654	#
655	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
656	studycourse = getattr(student_obj,"study_course",None)
657	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
658	l = getattr(studycourse,level,None)
659	if l is None:
660	studycourse.invokeFactory('StudentStudyLevel', level)
661	l = getattr(studycourse, level)
662	self.portal_workflow.doActionFor(l,'open',dest_container=l)
663	l.getContent().edit(mapping={'Title': "Level %s" % level})
664	###)
665
666	security.declarePublic('getHallInfo') ###(
667	def getHallInfo(self,bed):
668	"""return Hall Info"""
669	info = {}
670	hall,block,room,letter = bed.split('_')
671	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
672	if res and len(res) == 1:
673	hall_brain = res[0]
674	hall_doc = hall_brain.getObject().getContent()
675	else:
676	return info
677	info['hall_title'] = hall_brain.Title
678	info['maintenance_code'] = hall_doc.maintenance_code
679	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
680	batch_doc = None
681	for brain in res:
682	if brain.id.startswith(info['maintenance_code']):
683	batch_doc = brain.getObject().getContent()
684	break
685	if batch_doc is None:
686	info['maintenance_fee'] = None
687	else:
688	info['maintenance_fee'] = batch_doc.cost
689	return info
690	###)
691
692	security.declarePublic('showFsPicture') ###(
693	def showFsPicture(self,path):
694	"""return a picture from the filesystem"""
695	picture_path = "%s/import/%s" % (i_home,path)
696	if os.path.exists(picture_path):
697	return open(picture_path).read()
698
699	###)
700
701	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
702	def deleteAllCourses(self,department="All"):
703	''' delete the courses'''
704	pm = self.portal_membership
705	member = pm.getAuthenticatedMember()
706
707	if str(member) not in ("henrik","joachim"):
708	return "not possible"
709	if department == "All":
710	res = self.portal_catalog({'meta_type': 'Department'})
711	if len(res) < 1:
712	return "No Departments found"
713
714	deleted = []
715	for dep in res:
716	cf = dep.getObject().courses
717	if cf:
718	cf.manage_delObjects(ids=cf.objectIds())
719	deleted.append("deleted Courses in %s" % dep.getId)
720	return "\r".join(deleted)
721	###)
722
723	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
724	def getLogfileLines(self,filename="event.log",numlines=20):
725	"""Get last NUMLINES lines of logfile FILENAME.
726
727	Return last lines' of a file in the instances logfile directory as
728	a list. The number of returned lines equals `numlines' or less. If
729	less than `numlines' lines are available, the whole file ist
730	returned. If the file can not be opened or some other error
731	occurs, empty list is returend.
732	"""
733	result = []
734	lines_hit = 0
735
736	# We only handle files in instances' log directory...
737	logpath = os.path.join(i_home, "log")
738	filename = str(os.path.abspath( os.path.join( logpath, filename )))
739	if not filename.startswith( logpath ):
740	# Attempt to access file outside log-dir...
741	return []
742
743	try:
744	fd = file( filename, "rb" )
745	except IOError:
746	return []
747	if not fd:
748	return []
749
750	if os.linesep == None:
751	linesep = '\n'
752	else:
753	linesep = os.linesep
754
755	# Try to find 'numlines' times a lineseparator, searching from end
756	# and moving to the beginning of file...
757	fd.seek( 0, 2) # Move to end of file...
758	while lines_hit < numlines:
759	if fd.read(1) == linesep[-1]: # This moves filedescriptor
760	# one step forward...
761	lines_hit += 1
762	try:
763	fd.seek( -2, 1) # Go two bytes back from current pos...
764	except IOError:
765	# We cannot go back two bytes. Maybe the file is too small...
766	break
767	fd.seek(2,1)
768
769	# Read all lines from current position...
770	result = fd.readlines()
771	# Remove line endings...
772	result = [x.strip() for x in result]
773	fd.close()
774	return result
775	###)
776
777	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
778	def getCallbacksFromLog(self,filename):
779	"""fix Online Payment Transactions from Z2.log entries"""
780	import transaction
781	import random
782	from cgi import parse_qs
783	from urlparse import urlparse
784	#from pdb import set_trace
785	wftool = self.portal_workflow
786	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
787	students_folder = self.portal_url.getPortalObject().campus.students
788	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
789	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
790	s += r'(?P<intersw>.)" "(?P<agent>.)"'
791	data = re.compile(s)
792	start = True
793	tr_count = 1
794	total = 0
795	#name = 'pume_results'
796	#name = 'epaymentsuccessful_z2log2'
797	name = filename
798	no_import = []
799	imported = []
800	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
801	try:
802	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
803	except:
804	logger.error('Error reading %s' % name)
805	return
806	tas = []
807	for line in transactions:
808	dict = {}
809	items = data.search(line)
810	dict['idict'] = idict = items.groupdict()
811	#print idict
812	#from pdb import set_trace;set_trace()
813	urlparsed = urlparse(idict['get'][4:])
814	#print urlparsed
815	path = urlparsed[2].split('/')
816	dict['student_id'] = student_id = path[8]
817	dict['payment_id'] = payment_id = path[10]
818	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
819	tas.append(dict)
820	tr_count += 1
821	return tas
822	###)
823
824	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
825	def importOnlinePaymentTransactions(self):
826	"""load Online Payment Transactions from CSV values"""
827	import transaction
828	import random
829	#from pdb import set_trace
830	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
831	opt = self.online_payments_import
832	students_folder = self.portal_url.getPortalObject().campus.students
833	start = True
834	tr_count = 1
835	total = 0
836	#name = 'pume_results'
837	name = 'OnlineTransactions'
838	no_import = []
839	imported = []
840	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
841	try:
842	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
843	except:
844	logger.error('Error reading %s.csv' % name)
845	return
846	for pay_transaction in transactions:
847	if start:
848	start = False
849	logger.info('Start loading from %s.csv' % name)
850	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
851	no_import.append('%s,"Error"' % s)
852	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
853	format_error = format + ',"%(Error)s"'
854	data = {}
855
856	# format of the first file sent by Tayo
857	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
858	#data['student_id'] = student_id = pay_transaction['Payer ID']
859	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
860	#data['response_code'] = response_code = pay_transaction['Resp Code']
861	#data['amount'] = amount = pay_transaction['Amount']
862
863	# format of the second file sent by Tayo
864	#data['datetime'] = date = 0
865	#data['student_id'] = student_id = pay_transaction['Payer ID']
866	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
867	#data['response_code'] = response_code = '00'
868	#data['amount'] = amount = pay_transaction['Amount']
869
870	# format of the third file sent by Kehinde
871	data['datetime'] = date = 0
872	data['student_id'] = student_id = pay_transaction['customer_id']
873	data['order_id'] = order_id = pay_transaction['merchant_reference']
874	data['response_code'] = response_code = '00'
875	data['amount'] = amount = pay_transaction['Amount']
876
877	dup = False
878	if response_code == "12":
879	continue
880	try:
881	opt.addRecord(**data)
882	except ValueError:
883	dup = True
884	#from pdb import set_trace;set_trace()
885	if dup:
886	if response_code == "00":
887	try:
888	opt.modifyRecord(**data)
889	except:
890	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
891	continue
892	else:
893	pay_transaction['Error'] = "Duplicate order_id"
894	no_import.append( format_error % pay_transaction)
895	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
896	continue
897	tr_count += 1
898	if tr_count > 1000:
899	if len(no_import) > 0:
900	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
901	'\n'.join(no_import) + '\n')
902	no_import = []
903	em = '%d transactions committed\n' % (tr_count)
904	transaction.commit()
905	regs = []
906	logger.info(em)
907	total += tr_count
908	tr_count = 0
909	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
910	'\n'.join(no_import))
911	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
912	###)
913
914	security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
915	def mass_create_faculty(self,mapping):
916	"create a faculty"
917	logger = logging.getLogger('WAeUPTool.mass_create_faculty')
918	academics_folder = self.portal_url.getPortalObject().campus.academics
919	fid = mapping['code']
920	if getattr(academics_folder,fid,None) is not None:
921	return '', "Faculty with ID: %s exists" % fid
922	logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
923	try:
924	academics_folder.invokeFactory('Faculty', fid)
925	except BadRequest,E:
926	return '', "%s" % E
927	f = getattr(academics_folder,fid,None)
928	f.getContent().edit(mapping=mapping)
929	return fid,''
930	###)
931
932	security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
933	def mass_edit_faculty(self,mapping):
934	"edit a faculty"
935	logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
936	academics_folder = self.portal_url.getPortalObject().campus.academics
937	fid = mapping['code']
938	f = getattr(academics_folder,fid,None)
939	if f is None:
940	return '', "Faculty with ID: %s does not exist" % fid
941	logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
942	f.getContent().edit(mapping=mapping)
943	return fid,''
944	###)
945
946	security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
947	def mass_create_department(self,mapping):
948	"create a department in the correct faculty"
949	logger = logging.getLogger('WAeUPTool.mass_create_department')
950	fid = mapping['faculty_code']
951	if getattr(self,'_v_faculties',None) is None:
952	res = self.portal_catalog(portal_type = "Faculty")
953	self._v_faculties = {}
954	for f in res:
955	self._v_faculties[f.getId] = f.getObject()
956	f = self._v_faculties.get(fid,None)
957	if f is None:
958	return '', "No Faculty with ID: %s" % fid
959	else:
960	did = mapping.get('code')
961	d = getattr(f,did,None)
962	if d is None or d.portal_type == "Faculty":
963	logger.info('Creating Department %(code)s, %(title)s' % mapping)
964	try:
965	f.invokeFactory('Department', did)
966	except BadRequest,E:
967	return '', "%s" % E
968	d = getattr(f,did)
969	d.invokeFactory('CoursesFolder','courses')
970	courses = getattr(d,'courses')
971	dict = {'Title': 'Courses'}
972	courses.getContent().edit(mapping=dict)
973	d.invokeFactory('CertificatesFolder','certificates')
974	certificates = getattr(d,'certificates')
975	dict = {'Title': 'Certificates'}
976	certificates.getContent().edit(mapping=dict)
977	d.getContent().edit(mapping=mapping)
978	return did,''
979	###)
980
981	security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
982	def mass_edit_department(self,mapping):
983	"create a department in the correct faculty"
984	logger = logging.getLogger('WAeUPTool.mass_create_department')
985	academics_folder = self.portal_url.getPortalObject().campus.academics
986	fid = mapping['faculty_code']
987	did = mapping.get('code')
988	try:
989	d = getattr(getattr(academics_folder,fid),did,None)
990	except KeyError:
991	return '', "Department %s or Faculty %s wrong" % (did,fid)
992	else:
993	if d is None or d.portal_type == "Faculty":
994	logger.info('Editing Department %(code)s, %(title)s' % mapping)
995	d.getContent().edit(mapping=mapping)
996	return did,''
997	###)
998
999	security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1000	def mass_create_course(self,mapping):
1001	if getattr(self,'_v_course_list',None) is None:
1002	self._v_course_list = []
1003	if getattr(self,'_v_departments',None) is None:
1004	res = self.portal_catalog(portal_type = "Department")
1005	self._v_departments = {}
1006	for d in res:
1007	self._v_departments[d.getId] = getattr(d.getObject(),"courses",None)
1008	did = mapping['department_code']
1009	d = self._v_departments.get(did,None)
1010	if d is None:
1011	return '', "No Department with ID: %s" % did
1012	course_id = mapping.get('code')
1013	if course_id in self._v_course_list:
1014	return '', "Duplicate Department ID: %s" % did
1015	c = getattr(d,course_id,None)
1016	if c is not None:
1017	return '', "Duplicate Department ID: %s" % did
1018	try:
1019	d.invokeFactory('Course', course_id)
1020	except BadRequest,E:
1021	return '', "%s" % E
1022	self._v_course_list.append(course_id)
1023	c = getattr(d,course_id)
1024	c.getContent().edit(mapping=mapping)
1025	return course_id,''
1026	###)
1027
1028	security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1029	def mass_edit_course(self,mapping):
1030	#import pdb;pdb.set_trace()
1031	course_id = mapping.get('code')
1032	res = self.portal_catalog(id=course_id)
1033	if not res:
1034	return '', "No Course with ID: %s" % course_id
1035	c = res[0].getObject()
1036	c.getContent().edit(mapping=mapping)
1037	return course_id,''
1038	###)
1039
1040	security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1041	def mass_create_certificate(self,mapping):
1042	if getattr(self,'_v_certificate_list',None) is None:
1043	self._v_certificate_list = []
1044	if getattr(self,'_v_departments',None) is None:
1045	res = self.portal_catalog(portal_type = "Department")
1046	self._v_departments = {}
1047	for d in res:
1048	self._v_departments[d.getId] = getattr(d.getObject(),"certificates",None)
1049	did = mapping['department_code']
1050	d = self._v_departments.get(did,None)
1051	if d is None:
1052	return '', "No Department with ID: %s" % did
1053	certificate_id = mapping.get('code')
1054	if certificate_id in self._v_certificate_list:
1055	return '', "Duplicate Department ID: %s" % did
1056	c = getattr(d,certificate_id,None)
1057	if c is not None:
1058	return '', "Duplicate Department ID: %s" % did
1059	try:
1060	d.invokeFactory('Certificate', certificate_id)
1061	except BadRequest,E:
1062	return '', "%s" % E
1063	self._v_certificate_list.append(certificate_id)
1064	c = getattr(d,certificate_id)
1065	c.getContent().edit(mapping=mapping)
1066	return certificate_id,''
1067	###)
1068
1069	security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1070	def mass_edit_certificate(self,mapping):
1071	#import pdb;pdb.set_trace()
1072	certificate_id = mapping.get('code')
1073	res = self.portal_catalog(id=certificate_id)
1074	if not res:
1075	return '', "No Certificate with ID: %s" % did
1076	c = res[0].getObject()
1077	c.getContent().edit(mapping=mapping)
1078	return certificate_id,''
1079	###)
1080
1081	security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1082	def mass_create_certificate_course(self,mapping):
1083	if getattr(self,'_v_certificate_course_list',None) is None:
1084	self._v_certificate_course_list = []
1085	if getattr(self,'_v_ceritficates',None) is None:
1086	res = self.portal_catalog(portal_type = "Certificate")
1087	self._v_certificates = {}
1088	for cert in res:
1089	self._v_certificates[cert.getId] = cert.getObject()
1090	certificate_course_id = mapping.get('code')
1091	if not self.courses_catalog(code = certificate_course_id):
1092	return '', "No Course with ID: %s" % certificate_course_id
1093	cert_id = mapping['certificate_code']
1094	cert = self._v_certificates.get(cert_id,None)
1095	if cert is None:
1096	return '', "No Certificate with ID: %s" % cert_id
1097	if certificate_course_id in self._v_certificate_course_list:
1098	return '', "Duplicate CertificateCourse ID: %s" % certificate_course_id
1099	level_id = mapping.get('level')
1100	level = getattr(cert,level_id,None)
1101	if level is None:
1102	cert.invokeFactory('StudyLevel', level_id)
1103	level = getattr(cert,level_id,None)
1104	elif hasattr(level,certificate_course_id):
1105	return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1106	(certificate_course_id,cert_id,level_id)
1107	level.invokeFactory('CertificateCourse', certificate_course_id)
1108	self._v_certificate_course_list.append(certificate_course_id)
1109	c = getattr(level,certificate_course_id)
1110	c.getContent().edit(mapping=mapping)
1111	return certificate_course_id,''
1112	###)
1113
1114	field2types_student = { ###(
1115	'StudentApplication':
1116	{'id': 'application',
1117	'title': 'Application Data',
1118	'wf_transition_return': 'close',
1119	'wf_transition_admit': 'remain',
1120	'fields':
1121	('jamb_reg_no',
1122	'entry_mode',
1123	'entry_session',
1124	'jamb_score',
1125	'app_email',
1126	'jamb_age',
1127	'jamb_state',
1128	'jamb_lga',
1129	)
1130	},
1131	#'StudentPume':
1132	# {'id': 'pume',
1133	# 'title': 'Pume Data',
1134	# 'wf_transition_return': 'close',
1135	# 'wf_transition_admit': 'close',
1136	# 'fields':
1137	# ('pume_score',
1138	# )
1139	# },
1140	'StudentClearance':
1141	{'id': 'clearance',
1142	'title': 'Clearance/Eligibility Record',
1143	'wf_transition_return': 'close',
1144	'wf_transition_admit': 'remain',
1145	'fields':
1146	('matric_no',
1147	'nationality',
1148	'lga',
1149	'birthday',
1150	)
1151	},
1152	'StudentPersonal':
1153	{'id': 'personal',
1154	'title': 'Personal Data',
1155	'wf_transition_return': 'open',
1156	'wf_transition_admit': 'remain',
1157	'fields':
1158	('firstname',
1159	'middlename',
1160	'lastname',
1161	'sex',
1162	'email',
1163	'phone',
1164	'perm_address',
1165	)
1166	},
1167	'StudentStudyCourse':
1168	{'id': 'study_course',
1169	'title': 'Study Course',
1170	'wf_transition_return': 'open',
1171	'wf_transition_admit': 'remain',
1172	'fields':
1173	('study_course',
1174	'current_level',
1175	'current_session',
1176	'current_mode',
1177	'current_verdict',
1178	)
1179	},
1180
1181	'PaymentsFolder':
1182	{'id': 'payments',
1183	'title': 'Payments',
1184	'wf_transition_return': 'open',
1185	'wf_transition_admit': 'open',
1186	'fields':
1187	()
1188	},
1189	}
1190	###)
1191
1192	security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1193	def mass_create_student(self,mapping):
1194	"create a students record due import"
1195	logger = logging.getLogger('WAeUPTool.mass_create_student')
1196	students_folder = self.portal_url.getPortalObject().campus.students
1197	jamb_reg_no = mapping.get('jamb_reg_no',None)
1198	if jamb_reg_no:
1199	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1200	if res:
1201	return '',"jamb_reg_no exists"
1202	matric_no = mapping.get('matric_no',None)
1203	if matric_no:
1204	res = self.students_catalog(matric_no = matric_no)
1205	if res:
1206	return '',"matric_no exists"
1207	sid = self.waeup_tool.generateStudentId('?')
1208	students_folder.invokeFactory('Student', sid)
1209	student_obj = getattr(students_folder,sid)
1210	f2t = self.field2types_student
1211	d = {}
1212	d['jamb_sex'] = 'M'
1213	if mapping.get('sex'):
1214	d['jamb_sex'] = 'F'
1215	transition = mapping.get('reg_transition','admit')
1216	if transition not in ('admit','return'):
1217	return '',"no valid transition provided"
1218	for pt in f2t.keys():
1219	student_obj.invokeFactory(pt,f2t[pt]['id'])
1220	sub_obj = getattr(student_obj,f2t[pt]['id'])
1221	sub_doc = sub_obj.getContent()
1222	d['Title'] = f2t[pt]['title']
1223	for field in f2t[pt]['fields']:
1224	d[field] = mapping.get(field,'')
1225	if pt == "StudyCourse":
1226	for von,zu in (('entry_mode','current_mode'),
1227	('entry_session','current_session')):
1228	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1229	d[zu] = mapping[von]
1230	sub_doc.edit(mapping = d)
1231
1232	#import pdb;pdb.set_trace()
1233	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1234	if new_state != "remain":
1235	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1236	self.portal_workflow.doActionFor(student_obj,transition)
1237	student_obj.manage_setLocalRoles(sid, ['Owner',])
1238	return sid,''
1239	###)
1240
1241	security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1242	def mass_edit_student(self,mapping):
1243	"edit a students record due import"
1244	logger = logging.getLogger('WAeUPTool.mass_edit_student')
1245	students_folder = self.portal_url.getPortalObject().campus.students
1246	sid = mapping.get('id',None)
1247	jamb_reg_no = mapping.get('jamb_reg_no',None)
1248	matric_no = mapping.get('matric_no',None)
1249	editable_keys = mapping.keys()
1250	if sid:
1251	res = self.students_catalog(id = sid)
1252	if not res:
1253	return '',"no student with id %s" % sid
1254	if matric_no and res[0].matric_no and\
1255	matric_no != res[0].matric_no:
1256	logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1257	if jamb_reg_no and res[0].jamb_reg_no and\
1258	jamb_reg_no != res[0].jamb_reg_no:
1259	logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1260	elif jamb_reg_no:
1261	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1262	if not res:
1263	return '',"no student with jamb_reg_no %s" % jamb_reg_no
1264	editable_keys.remove('jamb_reg_no')
1265	elif matric_no:
1266	res = self.students_catalog(matric_no = matric_no)
1267	if not res:
1268	return '',"no student with matric_no %s" % matric_no
1269	editable_keys.remove('matric_no')
1270
1271	## included only to change wf state from admitted to returning
1272	#if res[0].review_state not in ('admitted','objection_raised'):
1273	# return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1274	## end inclusion
1275
1276	sid = res[0].id
1277	student_obj = getattr(students_folder,sid)
1278	f2t = self.field2types_student
1279	d = {}
1280	#import pdb;pdb.set_trace()
1281	any_change = False
1282	for pt in f2t.keys():
1283	if pt == "student_application":
1284	d['jamb_sex'] = 'M'
1285	if mapping.get('sex'):
1286	d['jamb_sex'] = 'F'
1287	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1288	if intersect:
1289	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1290	if sub_obj is None:
1291	try:
1292	student_obj.invokeFactory(pt,f2t[pt]['id'])
1293	except:
1294	continue
1295	sub_obj = getattr(student_obj,f2t[pt]['id'])
1296	d['Title'] = f2t[pt]['title']
1297	sub_doc = sub_obj.getContent()
1298	for field in intersect:
1299	changed = False
1300	if getattr(sub_doc,field) != mapping.get(field,''):
1301	any_change = True
1302	changed = True
1303	d[field] = mapping.get(field,'')
1304	if changed:
1305	sub_doc.edit(mapping = d)
1306
1307
1308	## included only to change wf state from admitted to returning
1309	# if res[0].review_state in ('admitted','objection_raised'):
1310	# new_state = f2t[pt]['wf_transition_return']
1311	# sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1312	# if sub_obj and new_state != "remain":
1313	# try:
1314	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1315	# except:
1316	# #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1317	# pass
1318	#if res[0].review_state in ('admitted','objection_raised'):
1319	# wfaction = 'return'
1320	# try:
1321	# self.portal_workflow.doActionFor(student_obj,wfaction)
1322	# logger.info('%s, wf state changed' % sid)
1323	# any_change = True
1324	# except:
1325	# logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1326	# pass
1327	## end inclusion
1328
1329
1330	if any_change:
1331	return sid,''
1332	else:
1333	return sid,'not modified'
1334	###)
1335
1336	security.declareProtected(ModifyPortalContent,"importData")###(
1337	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1338	"""load data from CSV values"""
1339	import transaction
1340	import random
1341
1342	pm = self.portal_membership
1343	member = pm.getAuthenticatedMember()
1344
1345	logger = logging.getLogger('WAeUPTool.importData')
1346	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1347	students_folder = self.portal_url.getPortalObject().campus.students
1348	start = True
1349	tr_count = 1
1350	total_imported = 0
1351	total_not_imported = 0
1352	total = 0
1353	iname = "import_%s" % name
1354	stool = getToolByName(self, 'portal_schemas')
1355	ltool = getToolByName(self, 'portal_layouts')
1356	schema = stool._getOb(iname)
1357	if schema is None:
1358	em = 'No such schema %s' % iname
1359	logger.error('No such schema %s' % iname)
1360	return em
1361	layout = ltool._getOb(iname)
1362	if layout is None:
1363	em = 'No such layout %s' % iname
1364	logger.error(em)
1365	return em
1366	validators = {}
1367	for widget in layout.keys():
1368	validators[widget] = layout[widget].validate
1369	#import pdb;pdb.set_trace()
1370	mode = "create"
1371	if edit:
1372	mode = "edit"
1373	importer_name = "mass_%(mode)s_%(name)s" % vars()
1374	importer = getattr(self, '%s' % importer_name,None)
1375	if importer is None:
1376	em = 'No importer function %s' % importer_name
1377	logger.error(em)
1378	return em
1379	not_imported = []
1380	imported = []
1381	try:
1382	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1383	except:
1384	em = 'Error reading %s.csv' % filename
1385	logger.error(em)
1386	return em
1387	for item in items:
1388	if start:
1389	start = False
1390	adapters = [MappingStorageAdapter(schema, item)]
1391	logger.info('%s starts import from %s.csv' % (member,filename))
1392	attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1393	import_keys = [k for k in attrs if not k.startswith('ignore')]
1394	diff2schema = set(import_keys).difference(set(schema.keys()))
1395	diff2layout = set(import_keys).difference(set(layout.keys()))
1396	if diff2schema:
1397	em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1398	return em
1399	s = ','.join(['"%s"' % fn for fn in import_keys])
1400	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1401	s = '"id",' + s
1402	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1403	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1404	format_error = format + ',"%(Error)s"'
1405	format = '"%(id)s",'+ format
1406
1407	dm = DataModel(item, adapters,context=self)
1408	ds = DataStructure(data=item,datamodel=dm)
1409	error_string = ""
1410	for k in import_keys:
1411	#import pdb;pdb.set_trace()
1412	if not validators[k](ds,mode=mode):
1413	error_string += " %s : %s" % (k,
1414	self.translation_service(ds.getError(k),
1415	ds.getErrorMapping(k)))
1416	if not error_string:
1417	item.update(dm)
1418	item['id'],error = importer(item)
1419	if error:
1420	error_string += error
1421	if error_string:
1422	item['Error'] = error_string
1423	not_imported.append(format_error % item)
1424	total_not_imported += 1
1425	else:
1426	em = format % item
1427	imported.append(em)
1428	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1429	tr_count += 1
1430	total_imported += 1
1431	total += 1
1432	if total and not total % 100:
1433	transaction.commit()
1434	if len(not_imported) > 0:
1435	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1436	'\n'.join(not_imported) + '\n')
1437	not_imported = []
1438	if len(imported) > 0:
1439	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1440	'\n'.join(imported) + '\n')
1441	imported = []
1442	em = '%d transactions committed\n' % (tr_count)
1443	regs = []
1444	logger.info(em)
1445	tr_count = 0
1446	if len(imported) > 0:
1447	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1448	'\n'.join(imported))
1449	if len(not_imported) > 0:
1450	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1451	'\n'.join(not_imported))
1452	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1453	return em
1454	###)
1455
1456	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1457	def moveImagesToFS(self,student_id="O738726"):
1458	"move the images to the filesystem"
1459	images_dir = os.path.join("%s/images" % i_home,student_id)
1460	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1461	stool = getToolByName(self, 'portal_schemas')
1462	schemas = ['student_application',
1463	'student_clearance',
1464	]
1465	created = False
1466	for schema_id in schemas:
1467	schema = stool._getOb(schema_id)
1468	object = getattr(student_folder,schema_id[len('student_'):],None)
1469	if object is None:
1470	continue
1471	doc = object.getContent()
1472	for key in schema.keys():
1473	if schema[key].meta_type != "CPS Image Field":
1474	continue
1475	#import pdb;pdb.set_trace()
1476	image = getattr(doc,key)
1477	if not hasattr(image,"data"):
1478	continue
1479	if not created:
1480	if not os.path.exists(images_dir):
1481	os.mkdir(images_dir)
1482	created = True
1483	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1484	open(filename,"wb").write(str(image.data))
1485
1486	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: