Context navigation

source: WAeUP_SRP/base/WAeUPTool.py @ 2277

Last change on this file since 2277 was 2268, checked in by Henrik Bettermann, 17 years ago
allow overwriting of matric_no or jamb_reg_no if student id is provided
Property svn:keywords set to `Id`
File size: 55.5 KB

Line
1	#-- mode: python; mode: fold --
2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
3	# Author: Joachim Schmitz (js@aixtraware.de)
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License version 2 as published
7	# by the Free Software Foundation.
8	#
9	# This program is distributed in the hope that it will be useful,
10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	# GNU General Public License for more details.
13	#
14	# You should have received a copy of the GNU General Public License
15	# along with this program; if not, write to the Free Software
16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17	# 02111-1307, USA.
18	#
19	# $Id: WAeUPTool.py 2268 2007-09-21 20:11:08Z henrik $
20	"""The WAeUP Tool Box.
21	"""
22
23	from AccessControl import ClassSecurityInfo
24	from Acquisition import aq_inner
25	from Acquisition import aq_parent
26	from Globals import DTMLFile
27	from Globals import InitializeClass
28	from OFS.SimpleItem import SimpleItem
29
30	from Products.CMFCore.utils import getToolByName
31	from Products.CPSSchemas.DataStructure import DataStructure
32	from Products.CPSSchemas.DataModel import DataModel
33	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34	from Products.CMFCore.ActionProviderBase import ActionProviderBase
35	from Products.CMFCore.permissions import View
36	from Products.ZCatalog.ZCatalog import ZCatalog
37	from Products.CMFCore.permissions import ModifyPortalContent
38	from Products.CMFCore.permissions import ManagePortal
39	from Products.CMFCore.utils import UniqueObject
40	from Products.CMFCore.URLTool import URLTool
41	from Products.CMFCore.utils import getToolByName
42	from Students import makeCertificateCode
43	from Globals import package_home,INSTANCE_HOME
44	p_home = package_home(globals())
45	i_home = INSTANCE_HOME
46	import DateTime,time
47	import logging
48	import transaction
49	import csv,re,os,sys
50	from Products.AdvancedQuery import Eq, Between, Le,In
51
52	def getObject(object,name):
53	if object.hasObject(name):
54	return getattr(object,name)
55	return None
56
57	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58	"""WAeUP tool"""
59
60	id = 'waeup_tool'
61	meta_type = 'WAeUP Tool'
62	_actions = ()
63	security = ClassSecurityInfo()
64	security.declareObjectProtected(View)
65	manage_options = ( ActionProviderBase.manage_options
66	+ SimpleItem.manage_options
67	)
68
69	def rwrite(self,s): ###(
70	response = self.REQUEST.RESPONSE
71	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
72	response.write("%s<br />\r\n" % s)
73	###)
74
75	def sleep(self,secs): ###(
76	"sleep"
77	import time
78	time.sleep(secs)
79	return
80
81	###)
82
83	security.declareProtected(ModifyPortalContent,'openLog') ###(
84	def openLog(self,name):
85	"""open a log file"""
86	version = 1
87	path = "%s/log/%s_%d.log" % (i_home,name,version)
88	while os.path.exists(path):
89	version += 1
90	path = "%s/log/%s_%d.log" % (i_home,name,version)
91	log = open(path,"w")
92	return log
93	###)
94
95	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
96	def bypassQueueCatalog(self,enable=True):
97	"""bypass the QueueCatalog by setting all indexes to process imediate,
98	if enable is True (default) the old settings are restored
99	"""
100
101	###)
102
103	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
104	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
105	"""measure Object access Time"""
106	import random
107	if hasattr(self,'portal_catalog_real'):
108	aq_portal = self.portal_catalog_real.evalAdvancedQuery
109	else:
110	aq_portal = self.portal_catalog.evalAdvancedQuery
111	nr_pts = int(nr_pts)
112	probe = int(probe)
113	intervall = probe/10
114	objects = ("application","clearance","personal")
115	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
116	#i = random.randrange(num_objects)
117	count = 0
118	found = 0
119	not_found = 0
120	t_found = 0
121	t_not_found = 0
122	time_found = time_not_found = 0.0
123	t_time_found = t_time_not_found = 0.0
124	accessed = []
125	t_min = 1000
126	t_max = 0
127	#import pdb;pdb.set_trace()
128	students = self.portal_catalog(portal_type="Student")
129	num_students = len(students)
130	if method == "d":
131	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
132	res = aq_portal(query)
133	brains = {}
134	for r in res:
135	sid = r.relative_path.split('/')[-2]
136	if brains.has_key(sid):
137	brains[sid][r.portal_type] = r
138	else:
139	brains[sid] = {r.portal_type : r}
140	brains_list = brains.keys()
141	num_objects = len(brains_list)
142	else:
143	num_objects = num_students
144	print "="*40
145	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
146	probe,
147	nr_pts,
148	num_objects)
149	print "nr found/not time found/not min/max"
150	elapse = time.time()
151	i_elapse = time.time()
152	c_elapse = time.clock()
153	for c in range(1,probe + 1):
154	i = random.randrange(num_objects)
155	if method in ('a','b','c'):
156	student_brain = students[i]
157	elif method == "d":
158	#import pdb;pdb.set_trace()
159	student_brain = brains[brains_list[i]]
160	if method == "c":
161	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
162	res = aq_portal(query)
163	this_portal_types = [r.portal_type for r in res]
164	for i in range(nr_pts):
165	oid = objects[i]
166	if method == "a":
167	try:
168	student_path = student_brain.getPath()
169	path = "%s/%s" % (student_path,oid)
170	doc = self.unrestrictedTraverse(path).getContent()
171	found += 1
172	i_time = time.time() - i_elapse
173	time_found += i_time
174	except:
175	not_found += 1
176	i_time = time.time() - i_elapse
177	time_not_found += i_time
178	pass
179	elif method == "b":
180	try:
181	student_object = student_brain.getObject()
182	doc = getattr(student_object,oid).getContent()
183	found += 1
184	i_time = time.time() - i_elapse
185	time_found += i_time
186	except:
187	i_time = time.time() - i_elapse
188	time_not_found += i_time
189	not_found += 1
190	pass
191	elif method == "c":
192	if portal_types[i] in this_portal_types:
193	found += 1
194	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
195	i_time = time.time() - i_elapse
196	time_found += i_time
197	else:
198	not_found += 1
199	i_time = time.time() - i_elapse
200	time_not_found += i_time
201	elif method == "d":
202	if student_brain.has_key(portal_types[i]):
203	found += 1
204	doc = student_brain[portal_types[i]].getObject().getContent()
205	i_time = time.time() - i_elapse
206	time_found += i_time
207	else:
208	not_found += 1
209	i_time = time.time() - i_elapse
210	time_not_found += i_time
211	i_elapse = time.time()
212	if c and (c % intervall == 0):
213	#i_time = time.time() - i_elapse
214	t_per = 0.0
215	if found:
216	t_per = time_found/found
217	if t_per > t_max:
218	t_max = t_per
219	if t_per > 0.0 and t_per < t_min:
220	t_min = t_per
221	itf = 0.0
222	if found:
223	itf = time_found/found
224	itnf = 0.0
225	if not_found :
226	itnf = time_not_found / not_found
227	interval_time = time_found + time_not_found
228	s = "%(c)d: %(found)d/%(not_found)d " % vars()
229	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
230	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
231	print s
232	t_found += found
233	t_not_found += not_found
234	t_time_found += time_found
235	t_time_not_found += time_not_found
236	time_found = time_not_found = 0.0
237	found = not_found = 0
238	# t_found += found
239	# t_not_found += not_found
240	elapse = time.time() - elapse
241	itf = 0.0
242	if t_found:
243	itf = t_time_found/t_found
244	itnf = 0.0
245	if t_not_found:
246	itnf = t_time_not_found / t_not_found
247	#c_elapse = time.clock() - c_elapse
248	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
249	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
250	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
251	print "-"*40
252	print s
253	rel_found = float(t_found)/probe
254	rel_not_found = float(t_not_found)/probe
255	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
256	print estimated_total_time
257	###)
258
259	security.declareProtected(ModifyPortalContent,'writeLog') ###(
260	def writeLog(self,logfile,s):
261	"""write to the log file"""
262	logfile.write(s)
263
264	###)
265
266	def generateStudentId(self,letter): ###(
267	import random
268	r = random
269	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
270	if letter == '?':
271	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
272	sid = "%c%d" % (letter,r.randint(99999,1000000))
273	students = self.portal_url.getPortalObject().campus.students
274	## while hasattr(students, sid):
275	## sid = "%c%d" % (letter,r.randint(99999,1000000))
276	while self.students_catalog(id = sid):
277	sid = "%c%d" % (letter,r.randint(99999,1000000))
278	return sid
279	###)
280
281	def generatePassword(self,s=None): ###(
282	import random
283	r = random
284	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
285	if s is None:
286	s = 'abcdefghklmnpqrstuvwxy23456789'
287	pw = ''
288	while len(pw) < 6:
289	pw += r.choice(s)
290	return pw
291	###)
292
293	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
294	def removeDeletedDocIds(self, max=1000):
295	"""
296	remove deleted docids from repository commit after max
297	"""
298	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
299	repository = getToolByName(self, 'portal_repository')
300	pxtool = getToolByName(self, 'portal_proxies')
301	pxtool_infos = pxtool.getRevisionsUsed()
302
303	nb_revs = 0
304	docids_d = {} # all docids
305	unused_docids_d = {} # all docids that are unused
306	ids_unused_revs_docids = [] # ids for revs of unused docids
307	ids_unused_revs = [] # ids for unused revs
308	total = 0
309	idlist = repository.objectIds()
310	for id in idlist:
311	docid, rev = repository._splitId(id)
312	if docid is None:
313	logger.info("invalid doc_id %s" % docid)
314	continue
315	nb_revs += 1
316	docids_d[docid] = None
317	if not pxtool_infos.has_key(docid):
318	unused_docids_d[docid] = None
319	ids_unused_revs_docids.append(id)
320	ids_unused_revs.append(id)
321	elif not pxtool_infos[docid].has_key(rev):
322	ids_unused_revs.append(id)
323	if len(ids_unused_revs) >= max:
324	repository.manage_delObjects(ids_unused_revs)
325	#import pdb;pdb.set_trace()
326	transaction.commit()
327	total += max
328	logger.info('removed %d total %d unused docids ' % (max,total))
329	anz = len(ids_unused_revs)
330	if anz > 0:
331	repository.manage_delObjects(ids_unused_revs)
332	transaction.commit()
333	total += anz
334	logger.info('finished removing %d unused docids ' % (total))
335
336
337	###)
338
339	security.declareProtected(ModifyPortalContent,'getCredential') ###(
340	def getCredential(self,student_id):
341	"return a student password"
342	student_entry = getattr(self.portal_directories.students,student_id,None)
343	if student_entry is None:
344	return None
345	return getattr(student_entry,"password","not set")
346	###)
347
348	security.declarePublic('checkPassword') ###(
349	def checkPassword(self,student_id,password):
350	"return a student password"
351	student_entry = getattr(self.portal_directories.students,student_id,None)
352	if student_entry is None:
353	return False
354	return getattr(student_entry,"password","not set") == password
355	###)
356
357	security.declarePublic('editPassword') ###(
358	def editPassword(self,student_id,password):
359	"edit a student password"
360	student_entry = getattr(self.portal_directories.students,student_id,None)
361	if student_entry is None:
362	return
363	setattr(student_entry,'password',password)
364	###)
365
366	security.declareProtected(View,'doCommit') ###(
367	def doCommit(self,logger=None):
368	"commit some transactions"
369	transaction.commit()
370	###)
371
372	security.declarePublic('loadStudentFoto') ###(
373	def loadStudentFoto(self,student,filename,folder):
374	"return a student passport picture"
375	app = student.application
376	app_doc = app.getContent()
377	#clear = student.clearance
378	#clear_doc = clear.getContent()
379	#matric_no = clear_doc.matric_no.upper()
380	picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
381	picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
382	#import pdb;pdb.set_trace()
383	if os.path.exists(picture1):
384	file = open(picture1)
385	elif os.path.exists(picture2):
386	file = open(picture2)
387	else:
388	return "passport picture not found %s" % picture1
389	reopened = False
390	if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
391	self.portal_workflow.doActionFor(app,'open')
392	reopened = True
393	outfile = file.read()
394	app_doc.manage_addFile('passport',
395	file=outfile,
396	title="%s.jpg" % filename)
397	if reopened:
398	self.portal_workflow.doActionFor(app,'close')
399	return "successfully loaded passport picture"
400	###)
401
402	security.declareProtected(ModifyPortalContent,'createOne') ###(
403	def createOne(self,students_folder,student_brain,letter,commit=False):
404	sid = self.waeup_tool.generateStudentId(letter)
405	students_folder.invokeFactory('Student', sid)
406	student = getattr(students_folder,sid)
407	self.portal_workflow.doActionFor(student,'return')
408	student.manage_setLocalRoles(sid, ['Owner',])
409	matric_no = student_brain.matric_no
410	jamb_reg_no = student_brain.Entryregno
411	self.students_catalog.addRecord(id = sid,
412	matric_no = matric_no,
413	jamb_reg_no = jamb_reg_no,
414	sex = student_brain.Sex == "F",
415	name = "%s %s %s" % (student_brain.Firstname,
416	student_brain.Middlename,
417	student_brain.Lastname)
418	)
419	if commit:
420	transaction.commit()
421	return sid,jamb_reg_no
422	###)
423
424	security.declareProtected(ModifyPortalContent,'addStudent') ###(
425	def addStudent(self,dict):
426	students_folder = self.portal_url.getPortalObject().campus.students
427	sid = self.waeup_tool.generateStudentId('?')
428	students_folder.invokeFactory('Student', sid)
429	student_obj = getattr(students_folder,sid)
430	f2t = self.student_field2types
431	#from pdb import set_trace; set_trace()
432	d = {}
433	d['jamb_sex'] = 'M'
434	if dict.get('sex'):
435	d['jamb_sex'] = 'F'
436
437	entry_session = dict.get('entry_session')
438	if entry_session == self.getSessionId()[-2:]:
439	wfaction = 'admit'
440	wft = 'wf_transition_admit'
441	password = None
442	else:
443	wfaction = 'return'
444	wft = 'wf_transition_return'
445	password = self.generatePassword()
446	self.makeStudentMember(sid,password)
447
448	for pt in f2t.keys():
449	student_obj.invokeFactory(pt,f2t[pt]['id'])
450	sub_obj = getattr(student_obj,f2t[pt]['id'])
451	sub_doc = sub_obj.getContent()
452	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
453	d['Title'] = f2t[pt]['title']
454	for field in f2t[pt]['fields']:
455	d[field] = dict.get(field,'')
456	sub_doc.edit(mapping = d)
457	new_state = f2t[pt][wft]
458	if new_state != "remain":
459	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
460	self.portal_workflow.doActionFor(student_obj,wfaction)
461	student_obj.manage_setLocalRoles(sid, ['Owner',])
462	return sid,password
463	###)
464
465	security.declarePublic('getCertificateBrain') ###(
466	def getCertificateBrain(self,cert_id):
467	"do it"
468	res = ZCatalog.searchResults(self.portal_catalog_real,
469	{'portal_type':"Certificate",
470	'id': cert_id})
471	if res:
472	return res[0]
473	return None
474	###)
475
476	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
477	def get_csv_filenames(self):
478	"do it"
479	files = [file for file in os.listdir("%s/import/" % (i_home))
480	if file.endswith('.csv') and file.find('imported') == -1]
481	return files
482	###)
483
484	security.declarePublic('findStudentByMatricelNo') ###(
485	def findStudentByMatricelNo(self,matric_no):
486	"do it"
487	res = ZCatalog.searchResults(self.portal_catalog_real,
488	{'portal_type':"StudentClearance",
489	'SearchableText': matric_no})
490	if res:
491	return res[0]
492	return None
493	###)
494
495	security.declarePublic('makeStudentMember') ###(
496	def makeStudentMember(self,sid,password='uNsEt'):
497	"""make the student a member"""
498	membership = self.portal_membership
499	membership.addMember(sid,
500	password ,
501	roles=('Member',
502	'Student',
503	),
504	domains='',
505	properties = {'memberareaCreationFlag': False,
506	'homeless': True},)
507	member = membership.getMemberById(sid)
508	self.portal_registration.afterAdd(member, sid, password, None)
509	#self.manage_setLocalRoles(sid, ['Owner',])
510	###)
511
512	security.declarePublic('makeStudentData') ###(
513	def makeStudentData(self,student_id,email=None,phone_nr=None):
514	"create Datastructure for a returning Student"
515	#import pdb;pdb.set_trace()
516	logger = logging.getLogger('WAeUPTool.makeStudentData')
517	students_folder = self.portal_url.getPortalObject().campus.students
518	#res = self.students_catalog(id=student_id)
519	#if res:
520	# st = res[0]
521	#res = self.returning_import(matric_no = st.matric_no)
522	res = self.returning_import(id = student_id)
523	if res:
524	student = res[0]
525	else:
526	logger.info('Id %s not found in returning_import' % student_id)
527	return
528	logger.info('%s creates data structure' % student_id)
529	s_results = self.results_import(matric_no = student.matric_no)
530	if s_results:
531	lnr = self.getLevelFromResultsCosCode(s_results)
532	level = "%d00" % lnr
533	verdict,eligible = self.getVerdict(s_results[0].Verdict)
534	if eligible:
535	level = "%d00" % (lnr + 1)
536	else:
537	logger.info('matric_no %s not found in results_import' % student.matric_no)
538	level = 0
539	verdict = 'N/A'
540	#student should not be allowed to perform this transition
541	#wftool = self.portal_workflow
542	#wftool.doActionFor(student,'return')
543	certcode_org = student.Coursemajorcode
544	certcode = makeCertificateCode(certcode_org)
545	certificate_brain = self.getCertificateBrain(certcode)
546	if not certificate_brain:
547	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
548	logger.info(em)
549	matric_no = student.matric_no
550	sid = student_id
551	student_obj = getattr(students_folder,sid)
552	student_obj.invokeFactory('StudentApplication','application')
553	application = student_obj.application
554	self.portal_workflow.doActionFor(application,'open',dest_container=application)
555	da = {'Title': 'Application Data'}
556	student_obj.invokeFactory('StudentPersonal','personal')
557	da['jamb_reg_no'] = student.Entryregno
558	em = self.getEntryMode(student.Entryregno)
559	## em = student.Mode_of_Entry
560	## if em in ('DIRECT', 'DIRECT ENTRY',):
561	## em = 'DE'
562	## elif em in ('U.M.E', 'UNE',):
563	## em = 'UME'
564	## elif not em:
565	## em = "unknown"
566	da['entry_mode'] = em
567	personal = student_obj.personal
568	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
569	dp = {'Title': 'Personal Data'}
570	student_obj.invokeFactory('StudentClearance','clearance')
571	clearance = student_obj.clearance
572	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
573	dc = {'Title': 'Clearance/Eligibility Record'}
574	dc['matric_no'] = matric_no
575	state = student.State
576	lga = student.LGA
577	if state and lga:
578	lga = state + ' / ' + lga
579	else:
580	lga = "None"
581	da['jamb_lga'] = dc['lga'] = lga
582	da['app_email'] = dp['email'] = email
583	da['app_mobile'] = dp['phone'] = phone_nr
584	dp['firstname'] = student.Firstname
585	dp['middlename'] = student.Middlename
586	dp['lastname'] = student.Lastname
587	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
588	da['jamb_sex'] = student.Sex
589	dp['sex'] = student.Sex == 'F'
590	dp['perm_address'] = student.Permanent_Address
591	application.getContent().edit(mapping=da)
592	self.portal_workflow.doActionFor(application,'close',dest_container=application)
593	personal.getContent().edit(mapping=dp)
594	clearance.getContent().edit(mapping=dc)
595	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
596	## catd = {}
597	## catd['id'] = sid
598	## catd['entry_mode']= da['entry_mode']
599	## catd['matric_no'] = matric_no
600	## catd['jamb_reg_no'] = da['jamb_reg_no']
601	## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
602	## catd['sex'] = dp['sex']
603	## catd['level'] = level
604	## catd['verdict'] = verdict
605	## if certificate_brain:
606	## cpath = certificate_brain.getPath().split('/')
607	## catd['faculty'] = cpath[-4]
608	## catd['department'] = cpath[-3]
609	## catd['course'] = certcode
610	## self.students_catalog.modifyRecord(**catd)
611	#
612	# Study Course
613	#
614	student_obj.invokeFactory('StudentStudyCourse','study_course')
615	studycourse = student_obj.study_course
616	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
617	dsc = {}
618	dsc['study_course'] = certcode
619	dsc['current_level'] = level
620	dsc['current_verdict'] = verdict
621	dsc['current_mode'] = em
622	dsc['current_session'] = '06'
623	studycourse.getContent().edit(mapping=dsc)
624	#
625	# Level
626	#
627	## l = getattr(studycourse,level,None)
628	## if l is None:
629	## studycourse.invokeFactory('StudentStudyLevel', level)
630	## l = getattr(studycourse, level)
631	## self.portal_workflow.doActionFor(l,'open',dest_container=l)
632	## l.getContent().edit(mapping={'Title': "Level %s" % level})
633	###)
634
635	security.declarePublic('makeStudentLevel') ###(
636	def makeStudentLevel(self,student_id):
637	"create the StudyLevel for a returning Student"
638	#import pdb;pdb.set_trace()
639	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
640	students_folder = self.portal_url.getPortalObject().campus.students
641	res = self.students_catalog(id=student_id)
642	if res:
643	st = res[0]
644	course = st.course
645	matric_no = st.matric_no
646	level = st.level
647	res = self.results_import(matric_no = matric_no)
648	if res:
649	results = res
650	logger.info('%s creating Level %s' % (student_id,level))
651	#
652	# Level
653	#
654	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
655	studycourse = getattr(student_obj,"study_course",None)
656	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
657	l = getattr(studycourse,level,None)
658	if l is None:
659	studycourse.invokeFactory('StudentStudyLevel', level)
660	l = getattr(studycourse, level)
661	self.portal_workflow.doActionFor(l,'open',dest_container=l)
662	l.getContent().edit(mapping={'Title': "Level %s" % level})
663	###)
664
665	security.declarePublic('getHallInfo') ###(
666	def getHallInfo(self,bed):
667	"""return Hall Info"""
668	info = {}
669	hall,block,room,letter = bed.split('_')
670	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
671	if res and len(res) == 1:
672	hall_brain = res[0]
673	hall_doc = hall_brain.getObject().getContent()
674	else:
675	return info
676	info['hall_title'] = hall_brain.Title
677	info['maintenance_code'] = hall_doc.maintenance_code
678	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
679	batch_doc = None
680	for brain in res:
681	if brain.id.startswith(info['maintenance_code']):
682	batch_doc = brain.getObject().getContent()
683	break
684	if batch_doc is None:
685	info['maintenance_fee'] = None
686	else:
687	info['maintenance_fee'] = batch_doc.cost
688	return info
689	###)
690
691	security.declarePublic('showFsPicture') ###(
692	def showFsPicture(self,path):
693	"""return a picture from the filesystem"""
694	picture_path = "%s/import/%s" % (i_home,path)
695	if os.path.exists(picture_path):
696	return open(picture_path).read()
697
698	###)
699
700	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
701	def deleteAllCourses(self,department="All"):
702	''' delete the courses'''
703	pm = self.portal_membership
704	member = pm.getAuthenticatedMember()
705
706	if str(member) not in ("henrik","joachim"):
707	return "not possible"
708	if department == "All":
709	res = self.portal_catalog({'meta_type': 'Department'})
710	if len(res) < 1:
711	return "No Departments found"
712
713	deleted = []
714	for dep in res:
715	cf = dep.getObject().courses
716	if cf:
717	cf.manage_delObjects(ids=cf.objectIds())
718	deleted.append("deleted Courses in %s" % dep.getId)
719	return "\r".join(deleted)
720	###)
721
722	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
723	def getLogfileLines(self,filename="event.log",numlines=20):
724	"""Get last NUMLINES lines of logfile FILENAME.
725
726	Return last lines' of a file in the instances logfile directory as
727	a list. The number of returned lines equals `numlines' or less. If
728	less than `numlines' lines are available, the whole file ist
729	returned. If the file can not be opened or some other error
730	occurs, empty list is returend.
731	"""
732	result = []
733	lines_hit = 0
734
735	# We only handle files in instances' log directory...
736	logpath = os.path.join(i_home, "log")
737	filename = str(os.path.abspath( os.path.join( logpath, filename )))
738	if not filename.startswith( logpath ):
739	# Attempt to access file outside log-dir...
740	return []
741
742	try:
743	fd = file( filename, "rb" )
744	except IOError:
745	return []
746	if not fd:
747	return []
748
749	if os.linesep == None:
750	linesep = '\n'
751	else:
752	linesep = os.linesep
753
754	# Try to find 'numlines' times a lineseparator, searching from end
755	# and moving to the beginning of file...
756	fd.seek( 0, 2) # Move to end of file...
757	while lines_hit < numlines:
758	if fd.read(1) == linesep[-1]: # This moves filedescriptor
759	# one step forward...
760	lines_hit += 1
761	try:
762	fd.seek( -2, 1) # Go two bytes back from current pos...
763	except IOError:
764	# We cannot go back two bytes. Maybe the file is too small...
765	break
766	fd.seek(2,1)
767
768	# Read all lines from current position...
769	result = fd.readlines()
770	# Remove line endings...
771	result = [x.strip() for x in result]
772	fd.close()
773	return result
774	###)
775
776	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
777	def getCallbacksFromLog(self,filename):
778	"""fix Online Payment Transactions from Z2.log entries"""
779	import transaction
780	import random
781	from cgi import parse_qs
782	from urlparse import urlparse
783	#from pdb import set_trace
784	wftool = self.portal_workflow
785	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
786	students_folder = self.portal_url.getPortalObject().campus.students
787	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
788	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
789	s += r'(?P<intersw>.)" "(?P<agent>.)"'
790	data = re.compile(s)
791	start = True
792	tr_count = 1
793	total = 0
794	#name = 'pume_results'
795	#name = 'epaymentsuccessful_z2log2'
796	name = filename
797	no_import = []
798	imported = []
799	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
800	try:
801	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
802	except:
803	logger.error('Error reading %s' % name)
804	return
805	tas = []
806	for line in transactions:
807	dict = {}
808	items = data.search(line)
809	dict['idict'] = idict = items.groupdict()
810	#print idict
811	#from pdb import set_trace;set_trace()
812	urlparsed = urlparse(idict['get'][4:])
813	#print urlparsed
814	path = urlparsed[2].split('/')
815	dict['student_id'] = student_id = path[8]
816	dict['payment_id'] = payment_id = path[10]
817	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
818	tas.append(dict)
819	tr_count += 1
820	return tas
821	###)
822
823	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
824	def importOnlinePaymentTransactions(self):
825	"""load Online Payment Transactions from CSV values"""
826	import transaction
827	import random
828	#from pdb import set_trace
829	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
830	opt = self.online_payments_import
831	students_folder = self.portal_url.getPortalObject().campus.students
832	start = True
833	tr_count = 1
834	total = 0
835	#name = 'pume_results'
836	name = 'OnlineTransactions'
837	no_import = []
838	imported = []
839	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
840	try:
841	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
842	except:
843	logger.error('Error reading %s.csv' % name)
844	return
845	for pay_transaction in transactions:
846	if start:
847	start = False
848	logger.info('Start loading from %s.csv' % name)
849	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
850	no_import.append('%s,"Error"' % s)
851	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
852	format_error = format + ',"%(Error)s"'
853	data = {}
854
855	# format of the first file sent by Tayo
856	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
857	#data['student_id'] = student_id = pay_transaction['Payer ID']
858	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
859	#data['response_code'] = response_code = pay_transaction['Resp Code']
860	#data['amount'] = amount = pay_transaction['Amount']
861
862	# format of the second file sent by Tayo
863	#data['datetime'] = date = 0
864	#data['student_id'] = student_id = pay_transaction['Payer ID']
865	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
866	#data['response_code'] = response_code = '00'
867	#data['amount'] = amount = pay_transaction['Amount']
868
869	# format of the third file sent by Kehinde
870	data['datetime'] = date = 0
871	data['student_id'] = student_id = pay_transaction['customer_id']
872	data['order_id'] = order_id = pay_transaction['merchant_reference']
873	data['response_code'] = response_code = '00'
874	data['amount'] = amount = pay_transaction['Amount']
875
876	dup = False
877	if response_code == "12":
878	continue
879	try:
880	opt.addRecord(**data)
881	except ValueError:
882	dup = True
883	#from pdb import set_trace;set_trace()
884	if dup:
885	if response_code == "00":
886	try:
887	opt.modifyRecord(**data)
888	except:
889	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
890	continue
891	else:
892	pay_transaction['Error'] = "Duplicate order_id"
893	no_import.append( format_error % pay_transaction)
894	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
895	continue
896	tr_count += 1
897	if tr_count > 1000:
898	if len(no_import) > 0:
899	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
900	'\n'.join(no_import) + '\n')
901	no_import = []
902	em = '%d transactions committed\n' % (tr_count)
903	transaction.commit()
904	regs = []
905	logger.info(em)
906	total += tr_count
907	tr_count = 0
908	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
909	'\n'.join(no_import))
910	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
911	###)
912
913	security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
914	def mass_create_faculty(self,mapping):
915	"create a faculty"
916	logger = logging.getLogger('WAeUPTool.mass_create_faculty')
917	academics_folder = self.portal_url.getPortalObject().campus.academics
918	fid = mapping['code']
919	if getattr(academics_folder,fid,None) is not None:
920	return '', "Faculty with ID: %s exists" % fid
921	logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
922	academics_folder.invokeFactory('Faculty', fid)
923	f = getattr(academics_folder,fid,None)
924	f.getContent().edit(mapping=mapping)
925	return fid,''
926	###)
927
928	security.declareProtected(ModifyPortalContent,'mass_edit_Faculty') ###(
929	def mass_edit_faculty(self,mapping):
930	"edit a faculty"
931	logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
932	academics_folder = self.portal_url.getPortalObject().campus.academics
933	fid = mapping['code']
934	f = getattr(academics_folder,fid,None)
935	if f is None:
936	return '', "Faculty with ID: %s does not exist" % fid
937	logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
938	f.getContent().edit(mapping=mapping)
939	return fid,''
940	###)
941
942	security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
943	def mass_create_department(self,mapping):
944	"create a department in the correct faculty"
945	logger = logging.getLogger('WAeUPTool.mass_create_department')
946	fid = mapping['faculty_code']
947	if getattr(self,'_v_faculties',None) is None:
948	res = self.portal_catalog(portal_type = "Faculty")
949	self._v_faculties = {}
950	for f in res:
951	self._v_faculties[f.getId] = f.getObject()
952	f = self._v_faculties.get(fid,None)
953	if f is None:
954	return '', "No Faculty with ID: %s" % fid
955	else:
956	did = mapping.get('code')
957	d = getattr(f,did,None)
958	if d is None or d.portal_type == "Faculty":
959	logger.info('Creating Department %(code)s, %(title)s' % mapping)
960	f.invokeFactory('Department', did)
961	d = getattr(f,did)
962	d.invokeFactory('CoursesFolder','courses')
963	courses = getattr(d,'courses')
964	dict = {'Title': 'Courses'}
965	courses.getContent().edit(mapping=dict)
966	d.invokeFactory('CertificatesFolder','certificates')
967	certificates = getattr(d,'certificates')
968	dict = {'Title': 'Certificates'}
969	certificates.getContent().edit(mapping=dict)
970	d.getContent().edit(mapping=mapping)
971	return did,''
972	###)
973
974	security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
975	def mass_edit_department(self,mapping):
976	"create a department in the correct faculty"
977	logger = logging.getLogger('WAeUPTool.mass_create_department')
978	academics_folder = self.portal_url.getPortalObject().campus.academics
979	fid = mapping['faculty_code']
980	did = mapping.get('code')
981	try:
982	d = getattr(getattr(academics_folder,fid),did,None)
983	except KeyError:
984	return '', "Department %s or Faculty %s wrong" % (did,fid)
985	else:
986	if d is None or d.portal_type == "Faculty":
987	logger.info('Editing Department %(code)s, %(title)s' % mapping)
988	d.getContent().edit(mapping=mapping)
989	return did,''
990	###)
991
992	security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
993	def mass_create_course(self,mapping):
994	if getattr(self,'_v_course_list',None) is None:
995	self._v_course_list = []
996	if getattr(self,'_v_departments',None) is None:
997	res = self.portal_catalog(portal_type = "Department")
998	self._v_departments = {}
999	for d in res:
1000	self._v_departments[d.getId] = d.getObject()
1001	did = mapping['department_code']
1002	d = self._v_departments.get(did,None)
1003	if d is None:
1004	return '', "No Department with ID: %s" % fid
1005	course_id = mapping.get('code')
1006	if course_id in self._v_course_list:
1007	return '', "Duplicate Department ID: %s" % did
1008	c = getattr(d,course_id,None)
1009	if c is not None:
1010	return '', "Duplicate Department ID: %s" % did
1011	d.invokeFactory('Course', course_id)
1012	self._v_course_list.append(course_id)
1013	c = getattr(d,course_id)
1014	c.getContent().edit(mapping=mapping)
1015	return course_id,''
1016	###)
1017
1018	field2types_student = { ###(
1019	'StudentApplication':
1020	{'id': 'application',
1021	'title': 'Application Data',
1022	'wf_transition_return': 'close',
1023	'wf_transition_admit': 'remain',
1024	'fields':
1025	('jamb_reg_no',
1026	'entry_mode',
1027	'entry_session',
1028	'jamb_score',
1029	'app_email',
1030	'jamb_age',
1031	'jamb_state',
1032	'jamb_lga',
1033	)
1034	},
1035	#'StudentPume':
1036	# {'id': 'pume',
1037	# 'title': 'Pume Data',
1038	# 'wf_transition_return': 'close',
1039	# 'wf_transition_admit': 'close',
1040	# 'fields':
1041	# ('pume_score',
1042	# )
1043	# },
1044	'StudentClearance':
1045	{'id': 'clearance',
1046	'title': 'Clearance/Eligibility Record',
1047	'wf_transition_return': 'close',
1048	'wf_transition_admit': 'remain',
1049	'fields':
1050	('matric_no',
1051	'nationality',
1052	'lga',
1053	'birthday',
1054	)
1055	},
1056	'StudentPersonal':
1057	{'id': 'personal',
1058	'title': 'Personal Data',
1059	'wf_transition_return': 'open',
1060	'wf_transition_admit': 'remain',
1061	'fields':
1062	('firstname',
1063	'middlename',
1064	'lastname',
1065	'sex',
1066	'email',
1067	'phone',
1068	'perm_address',
1069	)
1070	},
1071	'StudentStudyCourse':
1072	{'id': 'study_course',
1073	'title': 'Study Course',
1074	'wf_transition_return': 'open',
1075	'wf_transition_admit': 'remain',
1076	'fields':
1077	('study_course',
1078	'current_level',
1079	'current_session',
1080	'current_mode',
1081	'current_verdict',
1082	)
1083	},
1084
1085	'PaymentsFolder':
1086	{'id': 'payments',
1087	'title': 'Payments',
1088	'wf_transition_return': 'open',
1089	'wf_transition_admit': 'open',
1090	'fields':
1091	()
1092	},
1093	}
1094	###)
1095
1096	security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1097	def mass_create_student(self,mapping):
1098	"create a students record due import"
1099	logger = logging.getLogger('WAeUPTool.mass_create_student')
1100	students_folder = self.portal_url.getPortalObject().campus.students
1101	jamb_reg_no = mapping.get('jamb_reg_no',None)
1102	if jamb_reg_no:
1103	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1104	if res:
1105	return '',"jamb_reg_no exists"
1106	matric_no = mapping.get('matric_no',None)
1107	if matric_no:
1108	res = self.students_catalog(matric_no = matric_no)
1109	if res:
1110	return '',"matric_no exists"
1111	sid = self.waeup_tool.generateStudentId('?')
1112	students_folder.invokeFactory('Student', sid)
1113	student_obj = getattr(students_folder,sid)
1114	f2t = self.field2types_student
1115	d = {}
1116	d['jamb_sex'] = 'M'
1117	if mapping.get('sex'):
1118	d['jamb_sex'] = 'F'
1119	transition = mapping.get('reg_transition','admit')
1120	if transition not in ('admit','return'):
1121	return '',"no valid transition provided"
1122	for pt in f2t.keys():
1123	student_obj.invokeFactory(pt,f2t[pt]['id'])
1124	sub_obj = getattr(student_obj,f2t[pt]['id'])
1125	sub_doc = sub_obj.getContent()
1126	d['Title'] = f2t[pt]['title']
1127	for field in f2t[pt]['fields']:
1128	d[field] = mapping.get(field,'')
1129	if pt == "StudyCourse":
1130	for von,zu in (('entry_mode','current_mode'),
1131	('entry_session','current_session')):
1132	if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1133	d[zu] = mapping[von]
1134	sub_doc.edit(mapping = d)
1135
1136	#import pdb;pdb.set_trace()
1137	new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1138	if new_state != "remain":
1139	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1140	self.portal_workflow.doActionFor(student_obj,transition)
1141	student_obj.manage_setLocalRoles(sid, ['Owner',])
1142	return sid,''
1143	###)
1144
1145	security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1146	def mass_edit_student(self,mapping):
1147	"edit a students record due import"
1148	logger = logging.getLogger('WAeUPTool.mass_edit_student')
1149	students_folder = self.portal_url.getPortalObject().campus.students
1150	sid = mapping.get('id',None)
1151	jamb_reg_no = mapping.get('jamb_reg_no',None)
1152	matric_no = mapping.get('matric_no',None)
1153	editable_keys = mapping.keys()
1154	if sid:
1155	res = self.students_catalog(id = sid)
1156	if not res:
1157	return '',"no student with id %s" % sid
1158	if matric_no and res[0].matric_no and\
1159	matric_no != res[0].matric_no:
1160	logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1161	if jamb_reg_no and res[0].jamb_reg_no and\
1162	jamb_reg_no != res[0].jamb_reg_no:
1163	logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1164	elif jamb_reg_no:
1165	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1166	if not res:
1167	return '',"no student with jamb_reg_no %s" % jamb_reg_no
1168	editable_keys.remove('jamb_reg_no')
1169	elif matric_no:
1170	res = self.students_catalog(matric_no = matric_no)
1171	if not res:
1172	return '',"no student with matric_no %s" % matric_no
1173	editable_keys.remove('matric_no')
1174
1175	## included only to change wf state from admitted to returning
1176	#if res[0].review_state not in ('admitted','objection_raised'):
1177	# return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1178	## end inclusion
1179
1180	sid = res[0].id
1181	student_obj = getattr(students_folder,sid)
1182	f2t = self.field2types_student
1183	d = {}
1184	#import pdb;pdb.set_trace()
1185	any_change = False
1186	for pt in f2t.keys():
1187	if pt == "student_application":
1188	d['jamb_sex'] = 'M'
1189	if mapping.get('sex'):
1190	d['jamb_sex'] = 'F'
1191	intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1192	if intersect:
1193	sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1194	if sub_obj is None:
1195	try:
1196	student_obj.invokeFactory(pt,f2t[pt]['id'])
1197	except:
1198	continue
1199	sub_obj = getattr(student_obj,f2t[pt]['id'])
1200	d['Title'] = f2t[pt]['title']
1201	sub_doc = sub_obj.getContent()
1202	for field in intersect:
1203	changed = False
1204	if getattr(sub_doc,field) != mapping.get(field,''):
1205	any_change = True
1206	changed = True
1207	d[field] = mapping.get(field,'')
1208	if changed:
1209	sub_doc.edit(mapping = d)
1210
1211
1212	## included only to change wf state from admitted to returning
1213	# if res[0].review_state in ('admitted','objection_raised'):
1214	# new_state = f2t[pt]['wf_transition_return']
1215	# sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1216	# if sub_obj and new_state != "remain":
1217	# try:
1218	# self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1219	# except:
1220	# #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1221	# pass
1222	#if res[0].review_state in ('admitted','objection_raised'):
1223	# wfaction = 'return'
1224	# try:
1225	# self.portal_workflow.doActionFor(student_obj,wfaction)
1226	# logger.info('%s, wf state changed' % sid)
1227	# any_change = True
1228	# except:
1229	# logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1230	# pass
1231	## end inclusion
1232
1233
1234	if any_change:
1235	return sid,''
1236	else:
1237	return sid,'not modified'
1238	###)
1239
1240	security.declareProtected(ModifyPortalContent,"importData")###(
1241	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1242	"""load data from CSV values"""
1243	import transaction
1244	import random
1245
1246	pm = self.portal_membership
1247	member = pm.getAuthenticatedMember()
1248
1249	logger = logging.getLogger('WAeUPTool.importData')
1250	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1251	students_folder = self.portal_url.getPortalObject().campus.students
1252	start = True
1253	tr_count = 1
1254	total_imported = 0
1255	total_not_imported = 0
1256	total = 0
1257	iname = "import_%s" % name
1258	stool = getToolByName(self, 'portal_schemas')
1259	ltool = getToolByName(self, 'portal_layouts')
1260	schema = stool._getOb(iname)
1261	if schema is None:
1262	em = 'No such schema %s' % iname
1263	logger.error('No such schema %s' % iname)
1264	return em
1265	layout = ltool._getOb(iname)
1266	if layout is None:
1267	em = 'No such layout %s' % iname
1268	logger.error(em)
1269	return em
1270	validators = {}
1271	for widget in layout.keys():
1272	validators[widget] = layout[widget].validate
1273	#import pdb;pdb.set_trace()
1274	if edit:
1275	importer_name = "mass_edit_%s" % name
1276	else:
1277	importer_name = "mass_create_%s" % name
1278	importer = getattr(self, '%s' % importer_name,None)
1279	if importer is None:
1280	em = 'No importer function %s' % importer_name
1281	logger.error(em)
1282	return em
1283	not_imported = []
1284	imported = []
1285	try:
1286	items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1287	except:
1288	em = 'Error reading %s.csv' % filename
1289	logger.error(em)
1290	return em
1291	for item in items:
1292	if start:
1293	start = False
1294	adapters = [MappingStorageAdapter(schema, item)]
1295	logger.info('%s starts import from %s.csv' % (member,filename))
1296	#import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1297	attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1298	import_keys = [k for k in attrs if not k.startswith('ignore')]
1299	diff2schema = set(import_keys).difference(set(schema.keys()))
1300	diff2layout = set(import_keys).difference(set(layout.keys()))
1301	if diff2schema:
1302	em = "not ignorable key(s) %s found in heading" % diff2schema
1303	return em
1304	s = ','.join(['"%s"' % fn for fn in import_keys])
1305	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1306	s = '"id",' + s
1307	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1308	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1309	format_error = format + ',"%(Error)s"'
1310	format = '"%(id)s",'+ format
1311
1312	dm = DataModel(item, adapters,context=self)
1313	ds = DataStructure(data=item,datamodel=dm)
1314	error_string = ""
1315	for k in import_keys:
1316	#import pdb;pdb.set_trace()
1317	if not validators[k](ds):
1318	error_string += " %s : %s" % (k,
1319	self.translation_service(ds.getError(k),
1320	ds.getErrorMapping(k)))
1321	if not error_string:
1322	item.update(dm)
1323	item['id'],error = importer(item)
1324	if error:
1325	error_string += error
1326	if error_string:
1327	item['Error'] = error_string
1328	not_imported.append(format_error % item)
1329	total_not_imported += 1
1330	else:
1331	em = format % item
1332	imported.append(em)
1333	logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1334	tr_count += 1
1335	total_imported += 1
1336	total += 1
1337	if total and not total % 100:
1338	transaction.commit()
1339	if len(not_imported) > 0:
1340	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1341	'\n'.join(not_imported) + '\n')
1342	not_imported = []
1343	if len(imported) > 0:
1344	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1345	'\n'.join(imported) + '\n')
1346	imported = []
1347	em = '%d transactions committed\n' % (tr_count)
1348	regs = []
1349	logger.info(em)
1350	tr_count = 0
1351	if len(imported) > 0:
1352	open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1353	'\n'.join(imported))
1354	if len(not_imported) > 0:
1355	open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1356	'\n'.join(not_imported))
1357	em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1358	return em
1359	###)
1360
1361	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: