Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 4784

Last change on this file since 4784 was 4558, checked in by Henrik Bettermann, 15 years ago
restore the final accommodation object when reimporting the data into the alumni portal
Property svn:keywords set to `Id`
File size: 77.6 KB

Rev	Line
[3219]	1	# -- mode: python; mode: fold; --
[197]	2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
	3	# Author: Joachim Schmitz (js@aixtraware.de)
	4	#
	5	# This program is free software; you can redistribute it and/or modify
	6	# it under the terms of the GNU General Public License version 2 as published
	7	# by the Free Software Foundation.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program; if not, write to the Free Software
	16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
	17	# 02111-1307, USA.
	18	#
	19	# $Id: WAeUPTool.py 4558 2009-12-16 09:56:53Z henrik $
[1174]	20	"""The WAeUP Tool Box.
[197]	21	"""
	22
	23	from AccessControl import ClassSecurityInfo
[828]	24	from Acquisition import aq_inner
	25	from Acquisition import aq_parent
	26	from Globals import DTMLFile
	27	from Globals import InitializeClass
	28	from OFS.SimpleItem import SimpleItem
[2283]	29	from zExceptions import BadRequest
[197]	30
[1890]	31	from Products.CMFCore.utils import getToolByName
[1747]	32	from Products.CPSSchemas.DataStructure import DataStructure
	33	from Products.CPSSchemas.DataModel import DataModel
	34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
[828]	35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
	36	from Products.CMFCore.permissions import View
	37	from Products.ZCatalog.ZCatalog import ZCatalog
	38	from Products.CMFCore.permissions import ModifyPortalContent
[1890]	39	from Products.CMFCore.permissions import ManagePortal
[197]	40	from Products.CMFCore.utils import UniqueObject
[1194]	41	from Products.CMFCore.URLTool import URLTool
[1747]	42	from Products.CMFCore.utils import getToolByName
[1151]	43	from Students import makeCertificateCode
[1285]	44	from Globals import package_home,INSTANCE_HOME
[3172]	45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
[3988]	46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport,PaymentImport
[3172]	47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
	48	from utils import makeDigest
[2094]	49	import DateTime,time
[1620]	50	import logging
[1170]	51	import transaction
[2193]	52	import csv,re,os,sys
[3172]	53	import md5
[3292]	54	from shutil import copy2,copy
[1707]	55	from Products.AdvancedQuery import Eq, Between, Le,In
[197]	56
[2413]	57	p_home = package_home(globals())
	58	i_home = INSTANCE_HOME
	59	images_base = os.path.join(i_home,"images")
[3198]	60	EMPTY = 'XXX'
[2413]	61
[2675]	62	def getImagesDir(student_id):
	63	return os.path.join("%s" % images_base,student_id[0],student_id)
[2413]	64
[1707]	65	def getObject(object,name):
	66	if object.hasObject(name):
	67	return getattr(object,name)
	68	return None
[1720]	69
[828]	70	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
[197]	71	"""WAeUP tool"""
	72
[828]	73	id = 'waeup_tool'
[197]	74	meta_type = 'WAeUP Tool'
[828]	75	_actions = ()
[197]	76	security = ClassSecurityInfo()
[828]	77	security.declareObjectProtected(View)
	78	manage_options = ( ActionProviderBase.manage_options
	79	+ SimpleItem.manage_options
	80	)
	81
[2999]	82	security.declareProtected(View,'re_split') ###(
	83	def re_split(self,split_string,string):
	84	return re.split(split_string,string)
[3014]	85	###)
[2999]	86
	87	security.declareProtected(View,'difference') ###(
	88	def difference(self,l1,l2):
	89	return set(l1).difference(set(l2))
[3014]	90	###)
[2999]	91
[1818]	92	def rwrite(self,s): ###(
[1707]	93	response = self.REQUEST.RESPONSE
	94	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
	95	response.write("%s<br />\r\n" % s)
[1818]	96	###)
[1174]	97
[2695]	98	def addtodict(self,d,key,item): ###(
	99	d[key].append(item)
	100	return d[key]
	101	###)
	102
[1818]	103	def sleep(self,secs): ###(
	104	"sleep"
	105	import time
	106	time.sleep(secs)
	107	return
[3473]	108	###)
[1827]	109
[2695]	110	security.declareProtected(View,'updateRoleMappingsFor') ###(
	111	def updateRoleMappingsFor(self,wf_definition,ob):
	112	"do so for public"
	113	wf_def = getattr(self.portal_workflow,wf_definition)
	114	wf_def.updateRoleMappingsFor(ob)
	115	###)
[2699]	116
[2999]	117	security.declareProtected(View,'getStatesLgas') ###(
	118	def getStatesLgas(self):
	119	"""return lga info"""
	120	voc = getattr(self.portal_vocabularies,'local_gov_areas')
	121	states = []
	122	lgas = []
	123	d = {}
	124	wd = {}
	125	for k,v in voc.items():
	126	parts = v.split(' / ')
	127	if len(parts) == 1:
	128	state = parts[0].lower()
	129	lga = ""
	130	elif len(parts) == 2:
	131	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
	132	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
	133	else:
	134	continue
	135	if state not in states:
	136	states.append(state)
	137	if lga not in lgas:
	138	lgas.append(lga)
	139	words = re.split('[^a-zA-Z0-9/]',k)
	140	words.sort()
	141	wd[k] = words
	142	d[k] = v
	143	mapping = {}
	144	mapping['word_dict'] = wd
	145	mapping['lga_dict'] = d
	146	mapping['states'] = states
	147	mapping['lgas'] = lgas
	148	return mapping
	149	###)
	150
[3014]	151	security.declareProtected(View,'findLga') ###(
	152	def findLga(self,words,words_dict):
	153	words = re.split('[^a-zA-Z0-9/]',words)
	154	lga_words = []
	155	for word in words:
	156	if word:
	157	lga_words += word.strip().lower(),
	158	lga_words.sort()
	159	state_lga = ''
	160	while not state_lga:
	161	for k,l in words_dict.items():
	162	if lga_words == l:
	163	state_lga = k
	164	break
	165	break
	166	return state_lga
	167	###)
[2975]	168	security.declareProtected(View,'getAccessInfo') ###(
	169	def getAccessInfo(self,context):
	170	"return a dict with access_info"
	171	logger = logging.getLogger('WAeUPTool.getAccessInfo')
	172	mtool = self.portal_membership
	173	member = mtool.getAuthenticatedMember()
	174	member_id = str(member)
	175	info = {}
	176	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
	177	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
	178	is_staff = info['is_staff'] = not is_anonymous and not is_student
	179	roles = member.getRolesInContext(context)
[3452]	180	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
	181	"SectionManager" in roles or
	182	"Manager" in roles)
	183	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
[3858]	184	#is_allowed = info['is_allowed'] = not is_anonymous
[2975]	185	requested_id = context.getStudentId()
	186	student_id = None
[3858]	187	if not is_anonymous and requested_id:
	188	if (is_student and member_id == requested_id) or is_staff:
	189	student_id = requested_id
	190	else:
[3025]	191	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
[2975]	192	info['student_id'] = student_id
	193	return info
	194	###)
	195
[1818]	196	security.declareProtected(ModifyPortalContent,'openLog') ###(
[1716]	197	def openLog(self,name):
	198	"""open a log file"""
	199	version = 1
	200	path = "%s/log/%s_%d.log" % (i_home,name,version)
	201	while os.path.exists(path):
	202	version += 1
	203	path = "%s/log/%s_%d.log" % (i_home,name,version)
	204	log = open(path,"w")
	205	return log
[2259]	206	###)
[1716]	207
[2259]	208	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
	209	def bypassQueueCatalog(self,enable=True):
	210	"""bypass the QueueCatalog by setting all indexes to process imediate,
	211	if enable is True (default) the old settings are restored
	212	"""
[1818]	213
[2259]	214	###)
	215
[2094]	216	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
	217	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
	218	"""measure Object access Time"""
	219	import random
	220	if hasattr(self,'portal_catalog_real'):
	221	aq_portal = self.portal_catalog_real.evalAdvancedQuery
	222	else:
	223	aq_portal = self.portal_catalog.evalAdvancedQuery
	224	nr_pts = int(nr_pts)
	225	probe = int(probe)
	226	intervall = probe/10
	227	objects = ("application","clearance","personal")
	228	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
	229	#i = random.randrange(num_objects)
	230	count = 0
	231	found = 0
	232	not_found = 0
	233	t_found = 0
	234	t_not_found = 0
	235	time_found = time_not_found = 0.0
	236	t_time_found = t_time_not_found = 0.0
	237	accessed = []
	238	t_min = 1000
	239	t_max = 0
	240	#import pdb;pdb.set_trace()
	241	students = self.portal_catalog(portal_type="Student")
	242	num_students = len(students)
	243	if method == "d":
	244	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
	245	res = aq_portal(query)
	246	brains = {}
	247	for r in res:
	248	sid = r.relative_path.split('/')[-2]
	249	if brains.has_key(sid):
	250	brains[sid][r.portal_type] = r
	251	else:
	252	brains[sid] = {r.portal_type : r}
	253	brains_list = brains.keys()
	254	num_objects = len(brains_list)
	255	else:
	256	num_objects = num_students
	257	print "="*40
	258	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
	259	probe,
	260	nr_pts,
	261	num_objects)
	262	print "nr found/not time found/not min/max"
	263	elapse = time.time()
	264	i_elapse = time.time()
	265	c_elapse = time.clock()
	266	for c in range(1,probe + 1):
	267	i = random.randrange(num_objects)
	268	if method in ('a','b','c'):
	269	student_brain = students[i]
	270	elif method == "d":
	271	#import pdb;pdb.set_trace()
	272	student_brain = brains[brains_list[i]]
	273	if method == "c":
	274	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
	275	res = aq_portal(query)
	276	this_portal_types = [r.portal_type for r in res]
	277	for i in range(nr_pts):
	278	oid = objects[i]
	279	if method == "a":
	280	try:
	281	student_path = student_brain.getPath()
	282	path = "%s/%s" % (student_path,oid)
	283	doc = self.unrestrictedTraverse(path).getContent()
	284	found += 1
	285	i_time = time.time() - i_elapse
	286	time_found += i_time
	287	except:
	288	not_found += 1
	289	i_time = time.time() - i_elapse
	290	time_not_found += i_time
	291	pass
	292	elif method == "b":
	293	try:
	294	student_object = student_brain.getObject()
	295	doc = getattr(student_object,oid).getContent()
	296	found += 1
	297	i_time = time.time() - i_elapse
	298	time_found += i_time
	299	except:
	300	i_time = time.time() - i_elapse
	301	time_not_found += i_time
	302	not_found += 1
	303	pass
	304	elif method == "c":
	305	if portal_types[i] in this_portal_types:
	306	found += 1
	307	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
	308	i_time = time.time() - i_elapse
	309	time_found += i_time
	310	else:
	311	not_found += 1
	312	i_time = time.time() - i_elapse
	313	time_not_found += i_time
	314	elif method == "d":
	315	if student_brain.has_key(portal_types[i]):
	316	found += 1
	317	doc = student_brain[portal_types[i]].getObject().getContent()
	318	i_time = time.time() - i_elapse
	319	time_found += i_time
	320	else:
	321	not_found += 1
	322	i_time = time.time() - i_elapse
	323	time_not_found += i_time
	324	i_elapse = time.time()
	325	if c and (c % intervall == 0):
	326	#i_time = time.time() - i_elapse
	327	t_per = 0.0
	328	if found:
	329	t_per = time_found/found
	330	if t_per > t_max:
	331	t_max = t_per
	332	if t_per > 0.0 and t_per < t_min:
[2119]	333	t_min = t_per
[2094]	334	itf = 0.0
	335	if found:
	336	itf = time_found/found
	337	itnf = 0.0
	338	if not_found :
	339	itnf = time_not_found / not_found
[2119]	340	interval_time = time_found + time_not_found
	341	s = "%(c)d: %(found)d/%(not_found)d " % vars()
[2094]	342	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	343	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	344	print s
	345	t_found += found
	346	t_not_found += not_found
	347	t_time_found += time_found
	348	t_time_not_found += time_not_found
	349	time_found = time_not_found = 0.0
	350	found = not_found = 0
	351	# t_found += found
	352	# t_not_found += not_found
	353	elapse = time.time() - elapse
	354	itf = 0.0
	355	if t_found:
	356	itf = t_time_found/t_found
	357	itnf = 0.0
	358	if t_not_found:
	359	itnf = t_time_not_found / t_not_found
	360	#c_elapse = time.clock() - c_elapse
[2119]	361	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
[2094]	362	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	363	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	364	print "-"*40
	365	print s
	366	rel_found = float(t_found)/probe
	367	rel_not_found = float(t_not_found)/probe
	368	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
	369	print estimated_total_time
	370	###)
	371
[1818]	372	security.declareProtected(ModifyPortalContent,'writeLog') ###(
[1716]	373	def writeLog(self,logfile,s):
	374	"""write to the log file"""
	375	logfile.write(s)
[3473]	376	###)
[1716]	377
[1151]	378	def generateStudentId(self,letter): ###(
	379	import random
[3812]	380	logger = logging.getLogger('WAeUPTool.generateStudentId')
[1151]	381	r = random
[1194]	382	if letter == '?':
[2695]	383	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
[1151]	384	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1720]	385	students = self.portal_url.getPortalObject().campus.students
[4302]	386	while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid) or self.removed_student_ids(id = sid):
	387	if self.waeup_tool.picturePathExists(sid):
	388	logger.info('picture path %s exists or in ' % sid)
[1720]	389	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1151]	390	return sid
[1460]	391	###)
[1415]	392
	393	def generatePassword(self,s=None): ###(
	394	import random
	395	r = random
	396	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
	397	if s is None:
	398	s = 'abcdefghklmnpqrstuvwxy23456789'
	399	pw = ''
	400	while len(pw) < 6:
	401	pw += r.choice(s)
	402	return pw
[1151]	403	###)
[828]	404
[2645]	405	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
	406	def dumpSchoolfeePayments(self):
	407	"dump paid schoolfees"
[2666]	408	mtool = self.portal_membership
	409	member = mtool.getAuthenticatedMember()
[2645]	410	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
	411	aq_student = self.students_catalog.evalAdvancedQuery
	412	query = In('review_state',('schoolfee_paid',
	413	'courses_registered',
	414	'courses_validated',
	415	))
	416	res = aq_student(query)
[2666]	417	#import pdb;pdb.set_trace()
[2645]	418	l = []
	419	logger.info("start for %d" % len(res))
	420	count = 1
	421	log_after = 100
	422	for student in res:
[2656]	423	if not count % log_after:
[2645]	424	logger.info("processed %d total %d" % (log_after,count))
	425	count += 1
	426	fee_dict =self.getSchoolFee(student)
	427	fulltime = student.mode.endswith('_ft')
	428	d = {}
	429	d['student_id'] = student.id
[2666]	430	d['name'] = student.name
	431	d['amount'] = fee_dict.get(new_returning)
[2645]	432	l += d,
[2666]	433	csv_name = self.dumpListToCSV(l,'payments')
	434	logger.info('%s dumped payments to %s' % (member,export_file))
[2645]	435	###)
	436
[2666]	437	security.declarePublic('dumpListToCSV') ###(
	438	def dumpListToCSV(self,l,filename,fields=None):
	439	"""dump a list of dicts to a CSV file"""
	440	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	441	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
	442	if fields is None:
	443	fields = l[0].keys()
	444	headline = ','.join(fields)
	445	out = open(export_file,"wb")
	446	out.write(headline +'\n')
	447	out.close()
	448	out = open(export_file,"a")
	449	csv_writer = csv.DictWriter(out,fields,)
	450	csv_writer.writerows(l)
	451	return export_file
	452	###)
	453
[2643]	454	security.declareProtected(ManagePortal, 'listMembers') ###(
	455	def listMembers(self):
	456	"list all members"
	457	mtool = self.portal_membership
	458	member = mtool.getAuthenticatedMember()
	459	logger = logging.getLogger('WAeUPTool.listMembers')
	460	if str(member) not in ('admin','joachim'):
	461	logger.info('%s tried to list members' % (member))
	462	return None
	463	members = self.portal_directories.members
	464	all = members.listEntryIdsAndTitles()
	465	l = []
	466	for user_id,name in all:
	467	d = {}
	468	d['user_id'] = user_id
	469	d['name'] = name
	470	d['pw'] = getattr(getattr(members,user_id),'password')
[2645]	471	d['email'] = getattr(getattr(members,user_id),'email')
	472	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
	473	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
[2643]	474	l += d,
	475	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	476	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
[3272]	477	logger.info('%s dumped member list to %s' % (member,export_file))
[2643]	478	fields = l[0].keys()
	479	headline = ','.join(fields)
	480	out = open(export_file,"wb")
	481	out.write(headline +'\n')
	482	out.close()
	483	out = open(export_file,"a")
	484	csv_writer = csv.DictWriter(out,fields,)
	485	csv_writer.writerows(l)
	486	###)
[3300]	487
[3272]	488	security.declareProtected(ManagePortal, 'listStudents') ###(
	489	def listStudents(self):
	490	"list all students"
	491	mtool = self.portal_membership
	492	member = mtool.getAuthenticatedMember()
	493	logger = logging.getLogger('WAeUPTool.listStudents')
	494	if str(member) not in ('admin','joachim'):
	495	logger.info('%s tried to list students' % (member))
	496	return None
	497	students = self.portal_directories.students
	498	all = students.listEntryIdsAndTitles()
	499	l = []
	500	for user_id,name in all:
	501	d = {}
	502	d['user_id'] = user_id
	503	d['name'] = name
	504	d['pw'] = getattr(getattr(students,user_id),'password')
	505	d['email'] = getattr(getattr(students,user_id),'email')
	506	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
	507	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
	508	l += d,
	509	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	510	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
	511	logger.info('%s dumped student list to %s' % (member,export_file))
	512	fields = l[0].keys()
	513	headline = ','.join(fields)
	514	out = open(export_file,"wb")
	515	out.write(headline +'\n')
	516	out.close()
	517	out = open(export_file,"a")
	518	csv_writer = csv.DictWriter(out,fields,)
	519	csv_writer.writerows(l)
[3300]	520	###)
[2656]	521
[1890]	522	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
	523	def removeDeletedDocIds(self, max=1000):
	524	"""
	525	remove deleted docids from repository commit after max
	526	"""
	527	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
	528	repository = getToolByName(self, 'portal_repository')
	529	pxtool = getToolByName(self, 'portal_proxies')
[2632]	530	logger.info('start')
[1890]	531	pxtool_infos = pxtool.getRevisionsUsed()
[2632]	532	logger.info('found %d used revisions ' % (len(pxtool_infos)))
[1895]	533
[1890]	534	nb_revs = 0
	535	docids_d = {} # all docids
	536	unused_docids_d = {} # all docids that are unused
	537	ids_unused_revs_docids = [] # ids for revs of unused docids
[2632]	538	unused_ids = [] # ids for unused revs
[1890]	539	total = 0
[1895]	540	idlist = repository.objectIds()
[2632]	541	to_delete = 0
	542	found = False
[1895]	543	for id in idlist:
[1890]	544	docid, rev = repository._splitId(id)
	545	if docid is None:
	546	logger.info("invalid doc_id %s" % docid)
	547	continue
	548	nb_revs += 1
	549	if not pxtool_infos.has_key(docid):
[2632]	550	found = True
	551	to_delete += 1
	552	unused_ids.append(id)
[1890]	553	elif not pxtool_infos[docid].has_key(rev):
[2632]	554	found = True
	555	to_delete += 1
	556	unused_ids.append(id)
	557	if found and not to_delete % max:
	558	found = False
[1890]	559	#import pdb;pdb.set_trace()
[2632]	560	repository.manage_delObjects(unused_ids)
[1890]	561	transaction.commit()
[2632]	562	logger.info('removed %d total %d unused docids ' % (max,to_delete))
	563	else:
	564	if unused_ids:
	565	repository.manage_delObjects(unused_ids)
	566	transaction.commit()
	567	logger.info('finished removing %d unused docids ' % (to_delete))
[3473]	568	###)
[1890]	569
[3858]	570	security.declarePublic('getCredential') ###(
[1250]	571	def getCredential(self,student_id):
	572	student_entry = getattr(self.portal_directories.students,student_id,None)
[3858]	573	#import pdb;pdb.set_trace()
	574	#if not self.isStaff():
	575	# mtool = self.portal_membership
	576	# member = mtool.getAuthenticatedMember()
	577	# logger = logging.getLogger('WAeUPTool.getCredential')
	578	# logger.info('%s tried to access password of %s' % (member,student_id))
	579	# return None
[1250]	580	if student_entry is None:
[4022]	581	return "not set"
[1263]	582	return getattr(student_entry,"password","not set")
[1261]	583	###)
[1250]	584
[2747]	585	security.declarePublic('checkPassword') ###(
[1467]	586	def checkPassword(self,student_id,password):
[1460]	587	student_entry = getattr(self.portal_directories.students,student_id,None)
	588	if student_entry is None:
	589	return False
	590	return getattr(student_entry,"password","not set") == password
[2747]	591	###)
[2712]	592
[2747]	593	security.declarePublic('checkGenericPassword') ###(
[2710]	594	def checkGenericPassword(self,member_id):
	595	member_entry = getattr(self.portal_directories.members,member_id,None)
	596	if member_entry is None:
	597	return False
[2712]	598	ltool = getToolByName(self, 'portal_layouts')
[2710]	599	unsecure_words = ltool._getOb('members')['w__password'].check_words
[2712]	600	password = getattr(member_entry,"password","not set")
	601	is_unsecure = password in unsecure_words
	602	if is_unsecure:
[2713]	603	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
[2712]	604	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
	605	return is_unsecure
[2747]	606	###)
[1460]	607
[4027]	608	security.declarePublic('editPassword') ###(
[1467]	609	def editPassword(self,student_id,password):
	610	"edit a student password"
	611	student_entry = getattr(self.portal_directories.students,student_id,None)
	612	if student_entry is None:
[1571]	613	return
[1467]	614	setattr(student_entry,'password',password)
	615	###)
	616
[2555]	617	security.declareProtected(ModifyPortalContent,'doCommit') ###(
[1401]	618	def doCommit(self,logger=None):
	619	"commit some transactions"
	620	transaction.commit()
	621	###)
	622
[1285]	623	security.declarePublic('loadStudentFoto') ###(
[1813]	624	def loadStudentFoto(self,student,filename,folder):
[1285]	625	"return a student passport picture"
[2413]	626	#import pdb;pdb.set_trace()
	627	picture ="%s/import/%s/%s" % (i_home,folder,filename)
	628	student_id = student.getId()
[2675]	629	images_dir = getImagesDir(student_id)
[2413]	630	if not os.path.exists(images_dir):
	631	os.mkdir(images_dir)
	632	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	633	for extension in ('.jpg','.JPG'):
	634	fullname = "%(picture)s%(extension)s" % vars()
	635	if os.path.exists(fullname):
	636	copy2(fullname,image_name)
	637	return "successfully copied passport picture"
[2467]	638	return "passport picture not found: %s.jpg or .JPG" % picture
[2413]	639	###)
	640
[1285]	641
[1170]	642	security.declareProtected(ModifyPortalContent,'createOne') ###(
[1194]	643	def createOne(self,students_folder,student_brain,letter,commit=False):
	644	sid = self.waeup_tool.generateStudentId(letter)
[1170]	645	students_folder.invokeFactory('Student', sid)
	646	student = getattr(students_folder,sid)
	647	self.portal_workflow.doActionFor(student,'return')
	648	student.manage_setLocalRoles(sid, ['Owner',])
	649	matric_no = student_brain.matric_no
	650	jamb_reg_no = student_brain.Entryregno
	651	self.students_catalog.addRecord(id = sid,
	652	matric_no = matric_no,
	653	jamb_reg_no = jamb_reg_no,
	654	sex = student_brain.Sex == "F",
	655	name = "%s %s %s" % (student_brain.Firstname,
	656	student_brain.Middlename,
	657	student_brain.Lastname)
	658	)
	659	if commit:
	660	transaction.commit()
	661	return sid,jamb_reg_no
	662	###)
	663
[3840]	664	# Functions without docstring cannot be called from outside (via the URL),
	665	# thus we can declare addStudent public.
[3838]	666	security.declarePublic('addStudent') ###(
[1752]	667	def addStudent(self,dict):
[1415]	668	students_folder = self.portal_url.getPortalObject().campus.students
	669	sid = self.waeup_tool.generateStudentId('?')
	670	students_folder.invokeFactory('Student', sid)
	671	student_obj = getattr(students_folder,sid)
[3481]	672	f2t = StudentImport.field2types_student
[1772]	673	#from pdb import set_trace; set_trace()
	674	d = {}
[2297]	675	#d['jamb_sex'] = 'M'
	676	#if dict.get('sex'):
	677	# d['jamb_sex'] = 'F'
[1904]	678
[1898]	679	entry_session = dict.get('entry_session')
[2454]	680	if entry_session == self.getSessionId()[0]:
[1898]	681	wfaction = 'admit'
[1991]	682	wft = 'wf_transition_admit'
[1900]	683	password = None
[1904]	684	else:
[1900]	685	wfaction = 'return'
[1991]	686	wft = 'wf_transition_return'
[1900]	687	password = self.generatePassword()
[1904]	688	self.makeStudentMember(sid,password)
	689
[1728]	690	for pt in f2t.keys():
	691	student_obj.invokeFactory(pt,f2t[pt]['id'])
	692	sub_obj = getattr(student_obj,f2t[pt]['id'])
	693	sub_doc = sub_obj.getContent()
[1898]	694	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
[3481]	695	#d['Title'] = f2t[pt]['title']
[1728]	696	for field in f2t[pt]['fields']:
[1749]	697	d[field] = dict.get(field,'')
[1728]	698	sub_doc.edit(mapping = d)
[1898]	699	new_state = f2t[pt][wft]
[1772]	700	if new_state != "remain":
	701	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
[1415]	702	self.portal_workflow.doActionFor(student_obj,wfaction)
	703	student_obj.manage_setLocalRoles(sid, ['Owner',])
	704	return sid,password
	705	###)
	706
[1151]	707	security.declarePublic('getCertificateBrain') ###(
	708	def getCertificateBrain(self,cert_id):
	709	"do it"
[1849]	710	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	711	{'portal_type':"Certificate",
	712	'id': cert_id})
	713	if res:
	714	return res[0]
	715	return None
	716	###)
[1160]	717
[1756]	718	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
	719	def get_csv_filenames(self):
	720	"do it"
[1759]	721	files = [file for file in os.listdir("%s/import/" % (i_home))
[3143]	722	if file.endswith('.csv') and (file.find('imported') == -1 and
	723	file.find('pending') == -1)]
[1756]	724	return files
	725	###)
	726
[1151]	727	security.declarePublic('findStudentByMatricelNo') ###(
	728	def findStudentByMatricelNo(self,matric_no):
	729	"do it"
[1849]	730	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	731	{'portal_type':"StudentClearance",
	732	'SearchableText': matric_no})
	733	if res:
	734	return res[0]
	735	return None
	736	###)
	737
[3817]	738
	739	security.declarePublic('getOfficerName') ###(
	740	def getOfficerName(self,mid):
	741	"""get the real name of a member"""
	742	membership = self.portal_membership
	743	member_record = membership.getMemberById(mid)
	744	return member_record.getProperty('fullname',None)
	745	###)
	746
	747
[2557]	748	security.declarePublic('makeStudentMember') ###(
[1151]	749	def makeStudentMember(self,sid,password='uNsEt'):
	750	"""make the student a member"""
	751	membership = self.portal_membership
	752	membership.addMember(sid,
	753	password ,
	754	roles=('Member',
	755	'Student',
	756	),
	757	domains='',
	758	properties = {'memberareaCreationFlag': False,
	759	'homeless': True},)
	760	member = membership.getMemberById(sid)
	761	self.portal_registration.afterAdd(member, sid, password, None)
[1261]	762	#self.manage_setLocalRoles(sid, ['Owner',])
[1151]	763	###)
[1160]	764
[2560]	765	security.declareProtected(View,'makeStudentData') ###(
[1158]	766	def makeStudentData(self,student_id,email=None,phone_nr=None):
[1151]	767	"create Datastructure for a returning Student"
[1406]	768	#import pdb;pdb.set_trace()
[1571]	769	logger = logging.getLogger('WAeUPTool.makeStudentData')
[1151]	770	students_folder = self.portal_url.getPortalObject().campus.students
[1794]	771	#res = self.students_catalog(id=student_id)
	772	#if res:
	773	# st = res[0]
	774	#res = self.returning_import(matric_no = st.matric_no)
[1798]	775	res = self.returning_import(id = student_id)
[1151]	776	if res:
[1160]	777	student = res[0]
[1794]	778	else:
[1816]	779	logger.info('Id %s not found in returning_import' % student_id)
[1794]	780	return
[1580]	781	logger.info('%s creates data structure' % student_id)
[1794]	782	s_results = self.results_import(matric_no = student.matric_no)
[1816]	783	if s_results:
	784	lnr = self.getLevelFromResultsCosCode(s_results)
	785	level = "%d00" % lnr
	786	verdict,eligible = self.getVerdict(s_results[0].Verdict)
[2454]	787	#if eligible:
	788	# level = "%d00" % (lnr + 1)
[1816]	789	else:
	790	logger.info('matric_no %s not found in results_import' % student.matric_no)
[2656]	791	level = ''
	792	verdict = ''
[1171]	793	#student should not be allowed to perform this transition
[1174]	794	#wftool = self.portal_workflow
	795	#wftool.doActionFor(student,'return')
[1151]	796	certcode_org = student.Coursemajorcode
	797	certcode = makeCertificateCode(certcode_org)
	798	certificate_brain = self.getCertificateBrain(certcode)
	799	if not certificate_brain:
	800	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
	801	logger.info(em)
	802	matric_no = student.matric_no
	803	sid = student_id
	804	student_obj = getattr(students_folder,sid)
[3114]	805	if not getattr(student_obj,'application'):
	806	student_obj.invokeFactory('StudentApplication','application')
[1151]	807	application = student_obj.application
[1169]	808	self.portal_workflow.doActionFor(application,'open',dest_container=application)
[1151]	809	da = {'Title': 'Application Data'}
	810	student_obj.invokeFactory('StudentPersonal','personal')
	811	da['jamb_reg_no'] = student.Entryregno
[1462]	812	em = self.getEntryMode(student.Entryregno)
[1401]	813	da['entry_mode'] = em
[1151]	814	personal = student_obj.personal
	815	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
	816	dp = {'Title': 'Personal Data'}
	817	student_obj.invokeFactory('StudentClearance','clearance')
	818	clearance = student_obj.clearance
[1169]	819	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[1151]	820	dc = {'Title': 'Clearance/Eligibility Record'}
	821	dc['matric_no'] = matric_no
	822	state = student.State
	823	lga = student.LGA
	824	if state and lga:
	825	lga = state + ' / ' + lga
	826	else:
	827	lga = "None"
[1174]	828	da['jamb_lga'] = dc['lga'] = lga
[1173]	829	da['app_email'] = dp['email'] = email
	830	da['app_mobile'] = dp['phone'] = phone_nr
[1411]	831	dp['firstname'] = student.Firstname
	832	dp['middlename'] = student.Middlename
	833	dp['lastname'] = student.Lastname
	834	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
[1174]	835	da['jamb_sex'] = student.Sex
[1151]	836	dp['sex'] = student.Sex == 'F'
	837	dp['perm_address'] = student.Permanent_Address
	838	application.getContent().edit(mapping=da)
[1169]	839	self.portal_workflow.doActionFor(application,'close',dest_container=application)
[1151]	840	personal.getContent().edit(mapping=dp)
	841	clearance.getContent().edit(mapping=dc)
[1169]	842	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[1151]	843	#
	844	# Study Course
	845	#
	846	student_obj.invokeFactory('StudentStudyCourse','study_course')
	847	studycourse = student_obj.study_course
	848	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
	849	dsc = {}
	850	dsc['study_course'] = certcode
[1401]	851	dsc['current_level'] = level
	852	dsc['current_verdict'] = verdict
[3780]	853	dsc['current_mode'] = em #no longer used
[2466]	854	dsc['current_session'] = '05'
[1151]	855	studycourse.getContent().edit(mapping=dsc)
	856	#
	857	# Level
	858	#
[3473]	859	# l = getattr(studycourse,level,None)
	860	# if l is None:
	861	# studycourse.invokeFactory('StudentStudyLevel', level)
	862	# l = getattr(studycourse, level)
	863	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
	864	# l.getContent().edit(mapping={'Title': "Level %s" % level})
	865	###)
[1160]	866
[3349]	867	def init_timing(self): ###(
	868	if self.with_timing:
	869	if not hasattr(self,'_v_step_times'):
	870	self._v_timer_count = 0
	871	self._v_total = 0
	872	self._v_step_times = {}
	873	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	874	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
	875	self.timer_step = 0
	876	self.total_time = 0
	877	self.elapse = time.time()
	878	self.i_elapse = time.time()
	879	self.c_elapse = time.clock()
[3473]	880	###)
[3349]	881
	882	def do_timing(self): ###(
	883	if self.with_timing:
	884	try:
	885	raise 'dummy'
	886	except:
	887	frame = sys.exc_traceback.tb_frame.f_back
	888	locals = frame.f_locals
	889	globals = frame.f_globals
	890	functionname = frame.f_code.co_name
	891	filename = os.path.basename(frame.f_code.co_filename)
	892	lineno = frame.f_lineno
	893	mod_line = "%(functionname)s:%(lineno)s" % vars()
	894	i_time = time.time() - self.i_elapse
	895	td = {}
	896	if self._v_step_times.has_key(mod_line):
	897	a_time = self._v_step_times[mod_line]['a_time'] + i_time
	898	td['a_time'] = a_time
	899	else:
	900	td['a_time'] = i_time
	901	td['i_time'] = i_time
	902	self._v_step_times[mod_line] = td
	903	self.i_time = i_time
	904	self.total_time += i_time
	905	self.timer_step +=1
	906	self.i_elapse = time.time()
[3473]	907	###)
[3349]	908
	909	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
	910	def print_timing(self):
	911	if self.with_timing:
	912	l = []
	913	timer_count = self._v_timer_count + 1
	914	mod_lines = self._v_step_times.keys()
	915	mod_lines.sort(cmp,reverse=0)
	916	for mod_line in mod_lines:
	917	td = self._v_step_times[mod_line]
	918	i_time = td['i_time']
	919	a_time = td['a_time']/(self._v_timer_count + 1)
	920	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
	921	total_time = self.total_time
	922	total_avarage = self._v_total / timer_count
	923	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
	924	print "\r\n".join(l)
	925	out = open(self._v_timer_file,'a')
	926	out.write("\r\n".join(l))
	927	out.close()
[3473]	928	###)
[3349]	929
	930	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
	931	def get_timing_data(self):
	932	if self.with_timing:
	933	timer_count = self._v_timer_count + 1
	934	results = {}
	935	for k,d in self._v_step_times.items():
	936	dd = {}
	937	dd['a_time'] = d['a_time']/timer_count
	938	dd['i_time'] = d['i_time']
	939	dd['count'] = timer_count
	940	results[k] = dd
	941	dd = {}
	942	dd['a_time'] = self._v_total / timer_count
	943	dd['i_time'] = self.total_time
	944	dd['count'] = timer_count
	945	results["total"] = dd
	946	return results
	947	###)
	948
[2556]	949	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
[3349]	950	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
[2514]	951	"create Datastructure for an admitted Student"
	952	#import pdb;pdb.set_trace()
[3349]	953	logger = logging.getLogger('WAeUPTool.admitOneStudent')
	954	self.with_timing = with_timing
[3681]	955
[4539]	956	try:
[4548]	957	if brain.screening_type in ('cest','sandwich',) and brain.serial and brain.entry_session:
[4539]	958	if brain.course1:
	959	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
	960	else:
	961	reg_no = "%s%s/%s" % (brain.course_admitted[:3],brain.serial,brain.entry_session)
[4532]	962	else:
[4539]	963	reg_no = brain.reg_no
	964	except:
	965	logger.info('applicant record %s has errors' % (brain.reg_no))
	966	return
[3705]	967
[3681]	968	#ignore argument entry_session
	969	if not brain.entry_session:
	970	logger.info('no entry_session for %s provided' % (reg_no))
	971	return
[3961]	972	else:
	973	es = brain.entry_session
	974	if len(es) == 1:
	975	es = '0%c' % es
[3681]	976
[2727]	977	if not hasattr(self,"_v_certificates"):
[3339]	978	self._v_certificates = self.getCertificatesDict()
[2514]	979	students_folder = self.portal_url.getPortalObject().campus.students
[3705]	980
[3264]	981	res = self.students_catalog(jamb_reg_no = reg_no)
[3198]	982	if res:
[3800]	983	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
[3198]	984	return
[2514]	985	if brain.status != "admitted":
[3264]	986	logger.info('status of %s is %s' % (reg_no,brain.status))
[2540]	987	return
[2514]	988	pin_parts = brain.pin.split('-')
[2755]	989	if pin_parts and len(pin_parts) != 3:
[3264]	990	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
[2540]	991	return
[3764]	992	if not brain.course_admitted:
	993	logger.info('no course_admitted provided for %s' % (reg_no))
	994	return
[2727]	995	if brain.course_admitted not in self._v_certificates:
[3264]	996	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
[2727]	997	return
[2959]	998	if brain.sex not in (True,False):
[3264]	999	logger.info('sex of %s not available' % (reg_no))
[3002]	1000	return
[3349]	1001	self.init_timing()
[2540]	1002	student_id = self.generateStudentId('?')
[2514]	1003	students_folder.invokeFactory('Student', student_id)
	1004	student_object = getattr(students_folder,student_id)
[3349]	1005	self.do_timing()
[2576]	1006	if pin_password:
	1007	password = pin_parts[2]
	1008	self.makeStudentMember(student_id,password = password)
[2540]	1009	student_object.manage_setLocalRoles(student_id, ['Owner',])
[3349]	1010	self.do_timing()
[3264]	1011	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
[2514]	1012	#
	1013	# application
	1014	#
[2540]	1015	student_object.invokeFactory('StudentApplication','application')
	1016	application = student_object.application
[2575]	1017	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
[3350]	1018	#self.do_timing()
[2514]	1019	da = {'Title': 'Application Data'}
[3264]	1020	da['jamb_reg_no'] = reg_no
[2576]	1021
[2540]	1022	sex = 'M'
	1023	if brain.sex:
	1024	sex = 'F'
	1025	da['jamb_sex'] = sex
[2670]	1026	da['jamb_age'] = brain.jamb_age
[3341]	1027	da['app_reg_pin'] = brain.pin
[2670]	1028	da['jamb_lga'] = brain.jamb_lga
	1029	da['jamb_state'] = brain.jamb_state
[2575]	1030	da['jamb_score'] = brain.aggregate
[2514]	1031	da['app_email'] = brain.email
[2540]	1032	da['app_mobile'] = brain.phone
[3339]	1033
[3775]	1034	# entry_mode cannot be retrieved from the certtificate
	1035	# because ug_ft has two different entry modes
	1036
[3780]	1037	if brain.entry_mode: # does not happen because import_application
	1038	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
[3775]	1039	elif brain.screening_type == 'pume':
[3776]	1040	da['entry_mode'] = 'ume_ft'
[3775]	1041	elif brain.screening_type == 'pde':
[3776]	1042	da['entry_mode'] = 'de_ft'
[3775]	1043	else:
	1044	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
	1045
	1046	#elif brain.screening_type == 'pce':
	1047	# da['entry_mode'] = 'pce'
	1048	#elif brain.screening_type == 'prence':
	1049	# da['entry_mode'] = 'prence'
	1050	#else:
	1051	# da['entry_mode'] = ''
[3776]	1052
[3339]	1053
[3681]	1054	#da['entry_session'] = entry_session
[3961]	1055	da['entry_session'] = es
[2575]	1056	da['jamb_lastname'] = brain.lastname
	1057	da['jamb_middlename'] = brain.middlenames # different field names!
[2670]	1058	da['jamb_firstname'] = brain.firstname
[2575]	1059	da['screening_application_date'] = brain.application_date
	1060	da['date_of_birth'] = brain.date_of_birth
	1061	da['jamb_first_cos'] = brain.course1
	1062	da['jamb_second_cos'] = brain.course2
	1063	da['course3'] = brain.course3
	1064	da['screening_type'] = brain.screening_type
	1065	da['screening_score'] = brain.screening_score
	1066	da['screening_date'] = brain.screening_date
	1067	da['hq_type'] = brain.hq_type
	1068	da['hq_grade'] = brain.hq_grade
	1069	da['aos'] = brain.aos
[2576]	1070
[2514]	1071	application.getContent().edit(mapping=da)
[3349]	1072	self.do_timing()
[2575]	1073	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
[2514]	1074	#
	1075	# personal
	1076	#
[2540]	1077	student_object.invokeFactory('StudentPersonal','personal')
	1078	personal = student_object.personal
[2575]	1079	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
[3350]	1080	#self.do_timing()
[2514]	1081	dp = {'Title': 'Personal Data'}
	1082	dp['sex'] = brain.sex
	1083	dp['email'] = brain.email
[2540]	1084	dp['phone'] = brain.phone
[2514]	1085	dp['lastname'] = brain.lastname
[2575]	1086	dp['middlename'] = brain.middlenames # different field names!
[2576]	1087	dp['firstname'] = brain.firstname
[2514]	1088	personal.getContent().edit(mapping=dp)
[3349]	1089	self.do_timing()
[2514]	1090	#
	1091	# clearance
	1092	#
[2540]	1093	student_object.invokeFactory('StudentClearance','clearance')
	1094	clearance = student_object.clearance
[2575]	1095	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[2514]	1096	dc = {'Title': 'Clearance/Eligibility Record'}
	1097	dc['lga'] = brain.lga
[2670]	1098	dc['birthday'] = brain.date_of_birth
[2514]	1099	clearance.getContent().edit(mapping=dc)
[3349]	1100	self.do_timing()
[2575]	1101	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[2514]	1102	#
	1103	# study Course
	1104	#
[2540]	1105	student_object.invokeFactory('StudentStudyCourse','study_course')
	1106	studycourse = student_object.study_course
[2575]	1107	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
[3350]	1108	#self.do_timing()
[2514]	1109	dsc = {}
	1110	dsc['study_course'] = brain.course_admitted
	1111	dsc['current_verdict'] = ''
[3780]	1112	dsc['current_mode'] = da['entry_mode'] # no longer used
[2575]	1113	if da['entry_mode'].startswith('de'):
	1114	dsc['current_level'] = '200'
[3040]	1115	elif da['entry_mode'].startswith('pre'):
	1116	dsc['current_level'] = '000'
[2575]	1117	else:
	1118	dsc['current_level'] = '100'
[3961]	1119	dsc['current_session'] = es
[2514]	1120	studycourse.getContent().edit(mapping=dsc)
[3349]	1121	self.do_timing()
[2514]	1122	#
	1123	# payments folder
[2540]	1124	student_object.invokeFactory('PaymentsFolder','payments')
	1125	payments = getattr(student_object,'payments')
[3350]	1126	#self.do_timing()
[2514]	1127	dpay = {}
	1128	dpay['Title'] = 'Payments'
	1129	payments.getContent().edit(mapping=dpay)
[2540]	1130	self.portal_workflow.doActionFor(payments,'open')
[3349]	1131	self.do_timing()
[2580]	1132	#
	1133	# passport foto
	1134	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
	1135	brain.screening_type,
	1136	brain.reg_no)
[2675]	1137	images_dir = getImagesDir(student_id)
	1138	#images_dir = os.path.join("%s" % images_base,student_id)
	1139	letter_dir,student_dir = os.path.split(images_dir)
	1140	if not os.path.exists(letter_dir):
	1141	os.mkdir(letter_dir)
[2580]	1142	if not os.path.exists(images_dir):
	1143	os.mkdir(images_dir)
	1144	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	1145	if os.path.exists(app_picture):
	1146	copy2(app_picture,image_name)
	1147	else:
	1148	logger.info('passport of %s/%s not found: %s' % (student_id,
	1149	brain.reg_no,
	1150	app_picture))
[2656]	1151
[3349]	1152	self.do_timing()
	1153	self.print_timing()
	1154	if with_timing:
	1155	self.timer_step = 0
	1156	self._v_timer_count += 1
	1157	self._v_total += self.total_time
[2514]	1158	return student_id
	1159	###)
	1160
[2556]	1161	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
[1194]	1162	def makeStudentLevel(self,student_id):
	1163	"create the StudyLevel for a returning Student"
	1164	#import pdb;pdb.set_trace()
[1571]	1165	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
[1194]	1166	students_folder = self.portal_url.getPortalObject().campus.students
	1167	res = self.students_catalog(id=student_id)
	1168	if res:
	1169	st = res[0]
	1170	course = st.course
	1171	matric_no = st.matric_no
	1172	level = st.level
	1173	res = self.results_import(matric_no = matric_no)
	1174	if res:
	1175	results = res
[1571]	1176	logger.info('%s creating Level %s' % (student_id,level))
[1194]	1177	#
	1178	# Level
	1179	#
	1180	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1181	studycourse = getattr(student_obj,"study_course",None)
	1182	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
	1183	l = getattr(studycourse,level,None)
	1184	if l is None:
	1185	studycourse.invokeFactory('StudentStudyLevel', level)
	1186	l = getattr(studycourse, level)
	1187	self.portal_workflow.doActionFor(l,'open',dest_container=l)
	1188	l.getContent().edit(mapping={'Title': "Level %s" % level})
	1189	###)
[3984]	1190
	1191
	1192	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
	1193	def exportAllStudyLevels(self,student_id):
	1194	"export the StudyLevels for a student"
	1195	#import pdb;pdb.set_trace()
	1196	logger = logging.getLogger('WAeUPTool.exportAllStudyLevels')
	1197	students_folder = self.portal_url.getPortalObject().campus.students
	1198	res = self.students_catalog(id=student_id)
	1199	if res:
	1200	st = res[0]
	1201	course = st.course
	1202	matric_no = st.matric_no
	1203	level = st.level
	1204	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1205	studycourse = getattr(student_obj,"study_course",None)
	1206	levels = studycourse.objectIds()
	1207
	1208	stool = getToolByName(self, 'portal_schemas')
	1209	schema = stool._getOb('student_study_level')
	1210	fields = ['student_id','level']
	1211	fields.extend(schema.keys())
	1212	format = '"%(' + ')s","%('.join(fields) + ')s"'
	1213	export_file = "%s/export/study_levels_removed.csv" % (i_home)
	1214
[3989]	1215	if not os.path.exists(export_file):
	1216	file_handler = open(export_file,"a")
	1217	headline = ','.join(fields)
	1218	file_handler.write(headline +'\n')
	1219	else:
	1220	file_handler = open(export_file,"a")
	1221
[3984]	1222	for level in levels:
	1223	level_object = getattr(studycourse,level)
	1224	level_content = level_object.getContent()
	1225	d = {'student_id':student_id,'level':level}
	1226	for field in schema.keys():
	1227	d[field] = getattr(level_content,field,'')
	1228	line = format % d
	1229	file_handler.write(line +'\n')
	1230
	1231	###)
[1194]	1232
[2000]	1233	security.declarePublic('getHallInfo') ###(
	1234	def getHallInfo(self,bed):
	1235	"""return Hall Info"""
[828]	1236	info = {}
[3406]	1237	bedsplit = bed.split('_')
	1238	if len(bedsplit) == 4:
	1239	hall,block,room,letter = bed.split('_')
	1240	else:
	1241	info['maintenance_code'] = 'None'
	1242	return info
[1846]	1243	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
[828]	1244	if res and len(res) == 1:
	1245	hall_brain = res[0]
	1246	hall_doc = hall_brain.getObject().getContent()
	1247	else:
	1248	return info
	1249	info['hall_title'] = hall_brain.Title
	1250	info['maintenance_code'] = hall_doc.maintenance_code
[1846]	1251	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
[828]	1252	batch_doc = None
	1253	for brain in res:
	1254	if brain.id.startswith(info['maintenance_code']):
	1255	batch_doc = brain.getObject().getContent()
	1256	break
	1257	if batch_doc is None:
[2913]	1258	info['maintenance_fee'] = ''
[828]	1259	else:
	1260	info['maintenance_fee'] = batch_doc.cost
	1261	return info
[1151]	1262	###)
[828]	1263
[2396]	1264	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
	1265	def removePictureFolder(self,student_id):
	1266	"""remove picture_folder by renaming it"""
[3807]	1267	picture_path = getImagesDir(student_id)
	1268	dest_path = os.path.join("%s" % images_base,'removed',student_id)
	1269	dest_path = dest_path + "_removed"
	1270	if os.path.exists(dest_path) or not os.path.exists(picture_path):
[2396]	1271	return False
[3807]	1272	os.rename(picture_path,dest_path)
[2396]	1273	return True
	1274	###)
	1275
	1276	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
	1277	def restorePictureFolder(self,student_id):
	1278	"""restore picture_folder by renaming it"""
[3807]	1279	picture_path = getImagesDir(student_id)
	1280	orig_path = os.path.join("%s" % images_base,'removed',student_id)
	1281	orig_path = orig_path + "_removed"
	1282	if os.path.exists(picture_path) or not os.path.exists(orig_path):
	1283	return False
[4558]	1284	os.rename(orig_path,picture_path)
[2396]	1285	return True
	1286	###)
	1287
[2363]	1288	security.declarePublic('picturesExist') ###(
[2420]	1289	def picturesExist(self, ids,student_id=None):
[2363]	1290	"""check if pictures exist in the filesystem"""
[2420]	1291	if student_id is None:
	1292	student_id = self.getStudentId()
	1293	if student_id is None:
	1294	return False
[2675]	1295	picture_path = getImagesDir(student_id)
	1296	#picture_path = os.path.join(images_base,student_id)
[2363]	1297	if not os.path.exists(picture_path):
	1298	return False
	1299	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1300	return set(ids).issubset(set(pictures))
	1301	###)
[3805]	1302
[3812]	1303	security.declarePublic('picturePathExists') ###(
	1304	def picturePathExists(self, student_id=None):
	1305	"""check if picture path exists in the filesystem"""
	1306	if student_id is None:
	1307	student_id = self.getStudentId()
	1308	if student_id is None:
	1309	return False
	1310	picture_path = getImagesDir(student_id)
	1311	if os.path.exists(picture_path):
	1312	return True
	1313	return False
	1314	###)
[3805]	1315
[3812]	1316
[3807]	1317	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
	1318	def removeUnusedImageFolders(self):
[3806]	1319	"""check if an unused image folders exists in the filesystem"""
[3807]	1320	mtool = self.portal_membership
	1321	member = mtool.getAuthenticatedMember()
	1322	member_id = str(member)
	1323	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
[3806]	1324	abc = os.listdir(images_base)
[3805]	1325	ifolders = []
[3806]	1326	for i in abc:
[3805]	1327	picture_path = os.path.join(images_base,i)
	1328	ifolders.extend(os.listdir(picture_path))
	1329	unused_ids = []
	1330	for id in ifolders:
	1331	res = self.students_catalog(id=id)
	1332	if not res:
	1333	unused_ids.append(id)
[3807]	1334	#import pdb;pdb.set_trace()
	1335	if not id.endswith('removed'):
	1336	removed = self.waeup_tool.removePictureFolder(id)
	1337	if removed:
	1338	logger.info('%s: image folder %s successfully removed' % (member_id,id))
	1339	else:
	1340	logger.info('%s: image folder %s could not be removed' % (member_id,id))
	1341	return
[3805]	1342
	1343	###)
	1344
[2364]	1345	security.declarePublic('picturesList') ###(
	1346	def picturesList(self):
	1347	"""check if pictures exist in the filesystem"""
	1348	path = 'images'
	1349	student_id = self.getStudentId()
[2677]	1350	#picture_path = os.path.join(i_home,path,student_id)
	1351	picture_path = getImagesDir(student_id)
[2364]	1352	if not os.path.exists(picture_path):
	1353	return []
	1354	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1355	###)
	1356
[2114]	1357	security.declarePublic('showFsPicture') ###(
	1358	def showFsPicture(self,path):
	1359	"""return a picture from the filesystem"""
[2675]	1360	#picture_path = os.path.join(i_home,path)
	1361	picture_path = os.path.join(images_base,path)
[2335]	1362	response = self.REQUEST.RESPONSE
[2342]	1363	#import pdb;pdb.set_trace()
[2354]	1364	registry = getToolByName(self, 'mimetypes_registry')
	1365	mimetype = str(registry.lookupExtension(path.lower()) or
	1366	registry.lookupExtension('file.bin'))
[2114]	1367	if os.path.exists(picture_path):
[2354]	1368	response.setHeader('Content-type',mimetype)
[2114]	1369	return open(picture_path).read()
[2335]	1370	picture_path = os.path.join(i_home,'import',path)
	1371	if os.path.exists(picture_path):
	1372	return open(picture_path).read()
[2114]	1373	###)
	1374
[1151]	1375	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
	1376	def deleteAllCourses(self,department="All"):
	1377	''' delete the courses'''
	1378	pm = self.portal_membership
	1379	member = pm.getAuthenticatedMember()
[1160]	1380
[1151]	1381	if str(member) not in ("henrik","joachim"):
	1382	return "not possible"
	1383	if department == "All":
	1384	res = self.portal_catalog({'meta_type': 'Department'})
	1385	if len(res) < 1:
	1386	return "No Departments found"
[1160]	1387
[1151]	1388	deleted = []
	1389	for dep in res:
	1390	cf = dep.getObject().courses
	1391	if cf:
	1392	cf.manage_delObjects(ids=cf.objectIds())
	1393	deleted.append("deleted Courses in %s" % dep.getId)
	1394	return "\r".join(deleted)
[1160]	1395	###)
[1151]	1396
[1572]	1397	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
	1398	def getLogfileLines(self,filename="event.log",numlines=20):
	1399	"""Get last NUMLINES lines of logfile FILENAME.
[1160]	1400
[1572]	1401	Return last lines' of a file in the instances logfile directory as
	1402	a list. The number of returned lines equals `numlines' or less. If
	1403	less than `numlines' lines are available, the whole file ist
	1404	returned. If the file can not be opened or some other error
	1405	occurs, empty list is returend.
	1406	"""
	1407	result = []
	1408	lines_hit = 0
	1409
	1410	# We only handle files in instances' log directory...
	1411	logpath = os.path.join(i_home, "log")
	1412	filename = str(os.path.abspath( os.path.join( logpath, filename )))
	1413	if not filename.startswith( logpath ):
	1414	# Attempt to access file outside log-dir...
	1415	return []
	1416
	1417	try:
	1418	fd = file( filename, "rb" )
	1419	except IOError:
	1420	return []
	1421	if not fd:
	1422	return []
	1423
	1424	if os.linesep == None:
	1425	linesep = '\n'
	1426	else:
	1427	linesep = os.linesep
	1428
	1429	# Try to find 'numlines' times a lineseparator, searching from end
	1430	# and moving to the beginning of file...
	1431	fd.seek( 0, 2) # Move to end of file...
	1432	while lines_hit < numlines:
	1433	if fd.read(1) == linesep[-1]: # This moves filedescriptor
	1434	# one step forward...
	1435	lines_hit += 1
	1436	try:
	1437	fd.seek( -2, 1) # Go two bytes back from current pos...
	1438	except IOError:
	1439	# We cannot go back two bytes. Maybe the file is too small...
	1440	break
	1441	fd.seek(2,1)
	1442
	1443	# Read all lines from current position...
	1444	result = fd.readlines()
	1445	# Remove line endings...
	1446	result = [x.strip() for x in result]
	1447	fd.close()
	1448	return result
	1449	###)
	1450
[1700]	1451	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
[1665]	1452	def getCallbacksFromLog(self,filename):
	1453	"""fix Online Payment Transactions from Z2.log entries"""
	1454	import transaction
	1455	import random
	1456	from cgi import parse_qs
	1457	from urlparse import urlparse
	1458	#from pdb import set_trace
	1459	wftool = self.portal_workflow
	1460	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1461	students_folder = self.portal_url.getPortalObject().campus.students
	1462	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
	1463	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
	1464	s += r'(?P<intersw>.)" "(?P<agent>.)"'
	1465	data = re.compile(s)
	1466	start = True
	1467	tr_count = 1
	1468	total = 0
	1469	#name = 'pume_results'
	1470	#name = 'epaymentsuccessful_z2log2'
	1471	name = filename
	1472	no_import = []
	1473	imported = []
	1474	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
	1475	try:
	1476	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
	1477	except:
	1478	logger.error('Error reading %s' % name)
	1479	return
	1480	tas = []
	1481	for line in transactions:
	1482	dict = {}
	1483	items = data.search(line)
	1484	dict['idict'] = idict = items.groupdict()
	1485	#print idict
	1486	#from pdb import set_trace;set_trace()
	1487	urlparsed = urlparse(idict['get'][4:])
	1488	#print urlparsed
	1489	path = urlparsed[2].split('/')
	1490	dict['student_id'] = student_id = path[8]
	1491	dict['payment_id'] = payment_id = path[10]
	1492	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
	1493	tas.append(dict)
	1494	tr_count += 1
	1495	return tas
	1496	###)
	1497
[1620]	1498	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
	1499	def importOnlinePaymentTransactions(self):
	1500	"""load Online Payment Transactions from CSV values"""
	1501	import transaction
	1502	import random
	1503	#from pdb import set_trace
	1504	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
[1625]	1505	opt = self.online_payments_import
[1620]	1506	students_folder = self.portal_url.getPortalObject().campus.students
	1507	start = True
	1508	tr_count = 1
	1509	total = 0
	1510	#name = 'pume_results'
	1511	name = 'OnlineTransactions'
	1512	no_import = []
	1513	imported = []
	1514	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
	1515	try:
	1516	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1517	except:
	1518	logger.error('Error reading %s.csv' % name)
	1519	return
	1520	for pay_transaction in transactions:
	1521	if start:
	1522	start = False
	1523	logger.info('Start loading from %s.csv' % name)
	1524	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
	1525	no_import.append('%s,"Error"' % s)
	1526	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
	1527	format_error = format + ',"%(Error)s"'
	1528	data = {}
[1644]	1529
[1781]	1530	# format of the first file sent by Tayo
[1643]	1531	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
	1532	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1533	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1534	#data['response_code'] = response_code = pay_transaction['Resp Code']
	1535	#data['amount'] = amount = pay_transaction['Amount']
[1644]	1536
[1781]	1537	# format of the second file sent by Tayo
[1798]	1538	#data['datetime'] = date = 0
	1539	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1540	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1541	#data['response_code'] = response_code = '00'
	1542	#data['amount'] = amount = pay_transaction['Amount']
[1813]	1543
[1798]	1544	# format of the third file sent by Kehinde
[1643]	1545	data['datetime'] = date = 0
[1798]	1546	data['student_id'] = student_id = pay_transaction['customer_id']
	1547	data['order_id'] = order_id = pay_transaction['merchant_reference']
[1644]	1548	data['response_code'] = response_code = '00'
[1813]	1549	data['amount'] = amount = pay_transaction['Amount']
[1644]	1550
[1620]	1551	dup = False
	1552	if response_code == "12":
	1553	continue
	1554	try:
	1555	opt.addRecord(**data)
	1556	except ValueError:
	1557	dup = True
	1558	#from pdb import set_trace;set_trace()
	1559	if dup:
	1560	if response_code == "00":
[1798]	1561	try:
	1562	opt.modifyRecord(**data)
	1563	except:
	1564	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
	1565	continue
[1620]	1566	else:
[1674]	1567	pay_transaction['Error'] = "Duplicate order_id"
[1620]	1568	no_import.append( format_error % pay_transaction)
[1674]	1569	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
[1620]	1570	continue
	1571	tr_count += 1
	1572	if tr_count > 1000:
	1573	if len(no_import) > 0:
	1574	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1575	'\n'.join(no_import) + '\n')
	1576	no_import = []
[1645]	1577	em = '%d transactions committed\n' % (tr_count)
[1620]	1578	transaction.commit()
	1579	regs = []
	1580	logger.info(em)
	1581	total += tr_count
	1582	tr_count = 0
	1583	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1584	'\n'.join(no_import))
	1585	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1586	###)
[2237]	1587
[3172]	1588	security.declareProtected(ModifyPortalContent,"importData")###(
	1589	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
	1590	"""load data from CSV values"""
	1591	import transaction
	1592	import random
	1593	students_folder = self.portal_url.getPortalObject().campus.students
[3292]	1594	uploads_folder = self.portal_url.getPortalObject().campus.uploads
[3172]	1595	pending_only = False
[3177]	1596	pend_str = '--'
[3172]	1597	elapse = time.time()
	1598	#
	1599	# preparations
	1600	#
[3177]	1601	if filename == pend_str:
[3172]	1602	pending_only = True
	1603	importer_name = ''.join([part.capitalize() for part in name.split('_')])
[3191]	1604	importer = eval("%sImport" % importer_name)(self)
[3172]	1605	logger = importer.logger
	1606	if importer.init_errors:
[3191]	1607	logger.info(importer.init_errors)
	1608	return importer.init_errors
[3172]	1609	member = importer.member
[3320]	1610	#current = importer.current
[3172]	1611	import_date = importer.import_date
	1612	#
	1613	# not_imported
	1614	#
	1615	info = importer.info
	1616	data_keys = importer.data_keys
[3181]	1617	csv_keys = importer.csv_keys
[3172]	1618	#csv_keys.extend(info.keys())
	1619	headline_mapping = dict((k,k) for k in csv_keys)
	1620	#
	1621	# pending
	1622	#
	1623	pending_path = importer.pending_path
	1624	pending_tmp = importer.pending_tmp
	1625	pending_backup = importer.pending_backup
	1626	pending_fn = importer.pending_fn
	1627	imported_path = importer.imported_path
	1628	imported_fn = importer.imported_fn
[3180]	1629	commit_after = importer.commit_after
[3172]	1630	pending = []
	1631	pending_digests = []
	1632	#total_added_to_pending = 0
	1633	if not pending_only:
	1634	pending,pending_digests = importer.makeIdLists()
	1635	pending_at_start = len(pending)
	1636	datafile = open(pending_tmp,"w")
	1637	pending_csv_writer = csv.DictWriter(datafile,
	1638	csv_keys,
	1639	extrasaction='ignore')
	1640	pending_csv_writer.writerow(headline_mapping)
	1641	datafile.close()
	1642	#
	1643	# imported
	1644	#
	1645	if not os.path.exists(imported_path):
	1646	datafile = open(imported_path,"w")
	1647	imported_csv_writer = csv.DictWriter(datafile,
	1648	csv_keys,
	1649	extrasaction='ignore')
	1650	imported_csv_writer.writerow(headline_mapping)
	1651	datafile.close()
	1652	start = True
	1653	tr_count = 0
	1654	total = 0
	1655	total_added_to_imported = 0
	1656	total_pending = 0
[3292]	1657	import_source_done = ""
[3172]	1658	if pending_only:
	1659	import_source_path = pending_path
[2193]	1660	else:
[3172]	1661	import_source_path = "%s/import/%s.csv" % (i_home,filename)
[3292]	1662	import_source_done = "%s/import/%s.done" % (i_home,filename)
[3172]	1663	if not os.path.exists(import_source_path):
[3195]	1664	fn = os.path.split(import_source_path)[1]
[3268]	1665	em = 'no import file %(fn)s' % vars()
[3172]	1666	return em
	1667	import_source_fn = os.path.split(import_source_path)[1]
[3267]	1668	if not pending_only:
	1669	info['imported_from'] = import_source_fn
[3277]	1670	headline = csv.reader(open(import_source_path,"rb")).next()
	1671	if "import_mode" not in headline:
[3191]	1672	msg = 'import_mode must be in heading'
	1673	return msg
[3277]	1674	invalid_keys = importer.checkHeadline(headline)
	1675	if invalid_keys:
	1676	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
[3201]	1677
[3300]	1678	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
	1679	or k.strip() in info.keys())]
[3277]	1680	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
	1681	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
	1682	# if diff2schema and diff2schema != set(['id',]):
	1683	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
	1684	# return msg
[3172]	1685	#
	1686	# start importing
	1687	#
[2193]	1688	try:
[3172]	1689	reader = csv.DictReader(open(import_source_path,"rb"))
	1690	except:
	1691	msg = 'Error reading %s.csv' % filename
	1692	logger.error(msg)
	1693	return msg
	1694	items = [item for item in reader]
	1695	total_to_import = len(items)
[3181]	1696	tti_float = float(total_to_import)
[3172]	1697	if pending_only:
	1698	pending_at_start = total_to_import
	1699	count = 0
	1700	imported = []
[3391]	1701	old_commit_count = 0
[3204]	1702	error_count = imported_count = 0
[3172]	1703	already_in = 0
	1704	for record in items:
	1705	item = {}
[3198]	1706	empty_value_keys = []
[3172]	1707	for k,v in record.items():
[3346]	1708	if k is None:
	1709	continue
[3172]	1710	if v:
[3198]	1711	if v == EMPTY:
	1712	empty_value_keys += k,
	1713	v = ''
[3172]	1714	item[k.strip()] = v.strip()
	1715	count += 1
	1716	if start:
	1717	start = False
	1718	adapters = [MappingStorageAdapter(importer.schema, item)]
[3191]	1719	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
[3172]	1720	dm = DataModel(item, adapters,context=self)
	1721	ds = DataStructure(data=item,datamodel=dm)
	1722	error_string = ""
	1723	total += 1
[3271]	1724	import_mode = item.get('import_mode','')
[3191]	1725	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
	1726	if import_method is None:
[3271]	1727	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
[3251]	1728	elif (import_mode in importer.required_modes and
[3248]	1729	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
	1730	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
	1731	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
[3191]	1732	else:
	1733	for k in import_keys:
	1734	if k not in item.keys() or k not in importer.validators.keys():
	1735	continue
	1736	if not importer.validators[k](ds,mode=import_mode):
[3201]	1737	error_string += ' ++ '
	1738	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
[3172]	1739	ds.getErrorMapping(k)))
	1740	if error_string:
	1741	error = error_string
	1742	id = ''
	1743	mapping = item
[2699]	1744	else:
[3172]	1745	temp_item = item.copy()
	1746	temp_item.update(dm)
[3185]	1747	results = import_method(temp_item)
[3172]	1748	id = results[0]
	1749	error = results[1]
	1750	mapping = results[2]
[3267]	1751	#
	1752	# restore some values
	1753	#
[3198]	1754	for k in empty_value_keys:
	1755	mapping[k] = EMPTY
[3186]	1756	if mapping.has_key('sex'):
	1757	#import pdb;pdb.set_trace()
[3267]	1758	if mapping['sex'] == True:
[3186]	1759	mapping['sex'] = 'F'
[3267]	1760	elif mapping['sex'] == False:
[3186]	1761	mapping['sex'] = 'M'
[3267]	1762	if pending_only:
	1763	info['imported_from'] = item['imported_from']
[3181]	1764	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
[3172]	1765	info['error'] = error
[3326]	1766	info['import_record_no'] = count + 1
[3172]	1767	mapping.update(info)
[3178]	1768	log_list = []
[3172]	1769	if error:
[3204]	1770	error_count += 1
[3172]	1771	digest = makeDigest(mapping,data_keys)
	1772	if digest not in pending_digests:
	1773	pending_digests += digest,
	1774	pending.append(mapping)
	1775	if not pending_only:
[3251]	1776	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
[3172]	1777	else:
	1778	already_in += 1
	1779	pass
	1780	else:
	1781	imported_count += 1
	1782	imported += mapping,
[3251]	1783	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
[3178]	1784	if log_list:
	1785	time_till_now = time.time() - elapse
[3204]	1786	percent_finished = (error_count + imported_count)/tti_float*100
[3181]	1787	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
[3178]	1788	logger.info(' '.join(log_list))
[3172]	1789	finished = count > total_to_import - 1
[3277]	1790	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
	1791	if must_commit:
	1792	old_commit_count = imported_count
[3181]	1793
[3172]	1794	if must_commit or finished:
	1795	if len(imported):
	1796	transaction.commit()
	1797	datafile = open(imported_path,"a")
	1798	writer = csv.DictWriter(datafile,
	1799	csv_keys,
	1800	extrasaction='ignore')
	1801	writer.writerows(imported)
	1802	datafile.close()
	1803	total_added_to_imported += len(imported)
	1804	imported = []
	1805	if len(pending) > 0:
	1806	datafile = open(pending_tmp,"a")
	1807	writer = csv.DictWriter(datafile,
	1808	csv_keys,
	1809	extrasaction='ignore')
	1810	writer.writerows(pending)
	1811	datafile.close()
	1812	total_pending += len(pending)
	1813	#total_added_to_pending += len(pending)
	1814	pending = []
	1815	if not finished:
[3335]	1816	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
[3172]	1817	logger.info(msg)
	1818	elapse = time.time() - elapse
[3271]	1819	if os.path.exists(pending_path):
	1820	copy2(pending_path,pending_backup)
[3172]	1821	copy2(pending_tmp,pending_path)
[3191]	1822	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
[3181]	1823	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
[3172]	1824	if pending_only:
	1825	removed_pending = pending_at_start - total_pending
[3184]	1826	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
[2699]	1827	else:
[3181]	1828	added_pending = total_pending - pending_at_start
	1829	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
	1830	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
[3172]	1831	logger.info(msg)
[3292]	1832	if import_source_done:
	1833	copy(import_source_path,import_source_done)
	1834	os.remove(import_source_path)
	1835	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
	1836	if upload is not None:
	1837	upload_doc = upload.getContent()
	1838	mapping = {}
[3308]	1839	#mapping['import_date'] = DateTime.DateTime()
	1840	mapping['import_date'] = import_date
[3327]	1841	mapping['imported_by'] = importer.imported_by
[3292]	1842	mapping['import_message'] = msg
	1843	upload_doc.edit(mapping = mapping)
[3178]	1844	os.remove(pending_tmp)
[3172]	1845	return msg
[2514]	1846	###)
	1847
[2292]	1848	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
	1849	def moveImagesToFS(self,student_id="O738726"):
	1850	"move the images to the filesystem"
[2675]	1851	images_dir = getImagesDir(student_id)
	1852	#images_dir = os.path.join("%s" % images_base,student_id)
[2292]	1853	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1854	stool = getToolByName(self, 'portal_schemas')
	1855	schemas = ['student_application',
	1856	'student_clearance',
	1857	]
	1858	created = False
	1859	for schema_id in schemas:
	1860	schema = stool._getOb(schema_id)
	1861	object = getattr(student_folder,schema_id[len('student_'):],None)
	1862	if object is None:
	1863	continue
	1864	doc = object.getContent()
	1865	for key in schema.keys():
	1866	if schema[key].meta_type != "CPS Image Field":
	1867	continue
	1868	#import pdb;pdb.set_trace()
[2351]	1869	image = getattr(doc,key,None)
	1870	if not image or not hasattr(image,"data"):
[2292]	1871	continue
	1872	if not created:
	1873	if not os.path.exists(images_dir):
	1874	os.mkdir(images_dir)
	1875	created = True
	1876	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
	1877	open(filename,"wb").write(str(image.data))
[2500]	1878	###)
[2292]	1879
[2300]	1880	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
	1881	def movePassportToFS(self,student_id="O738726"):
	1882	"move the passports to the filesystem"
	1883	images_dir = os.path.join("%s" % i_home,'passports')
	1884	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1885	stool = getToolByName(self, 'portal_schemas')
	1886	schemas = ['student_application',
	1887	#'student_clearance',
	1888	]
	1889	created = False
	1890	for schema_id in schemas:
	1891	schema = stool._getOb(schema_id)
	1892	object = getattr(student_folder,schema_id[len('student_'):],None)
	1893	if object is None:
	1894	continue
	1895	doc = object.getContent()
	1896	for key in schema.keys():
	1897	if schema[key].meta_type != "CPS Image Field":
	1898	continue
	1899	#import pdb;pdb.set_trace()
	1900	image = getattr(doc,key)
	1901	if not hasattr(image,"data"):
	1902	continue
	1903	if not created:
	1904	if not os.path.exists(images_dir):
	1905	os.mkdir(images_dir)
	1906	created = True
	1907	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
	1908	open(filename,"wb").write(str(image.data))
[2500]	1909	###)
[3824]	1910
	1911
	1912	security.declarePublic('getConfigParams')
	1913	def getConfigParams(self,conf_id="configuration"):
	1914	conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
	1915	conf_obj = conf.getContent()
	1916	stool = getToolByName(self, 'portal_schemas')
	1917	schema = stool._getOb('configuration')
	1918	d = {}
	1919	for key in schema.keys():
	1920	d[key] = getattr(conf_obj,key,None)
	1921	return d
[2300]	1922
[3824]	1923
[828]	1924	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: