Context navigation

source: WAeUP_SRP/trunk/WAeUPTool.py @ 7900

Last change on this file since 7900 was 5656, checked in by Henrik Bettermann, 14 years ago
Log failed logins.
Property svn:keywords set to `Id`
File size: 85.7 KB

Rev	Line
[3219]	1	# -- mode: python; mode: fold; --
[197]	2	# (C) Copyright 2005 The WAeUP group <http://www.waeup.org>
	3	# Author: Joachim Schmitz (js@aixtraware.de)
	4	#
	5	# This program is free software; you can redistribute it and/or modify
	6	# it under the terms of the GNU General Public License version 2 as published
	7	# by the Free Software Foundation.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program; if not, write to the Free Software
	16	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
	17	# 02111-1307, USA.
	18	#
	19	# $Id: WAeUPTool.py 5656 2011-01-23 21:48:44Z henrik $
[1174]	20	"""The WAeUP Tool Box.
[197]	21	"""
	22
	23	from AccessControl import ClassSecurityInfo
[828]	24	from Acquisition import aq_inner
	25	from Acquisition import aq_parent
	26	from Globals import DTMLFile
	27	from Globals import InitializeClass
	28	from OFS.SimpleItem import SimpleItem
[2283]	29	from zExceptions import BadRequest
[197]	30
[1890]	31	from Products.CMFCore.utils import getToolByName
[1747]	32	from Products.CPSSchemas.DataStructure import DataStructure
	33	from Products.CPSSchemas.DataModel import DataModel
	34	from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
[828]	35	from Products.CMFCore.ActionProviderBase import ActionProviderBase
	36	from Products.CMFCore.permissions import View
	37	from Products.ZCatalog.ZCatalog import ZCatalog
	38	from Products.CMFCore.permissions import ModifyPortalContent
[1890]	39	from Products.CMFCore.permissions import ManagePortal
[197]	40	from Products.CMFCore.utils import UniqueObject
[1194]	41	from Products.CMFCore.URLTool import URLTool
[1747]	42	from Products.CMFCore.utils import getToolByName
[1151]	43	from Students import makeCertificateCode
[1285]	44	from Globals import package_home,INSTANCE_HOME
[3172]	45	from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
[3988]	46	from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport,PaymentImport
[3172]	47	from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
	48	from utils import makeDigest
[2094]	49	import DateTime,time
[1620]	50	import logging
[1170]	51	import transaction
[2193]	52	import csv,re,os,sys
[3172]	53	import md5
[5656]	54	import base64
[3292]	55	from shutil import copy2,copy
[1707]	56	from Products.AdvancedQuery import Eq, Between, Le,In
[197]	57
[2413]	58	p_home = package_home(globals())
	59	i_home = INSTANCE_HOME
	60	images_base = os.path.join(i_home,"images")
[3198]	61	EMPTY = 'XXX'
[2413]	62
[2675]	63	def getImagesDir(student_id):
	64	return os.path.join("%s" % images_base,student_id[0],student_id)
[2413]	65
[1707]	66	def getObject(object,name):
	67	if object.hasObject(name):
	68	return getattr(object,name)
	69	return None
[1720]	70
[828]	71	class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
[197]	72	"""WAeUP tool"""
	73
[828]	74	id = 'waeup_tool'
[197]	75	meta_type = 'WAeUP Tool'
[828]	76	_actions = ()
[197]	77	security = ClassSecurityInfo()
[828]	78	security.declareObjectProtected(View)
	79	manage_options = ( ActionProviderBase.manage_options
	80	+ SimpleItem.manage_options
	81	)
	82
[2999]	83	security.declareProtected(View,'re_split') ###(
	84	def re_split(self,split_string,string):
	85	return re.split(split_string,string)
[3014]	86	###)
[2999]	87
	88	security.declareProtected(View,'difference') ###(
	89	def difference(self,l1,l2):
	90	return set(l1).difference(set(l2))
[3014]	91	###)
[2999]	92
[1818]	93	def rwrite(self,s): ###(
[1707]	94	response = self.REQUEST.RESPONSE
	95	response.setHeader('Content-type','text/html; charset=ISO-8859-15')
	96	response.write("%s<br />\r\n" % s)
[1818]	97	###)
[1174]	98
[2695]	99	def addtodict(self,d,key,item): ###(
	100	d[key].append(item)
	101	return d[key]
	102	###)
	103
[1818]	104	def sleep(self,secs): ###(
	105	"sleep"
	106	import time
	107	time.sleep(secs)
	108	return
[3473]	109	###)
[1827]	110
[2695]	111	security.declareProtected(View,'updateRoleMappingsFor') ###(
	112	def updateRoleMappingsFor(self,wf_definition,ob):
	113	"do so for public"
	114	wf_def = getattr(self.portal_workflow,wf_definition)
	115	wf_def.updateRoleMappingsFor(ob)
	116	###)
[2699]	117
[2999]	118	security.declareProtected(View,'getStatesLgas') ###(
	119	def getStatesLgas(self):
	120	"""return lga info"""
	121	voc = getattr(self.portal_vocabularies,'local_gov_areas')
	122	states = []
	123	lgas = []
	124	d = {}
	125	wd = {}
	126	for k,v in voc.items():
	127	parts = v.split(' / ')
	128	if len(parts) == 1:
	129	state = parts[0].lower()
	130	lga = ""
	131	elif len(parts) == 2:
	132	state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
	133	lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
	134	else:
	135	continue
	136	if state not in states:
	137	states.append(state)
	138	if lga not in lgas:
	139	lgas.append(lga)
	140	words = re.split('[^a-zA-Z0-9/]',k)
	141	words.sort()
	142	wd[k] = words
	143	d[k] = v
	144	mapping = {}
	145	mapping['word_dict'] = wd
	146	mapping['lga_dict'] = d
	147	mapping['states'] = states
	148	mapping['lgas'] = lgas
	149	return mapping
	150	###)
	151
[3014]	152	security.declareProtected(View,'findLga') ###(
	153	def findLga(self,words,words_dict):
	154	words = re.split('[^a-zA-Z0-9/]',words)
	155	lga_words = []
	156	for word in words:
	157	if word:
	158	lga_words += word.strip().lower(),
	159	lga_words.sort()
	160	state_lga = ''
	161	while not state_lga:
	162	for k,l in words_dict.items():
	163	if lga_words == l:
	164	state_lga = k
	165	break
	166	break
	167	return state_lga
	168	###)
[2975]	169	security.declareProtected(View,'getAccessInfo') ###(
	170	def getAccessInfo(self,context):
	171	"return a dict with access_info"
	172	logger = logging.getLogger('WAeUPTool.getAccessInfo')
	173	mtool = self.portal_membership
	174	member = mtool.getAuthenticatedMember()
	175	member_id = str(member)
	176	info = {}
	177	is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
	178	is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
	179	is_staff = info['is_staff'] = not is_anonymous and not is_student
	180	roles = member.getRolesInContext(context)
[3452]	181	info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
	182	"SectionManager" in roles or
	183	"Manager" in roles)
	184	info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
[3858]	185	#is_allowed = info['is_allowed'] = not is_anonymous
[2975]	186	requested_id = context.getStudentId()
	187	student_id = None
[3858]	188	if not is_anonymous and requested_id:
	189	if (is_student and member_id == requested_id) or is_staff:
	190	student_id = requested_id
	191	else:
[3025]	192	logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
[2975]	193	info['student_id'] = student_id
	194	return info
	195	###)
	196
[1818]	197	security.declareProtected(ModifyPortalContent,'openLog') ###(
[1716]	198	def openLog(self,name):
	199	"""open a log file"""
[5564]	200	path = "%s/log/%s" % (i_home,name)
	201	#import pdb;pdb.set_trace()
	202	if os.path.exists(path):
	203	log = open(path,"rb")
	204	lines = log.readlines()
	205	return lines
	206	else:
	207	return None
[2259]	208	###)
[5564]	209
	210	security.declareProtected(ModifyPortalContent,'grepLog') ###(
	211	def grepLog(self,name,searchstring):
	212	"""open a log file"""
	213	path = "%s/log/%s" % (i_home,name)
	214	#import pdb;pdb.set_trace()
	215	if os.path.exists(path):
	216	popen = os.popen('grep %s %s' % (searchstring,path))
	217	lines = popen.readlines()
	218	popen.close()
	219	return lines
	220	else:
	221	return None
	222	###)
[1716]	223
[2259]	224	security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
	225	def bypassQueueCatalog(self,enable=True):
	226	"""bypass the QueueCatalog by setting all indexes to process imediate,
	227	if enable is True (default) the old settings are restored
	228	"""
[1818]	229
[2259]	230	###)
	231
[2094]	232	security.declareProtected(ModifyPortalContent,'measureOaT') ###(
	233	def measureOaT(self,method="a",probe="1000",nr_pts="1"):
	234	"""measure Object access Time"""
	235	import random
	236	if hasattr(self,'portal_catalog_real'):
	237	aq_portal = self.portal_catalog_real.evalAdvancedQuery
	238	else:
	239	aq_portal = self.portal_catalog.evalAdvancedQuery
	240	nr_pts = int(nr_pts)
	241	probe = int(probe)
	242	intervall = probe/10
	243	objects = ("application","clearance","personal")
	244	portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
	245	#i = random.randrange(num_objects)
	246	count = 0
	247	found = 0
	248	not_found = 0
	249	t_found = 0
	250	t_not_found = 0
	251	time_found = time_not_found = 0.0
	252	t_time_found = t_time_not_found = 0.0
	253	accessed = []
	254	t_min = 1000
	255	t_max = 0
	256	#import pdb;pdb.set_trace()
	257	students = self.portal_catalog(portal_type="Student")
	258	num_students = len(students)
	259	if method == "d":
	260	query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
	261	res = aq_portal(query)
	262	brains = {}
	263	for r in res:
	264	sid = r.relative_path.split('/')[-2]
	265	if brains.has_key(sid):
	266	brains[sid][r.portal_type] = r
	267	else:
	268	brains[sid] = {r.portal_type : r}
	269	brains_list = brains.keys()
	270	num_objects = len(brains_list)
	271	else:
	272	num_objects = num_students
	273	print "="*40
	274	print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
	275	probe,
	276	nr_pts,
	277	num_objects)
	278	print "nr found/not time found/not min/max"
	279	elapse = time.time()
	280	i_elapse = time.time()
	281	c_elapse = time.clock()
	282	for c in range(1,probe + 1):
	283	i = random.randrange(num_objects)
	284	if method in ('a','b','c'):
	285	student_brain = students[i]
	286	elif method == "d":
	287	#import pdb;pdb.set_trace()
	288	student_brain = brains[brains_list[i]]
	289	if method == "c":
	290	query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
	291	res = aq_portal(query)
	292	this_portal_types = [r.portal_type for r in res]
	293	for i in range(nr_pts):
	294	oid = objects[i]
	295	if method == "a":
	296	try:
	297	student_path = student_brain.getPath()
	298	path = "%s/%s" % (student_path,oid)
	299	doc = self.unrestrictedTraverse(path).getContent()
	300	found += 1
	301	i_time = time.time() - i_elapse
	302	time_found += i_time
	303	except:
	304	not_found += 1
	305	i_time = time.time() - i_elapse
	306	time_not_found += i_time
	307	pass
	308	elif method == "b":
	309	try:
	310	student_object = student_brain.getObject()
	311	doc = getattr(student_object,oid).getContent()
	312	found += 1
	313	i_time = time.time() - i_elapse
	314	time_found += i_time
	315	except:
	316	i_time = time.time() - i_elapse
	317	time_not_found += i_time
	318	not_found += 1
	319	pass
	320	elif method == "c":
	321	if portal_types[i] in this_portal_types:
	322	found += 1
	323	doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
	324	i_time = time.time() - i_elapse
	325	time_found += i_time
	326	else:
	327	not_found += 1
	328	i_time = time.time() - i_elapse
	329	time_not_found += i_time
	330	elif method == "d":
	331	if student_brain.has_key(portal_types[i]):
	332	found += 1
	333	doc = student_brain[portal_types[i]].getObject().getContent()
	334	i_time = time.time() - i_elapse
	335	time_found += i_time
	336	else:
	337	not_found += 1
	338	i_time = time.time() - i_elapse
	339	time_not_found += i_time
	340	i_elapse = time.time()
	341	if c and (c % intervall == 0):
	342	#i_time = time.time() - i_elapse
	343	t_per = 0.0
	344	if found:
	345	t_per = time_found/found
	346	if t_per > t_max:
	347	t_max = t_per
	348	if t_per > 0.0 and t_per < t_min:
[2119]	349	t_min = t_per
[2094]	350	itf = 0.0
	351	if found:
	352	itf = time_found/found
	353	itnf = 0.0
	354	if not_found :
	355	itnf = time_not_found / not_found
[2119]	356	interval_time = time_found + time_not_found
	357	s = "%(c)d: %(found)d/%(not_found)d " % vars()
[2094]	358	s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	359	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	360	print s
	361	t_found += found
	362	t_not_found += not_found
	363	t_time_found += time_found
	364	t_time_not_found += time_not_found
	365	time_found = time_not_found = 0.0
	366	found = not_found = 0
	367	# t_found += found
	368	# t_not_found += not_found
	369	elapse = time.time() - elapse
	370	itf = 0.0
	371	if t_found:
	372	itf = t_time_found/t_found
	373	itnf = 0.0
	374	if t_not_found:
	375	itnf = t_time_not_found / t_not_found
	376	#c_elapse = time.clock() - c_elapse
[2119]	377	s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
[2094]	378	s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
	379	s += "%(t_min)6.4f/%(t_max)6.4f" % vars()
	380	print "-"*40
	381	print s
	382	rel_found = float(t_found)/probe
	383	rel_not_found = float(t_not_found)/probe
	384	estimated_total_time = num_objects(rel_founditf + rel_not_found*itnf)
	385	print estimated_total_time
	386	###)
	387
[1818]	388	security.declareProtected(ModifyPortalContent,'writeLog') ###(
[1716]	389	def writeLog(self,logfile,s):
	390	"""write to the log file"""
	391	logfile.write(s)
[3473]	392	###)
[1716]	393
[1151]	394	def generateStudentId(self,letter): ###(
	395	import random
[3812]	396	logger = logging.getLogger('WAeUPTool.generateStudentId')
[1151]	397	r = random
[1194]	398	if letter == '?':
[2695]	399	letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
[1151]	400	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1720]	401	students = self.portal_url.getPortalObject().campus.students
[4302]	402	while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid) or self.removed_student_ids(id = sid):
[5019]	403	logger.info('skipped used id %s' % sid)
[1720]	404	sid = "%c%d" % (letter,r.randint(99999,1000000))
[1151]	405	return sid
[1460]	406	###)
[1415]	407
	408	def generatePassword(self,s=None): ###(
	409	import random
	410	r = random
	411	##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
	412	if s is None:
	413	s = 'abcdefghklmnpqrstuvwxy23456789'
	414	pw = ''
	415	while len(pw) < 6:
	416	pw += r.choice(s)
	417	return pw
[1151]	418	###)
[828]	419
[2645]	420	security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
	421	def dumpSchoolfeePayments(self):
	422	"dump paid schoolfees"
[2666]	423	mtool = self.portal_membership
	424	member = mtool.getAuthenticatedMember()
[2645]	425	logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
	426	aq_student = self.students_catalog.evalAdvancedQuery
	427	query = In('review_state',('schoolfee_paid',
	428	'courses_registered',
	429	'courses_validated',
	430	))
	431	res = aq_student(query)
[2666]	432	#import pdb;pdb.set_trace()
[2645]	433	l = []
	434	logger.info("start for %d" % len(res))
	435	count = 1
	436	log_after = 100
	437	for student in res:
[2656]	438	if not count % log_after:
[2645]	439	logger.info("processed %d total %d" % (log_after,count))
	440	count += 1
	441	fee_dict =self.getSchoolFee(student)
	442	fulltime = student.mode.endswith('_ft')
	443	d = {}
	444	d['student_id'] = student.id
[2666]	445	d['name'] = student.name
	446	d['amount'] = fee_dict.get(new_returning)
[2645]	447	l += d,
[2666]	448	csv_name = self.dumpListToCSV(l,'payments')
	449	logger.info('%s dumped payments to %s' % (member,export_file))
[2645]	450	###)
	451
[2666]	452	security.declarePublic('dumpListToCSV') ###(
	453	def dumpListToCSV(self,l,filename,fields=None):
	454	"""dump a list of dicts to a CSV file"""
	455	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	456	export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
	457	if fields is None:
	458	fields = l[0].keys()
	459	headline = ','.join(fields)
	460	out = open(export_file,"wb")
	461	out.write(headline +'\n')
	462	out.close()
	463	out = open(export_file,"a")
	464	csv_writer = csv.DictWriter(out,fields,)
	465	csv_writer.writerows(l)
	466	return export_file
	467	###)
	468
[2643]	469	security.declareProtected(ManagePortal, 'listMembers') ###(
	470	def listMembers(self):
	471	"list all members"
	472	mtool = self.portal_membership
	473	member = mtool.getAuthenticatedMember()
	474	logger = logging.getLogger('WAeUPTool.listMembers')
	475	if str(member) not in ('admin','joachim'):
	476	logger.info('%s tried to list members' % (member))
	477	return None
	478	members = self.portal_directories.members
	479	all = members.listEntryIdsAndTitles()
	480	l = []
	481	for user_id,name in all:
	482	d = {}
	483	d['user_id'] = user_id
	484	d['name'] = name
	485	d['pw'] = getattr(getattr(members,user_id),'password')
[2645]	486	d['email'] = getattr(getattr(members,user_id),'email')
	487	d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
	488	d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
[2643]	489	l += d,
	490	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	491	export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
[3272]	492	logger.info('%s dumped member list to %s' % (member,export_file))
[2643]	493	fields = l[0].keys()
	494	headline = ','.join(fields)
	495	out = open(export_file,"wb")
	496	out.write(headline +'\n')
	497	out.close()
	498	out = open(export_file,"a")
	499	csv_writer = csv.DictWriter(out,fields,)
	500	csv_writer.writerows(l)
	501	###)
[3300]	502
[3272]	503	security.declareProtected(ManagePortal, 'listStudents') ###(
	504	def listStudents(self):
	505	"list all students"
	506	mtool = self.portal_membership
	507	member = mtool.getAuthenticatedMember()
	508	logger = logging.getLogger('WAeUPTool.listStudents')
	509	if str(member) not in ('admin','joachim'):
	510	logger.info('%s tried to list students' % (member))
	511	return None
	512	students = self.portal_directories.students
	513	all = students.listEntryIdsAndTitles()
	514	l = []
	515	for user_id,name in all:
	516	d = {}
	517	d['user_id'] = user_id
	518	d['name'] = name
	519	d['pw'] = getattr(getattr(students,user_id),'password')
	520	d['email'] = getattr(getattr(students,user_id),'email')
	521	d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
	522	d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
	523	l += d,
	524	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	525	export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
	526	logger.info('%s dumped student list to %s' % (member,export_file))
	527	fields = l[0].keys()
	528	headline = ','.join(fields)
	529	out = open(export_file,"wb")
	530	out.write(headline +'\n')
	531	out.close()
	532	out = open(export_file,"a")
	533	csv_writer = csv.DictWriter(out,fields,)
	534	csv_writer.writerows(l)
[3300]	535	###)
[2656]	536
[1890]	537	security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
	538	def removeDeletedDocIds(self, max=1000):
	539	"""
	540	remove deleted docids from repository commit after max
	541	"""
	542	logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
	543	repository = getToolByName(self, 'portal_repository')
	544	pxtool = getToolByName(self, 'portal_proxies')
[2632]	545	logger.info('start')
[1890]	546	pxtool_infos = pxtool.getRevisionsUsed()
[2632]	547	logger.info('found %d used revisions ' % (len(pxtool_infos)))
[1895]	548
[1890]	549	nb_revs = 0
	550	docids_d = {} # all docids
	551	unused_docids_d = {} # all docids that are unused
	552	ids_unused_revs_docids = [] # ids for revs of unused docids
[2632]	553	unused_ids = [] # ids for unused revs
[1890]	554	total = 0
[1895]	555	idlist = repository.objectIds()
[2632]	556	to_delete = 0
	557	found = False
[1895]	558	for id in idlist:
[1890]	559	docid, rev = repository._splitId(id)
	560	if docid is None:
	561	logger.info("invalid doc_id %s" % docid)
	562	continue
	563	nb_revs += 1
	564	if not pxtool_infos.has_key(docid):
[2632]	565	found = True
	566	to_delete += 1
	567	unused_ids.append(id)
[1890]	568	elif not pxtool_infos[docid].has_key(rev):
[2632]	569	found = True
	570	to_delete += 1
	571	unused_ids.append(id)
	572	if found and not to_delete % max:
	573	found = False
[1890]	574	#import pdb;pdb.set_trace()
[2632]	575	repository.manage_delObjects(unused_ids)
[1890]	576	transaction.commit()
[2632]	577	logger.info('removed %d total %d unused docids ' % (max,to_delete))
	578	else:
	579	if unused_ids:
	580	repository.manage_delObjects(unused_ids)
	581	transaction.commit()
	582	logger.info('finished removing %d unused docids ' % (to_delete))
[3473]	583	###)
[1890]	584
[3858]	585	security.declarePublic('getCredential') ###(
[1250]	586	def getCredential(self,student_id):
	587	student_entry = getattr(self.portal_directories.students,student_id,None)
[3858]	588	#import pdb;pdb.set_trace()
	589	#if not self.isStaff():
	590	# mtool = self.portal_membership
	591	# member = mtool.getAuthenticatedMember()
	592	# logger = logging.getLogger('WAeUPTool.getCredential')
	593	# logger.info('%s tried to access password of %s' % (member,student_id))
	594	# return None
[1250]	595	if student_entry is None:
[4022]	596	return "not set"
[1263]	597	return getattr(student_entry,"password","not set")
[1261]	598	###)
[5656]	599
	600	security.declarePublic('getCredentialFromAuthHeader') ###(
	601	def getCredentialFromAuthHeader(self,request):
	602	return base64.b64decode(request._auth.replace('Basic ',''))
	603	###)
[1250]	604
[2747]	605	security.declarePublic('checkPassword') ###(
[1467]	606	def checkPassword(self,student_id,password):
[1460]	607	student_entry = getattr(self.portal_directories.students,student_id,None)
	608	if student_entry is None:
	609	return False
	610	return getattr(student_entry,"password","not set") == password
[2747]	611	###)
[2712]	612
[2747]	613	security.declarePublic('checkGenericPassword') ###(
[2710]	614	def checkGenericPassword(self,member_id):
	615	member_entry = getattr(self.portal_directories.members,member_id,None)
	616	if member_entry is None:
	617	return False
[2712]	618	ltool = getToolByName(self, 'portal_layouts')
[2710]	619	unsecure_words = ltool._getOb('members')['w__password'].check_words
[2712]	620	password = getattr(member_entry,"password","not set")
	621	is_unsecure = password in unsecure_words
	622	if is_unsecure:
[2713]	623	logger = logging.getLogger('WAeUPTool.checkGenericPassword')
[2712]	624	logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
	625	return is_unsecure
[2747]	626	###)
[1460]	627
[4027]	628	security.declarePublic('editPassword') ###(
[1467]	629	def editPassword(self,student_id,password):
	630	"edit a student password"
	631	student_entry = getattr(self.portal_directories.students,student_id,None)
	632	if student_entry is None:
[1571]	633	return
[1467]	634	setattr(student_entry,'password',password)
	635	###)
	636
[2555]	637	security.declareProtected(ModifyPortalContent,'doCommit') ###(
[1401]	638	def doCommit(self,logger=None):
	639	"commit some transactions"
	640	transaction.commit()
	641	###)
	642
[1285]	643	security.declarePublic('loadStudentFoto') ###(
[1813]	644	def loadStudentFoto(self,student,filename,folder):
[1285]	645	"return a student passport picture"
[2413]	646	#import pdb;pdb.set_trace()
	647	picture ="%s/import/%s/%s" % (i_home,folder,filename)
	648	student_id = student.getId()
[2675]	649	images_dir = getImagesDir(student_id)
[2413]	650	if not os.path.exists(images_dir):
	651	os.mkdir(images_dir)
	652	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	653	for extension in ('.jpg','.JPG'):
	654	fullname = "%(picture)s%(extension)s" % vars()
	655	if os.path.exists(fullname):
	656	copy2(fullname,image_name)
	657	return "successfully copied passport picture"
[2467]	658	return "passport picture not found: %s.jpg or .JPG" % picture
[2413]	659	###)
	660
[5655]	661	security.declarePublic('loadTransferStudentFoto') ###(
	662	def loadTransferStudentFoto(self,student_old,student_new):
	663	"return a student passport picture"
	664	images_dir_new = getImagesDir(student_new)
	665	images_dir_old = getImagesDir(student_old)
	666	if not os.path.exists(images_dir_new):
	667	os.mkdir(images_dir_new)
	668	image_name_new = os.path.join(images_dir_new,"passport_%(student_new)s.jpg" % vars())
	669	image_name_old = os.path.join(images_dir_old,"passport_%(student_old)s" % vars())
	670	#import pdb;pdb.set_trace()
	671	for extension in ('.jpg','.JPG'):
	672	fullname = "%(image_name_old)s%(extension)s" % vars()
	673	if os.path.exists(fullname):
	674	copy2(fullname,image_name_new)
	675	return "successfully copied passport picture"
	676	return "passport picture not found: %s.jpg or .JPG" % image_name_old
	677	###)
[1285]	678
[1170]	679	security.declareProtected(ModifyPortalContent,'createOne') ###(
[1194]	680	def createOne(self,students_folder,student_brain,letter,commit=False):
	681	sid = self.waeup_tool.generateStudentId(letter)
[1170]	682	students_folder.invokeFactory('Student', sid)
	683	student = getattr(students_folder,sid)
	684	self.portal_workflow.doActionFor(student,'return')
	685	student.manage_setLocalRoles(sid, ['Owner',])
	686	matric_no = student_brain.matric_no
	687	jamb_reg_no = student_brain.Entryregno
	688	self.students_catalog.addRecord(id = sid,
	689	matric_no = matric_no,
	690	jamb_reg_no = jamb_reg_no,
	691	sex = student_brain.Sex == "F",
	692	name = "%s %s %s" % (student_brain.Firstname,
	693	student_brain.Middlename,
	694	student_brain.Lastname)
	695	)
	696	if commit:
	697	transaction.commit()
	698	return sid,jamb_reg_no
	699	###)
	700
[3840]	701	# Functions without docstring cannot be called from outside (via the URL),
	702	# thus we can declare addStudent public.
[3838]	703	security.declarePublic('addStudent') ###(
[1752]	704	def addStudent(self,dict):
[1415]	705	students_folder = self.portal_url.getPortalObject().campus.students
	706	sid = self.waeup_tool.generateStudentId('?')
	707	students_folder.invokeFactory('Student', sid)
	708	student_obj = getattr(students_folder,sid)
[3481]	709	f2t = StudentImport.field2types_student
[1772]	710	#from pdb import set_trace; set_trace()
	711	d = {}
[2297]	712	#d['jamb_sex'] = 'M'
	713	#if dict.get('sex'):
	714	# d['jamb_sex'] = 'F'
[1904]	715
[1898]	716	entry_session = dict.get('entry_session')
[2454]	717	if entry_session == self.getSessionId()[0]:
[1898]	718	wfaction = 'admit'
[1991]	719	wft = 'wf_transition_admit'
[1900]	720	password = None
[1904]	721	else:
[1900]	722	wfaction = 'return'
[1991]	723	wft = 'wf_transition_return'
[1900]	724	password = self.generatePassword()
[1904]	725	self.makeStudentMember(sid,password)
	726
[1728]	727	for pt in f2t.keys():
[5023]	728	if pt == 'StudentAccommodation':
	729	continue
[1728]	730	student_obj.invokeFactory(pt,f2t[pt]['id'])
	731	sub_obj = getattr(student_obj,f2t[pt]['id'])
	732	sub_doc = sub_obj.getContent()
[1898]	733	#self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
[3481]	734	#d['Title'] = f2t[pt]['title']
[1728]	735	for field in f2t[pt]['fields']:
[1749]	736	d[field] = dict.get(field,'')
[1728]	737	sub_doc.edit(mapping = d)
[1898]	738	new_state = f2t[pt][wft]
[1772]	739	if new_state != "remain":
	740	self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
[1415]	741	self.portal_workflow.doActionFor(student_obj,wfaction)
	742	student_obj.manage_setLocalRoles(sid, ['Owner',])
	743	return sid,password
	744	###)
	745
[1151]	746	security.declarePublic('getCertificateBrain') ###(
	747	def getCertificateBrain(self,cert_id):
	748	"do it"
[1849]	749	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	750	{'portal_type':"Certificate",
	751	'id': cert_id})
	752	if res:
	753	return res[0]
	754	return None
	755	###)
[1160]	756
[1756]	757	security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
	758	def get_csv_filenames(self):
	759	"do it"
[1759]	760	files = [file for file in os.listdir("%s/import/" % (i_home))
[3143]	761	if file.endswith('.csv') and (file.find('imported') == -1 and
	762	file.find('pending') == -1)]
[1756]	763	return files
	764	###)
	765
[1151]	766	security.declarePublic('findStudentByMatricelNo') ###(
	767	def findStudentByMatricelNo(self,matric_no):
	768	"do it"
[1849]	769	res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]	770	{'portal_type':"StudentClearance",
	771	'SearchableText': matric_no})
	772	if res:
	773	return res[0]
	774	return None
	775	###)
	776
[3817]	777
	778	security.declarePublic('getOfficerName') ###(
	779	def getOfficerName(self,mid):
	780	"""get the real name of a member"""
	781	membership = self.portal_membership
	782	member_record = membership.getMemberById(mid)
[5210]	783	if member_record:
	784	firstname = member_record.getProperty('givenName','')
	785	lastname = member_record.getProperty('sn','')
	786	return firstname + ' ' + lastname.upper()
	787	else:
	788	return mid
[3817]	789	###)
	790
	791
[2557]	792	security.declarePublic('makeStudentMember') ###(
[1151]	793	def makeStudentMember(self,sid,password='uNsEt'):
	794	"""make the student a member"""
	795	membership = self.portal_membership
	796	membership.addMember(sid,
	797	password ,
	798	roles=('Member',
	799	'Student',
	800	),
	801	domains='',
	802	properties = {'memberareaCreationFlag': False,
	803	'homeless': True},)
	804	member = membership.getMemberById(sid)
	805	self.portal_registration.afterAdd(member, sid, password, None)
[1261]	806	#self.manage_setLocalRoles(sid, ['Owner',])
[1151]	807	###)
[1160]	808
[2560]	809	security.declareProtected(View,'makeStudentData') ###(
[1158]	810	def makeStudentData(self,student_id,email=None,phone_nr=None):
[1151]	811	"create Datastructure for a returning Student"
[1406]	812	#import pdb;pdb.set_trace()
[1571]	813	logger = logging.getLogger('WAeUPTool.makeStudentData')
[1151]	814	students_folder = self.portal_url.getPortalObject().campus.students
[1794]	815	#res = self.students_catalog(id=student_id)
	816	#if res:
	817	# st = res[0]
	818	#res = self.returning_import(matric_no = st.matric_no)
[1798]	819	res = self.returning_import(id = student_id)
[1151]	820	if res:
[1160]	821	student = res[0]
[1794]	822	else:
[1816]	823	logger.info('Id %s not found in returning_import' % student_id)
[1794]	824	return
[1580]	825	logger.info('%s creates data structure' % student_id)
[1794]	826	s_results = self.results_import(matric_no = student.matric_no)
[1816]	827	if s_results:
	828	lnr = self.getLevelFromResultsCosCode(s_results)
	829	level = "%d00" % lnr
	830	verdict,eligible = self.getVerdict(s_results[0].Verdict)
[2454]	831	#if eligible:
	832	# level = "%d00" % (lnr + 1)
[1816]	833	else:
	834	logger.info('matric_no %s not found in results_import' % student.matric_no)
[2656]	835	level = ''
	836	verdict = ''
[1171]	837	#student should not be allowed to perform this transition
[1174]	838	#wftool = self.portal_workflow
	839	#wftool.doActionFor(student,'return')
[1151]	840	certcode_org = student.Coursemajorcode
	841	certcode = makeCertificateCode(certcode_org)
	842	certificate_brain = self.getCertificateBrain(certcode)
	843	if not certificate_brain:
	844	em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
	845	logger.info(em)
	846	matric_no = student.matric_no
	847	sid = student_id
	848	student_obj = getattr(students_folder,sid)
[3114]	849	if not getattr(student_obj,'application'):
	850	student_obj.invokeFactory('StudentApplication','application')
[1151]	851	application = student_obj.application
[1169]	852	self.portal_workflow.doActionFor(application,'open',dest_container=application)
[1151]	853	da = {'Title': 'Application Data'}
	854	student_obj.invokeFactory('StudentPersonal','personal')
	855	da['jamb_reg_no'] = student.Entryregno
[1462]	856	em = self.getEntryMode(student.Entryregno)
[1401]	857	da['entry_mode'] = em
[1151]	858	personal = student_obj.personal
	859	self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
	860	dp = {'Title': 'Personal Data'}
	861	student_obj.invokeFactory('StudentClearance','clearance')
	862	clearance = student_obj.clearance
[1169]	863	self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[1151]	864	dc = {'Title': 'Clearance/Eligibility Record'}
	865	dc['matric_no'] = matric_no
	866	state = student.State
	867	lga = student.LGA
	868	if state and lga:
	869	lga = state + ' / ' + lga
	870	else:
	871	lga = "None"
[1174]	872	da['jamb_lga'] = dc['lga'] = lga
[1173]	873	da['app_email'] = dp['email'] = email
	874	da['app_mobile'] = dp['phone'] = phone_nr
[1411]	875	dp['firstname'] = student.Firstname
	876	dp['middlename'] = student.Middlename
	877	dp['lastname'] = student.Lastname
	878	da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
[1174]	879	da['jamb_sex'] = student.Sex
[1151]	880	dp['sex'] = student.Sex == 'F'
	881	dp['perm_address'] = student.Permanent_Address
	882	application.getContent().edit(mapping=da)
[1169]	883	self.portal_workflow.doActionFor(application,'close',dest_container=application)
[1151]	884	personal.getContent().edit(mapping=dp)
	885	clearance.getContent().edit(mapping=dc)
[1169]	886	self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[1151]	887	#
	888	# Study Course
	889	#
	890	student_obj.invokeFactory('StudentStudyCourse','study_course')
	891	studycourse = student_obj.study_course
	892	self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
	893	dsc = {}
	894	dsc['study_course'] = certcode
[1401]	895	dsc['current_level'] = level
	896	dsc['current_verdict'] = verdict
[3780]	897	dsc['current_mode'] = em #no longer used
[2466]	898	dsc['current_session'] = '05'
[1151]	899	studycourse.getContent().edit(mapping=dsc)
	900	#
	901	# Level
	902	#
[3473]	903	# l = getattr(studycourse,level,None)
	904	# if l is None:
	905	# studycourse.invokeFactory('StudentStudyLevel', level)
	906	# l = getattr(studycourse, level)
	907	# self.portal_workflow.doActionFor(l,'open',dest_container=l)
	908	# l.getContent().edit(mapping={'Title': "Level %s" % level})
	909	###)
[1160]	910
[3349]	911	def init_timing(self): ###(
	912	if self.with_timing:
	913	if not hasattr(self,'_v_step_times'):
	914	self._v_timer_count = 0
	915	self._v_total = 0
	916	self._v_step_times = {}
	917	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	918	self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
	919	self.timer_step = 0
	920	self.total_time = 0
	921	self.elapse = time.time()
	922	self.i_elapse = time.time()
	923	self.c_elapse = time.clock()
[3473]	924	###)
[3349]	925
	926	def do_timing(self): ###(
	927	if self.with_timing:
	928	try:
	929	raise 'dummy'
	930	except:
	931	frame = sys.exc_traceback.tb_frame.f_back
	932	locals = frame.f_locals
	933	globals = frame.f_globals
	934	functionname = frame.f_code.co_name
	935	filename = os.path.basename(frame.f_code.co_filename)
	936	lineno = frame.f_lineno
	937	mod_line = "%(functionname)s:%(lineno)s" % vars()
	938	i_time = time.time() - self.i_elapse
	939	td = {}
	940	if self._v_step_times.has_key(mod_line):
	941	a_time = self._v_step_times[mod_line]['a_time'] + i_time
	942	td['a_time'] = a_time
	943	else:
	944	td['a_time'] = i_time
	945	td['i_time'] = i_time
	946	self._v_step_times[mod_line] = td
	947	self.i_time = i_time
	948	self.total_time += i_time
	949	self.timer_step +=1
	950	self.i_elapse = time.time()
[3473]	951	###)
[3349]	952
	953	security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
	954	def print_timing(self):
	955	if self.with_timing:
	956	l = []
	957	timer_count = self._v_timer_count + 1
	958	mod_lines = self._v_step_times.keys()
	959	mod_lines.sort(cmp,reverse=0)
	960	for mod_line in mod_lines:
	961	td = self._v_step_times[mod_line]
	962	i_time = td['i_time']
	963	a_time = td['a_time']/(self._v_timer_count + 1)
	964	l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
	965	total_time = self.total_time
	966	total_avarage = self._v_total / timer_count
	967	l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
	968	print "\r\n".join(l)
	969	out = open(self._v_timer_file,'a')
	970	out.write("\r\n".join(l))
	971	out.close()
[3473]	972	###)
[3349]	973
	974	security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
	975	def get_timing_data(self):
	976	if self.with_timing:
	977	timer_count = self._v_timer_count + 1
	978	results = {}
	979	for k,d in self._v_step_times.items():
	980	dd = {}
	981	dd['a_time'] = d['a_time']/timer_count
	982	dd['i_time'] = d['i_time']
	983	dd['count'] = timer_count
	984	results[k] = dd
	985	dd = {}
	986	dd['a_time'] = self._v_total / timer_count
	987	dd['i_time'] = self.total_time
	988	dd['count'] = timer_count
	989	results["total"] = dd
	990	return results
	991	###)
	992
[2556]	993	security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
[3349]	994	def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
[2514]	995	"create Datastructure for an admitted Student"
	996	#import pdb;pdb.set_trace()
[3349]	997	logger = logging.getLogger('WAeUPTool.admitOneStudent')
	998	self.with_timing = with_timing
[3681]	999
[4539]	1000	try:
[5188]	1001	if brain.screening_type in ('dp','pt','ct','cest','sandwich','sandwich2008') and brain.serial and brain.entry_session:
[4539]	1002	if brain.course1:
	1003	reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
	1004	else:
	1005	reg_no = "%s%s/%s" % (brain.course_admitted[:3],brain.serial,brain.entry_session)
[5511]	1006	elif brain.screening_type in ('pume2','pde2'):
[5373]	1007	reg_no = brain.jamb_reg_no.upper()
[4532]	1008	else:
[4539]	1009	reg_no = brain.reg_no
	1010	except:
	1011	logger.info('applicant record %s has errors' % (brain.reg_no))
	1012	return
[3705]	1013
[3681]	1014	#ignore argument entry_session
	1015	if not brain.entry_session:
	1016	logger.info('no entry_session for %s provided' % (reg_no))
	1017	return
[3961]	1018	else:
	1019	es = brain.entry_session
	1020	if len(es) == 1:
	1021	es = '0%c' % es
[3681]	1022
[2727]	1023	if not hasattr(self,"_v_certificates"):
[3339]	1024	self._v_certificates = self.getCertificatesDict()
[2514]	1025	students_folder = self.portal_url.getPortalObject().campus.students
[3705]	1026
[3264]	1027	res = self.students_catalog(jamb_reg_no = reg_no)
[3198]	1028	if res:
[3800]	1029	logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
[3198]	1030	return
[2514]	1031	if brain.status != "admitted":
[3264]	1032	logger.info('status of %s is %s' % (reg_no,brain.status))
[2540]	1033	return
[2514]	1034	pin_parts = brain.pin.split('-')
[2755]	1035	if pin_parts and len(pin_parts) != 3:
[3264]	1036	logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
[2540]	1037	return
[3764]	1038	if not brain.course_admitted:
	1039	logger.info('no course_admitted provided for %s' % (reg_no))
	1040	return
[2727]	1041	if brain.course_admitted not in self._v_certificates:
[3264]	1042	logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
[2727]	1043	return
[2959]	1044	if brain.sex not in (True,False):
[3264]	1045	logger.info('sex of %s not available' % (reg_no))
[3002]	1046	return
[3349]	1047	self.init_timing()
[2540]	1048	student_id = self.generateStudentId('?')
[2514]	1049	students_folder.invokeFactory('Student', student_id)
	1050	student_object = getattr(students_folder,student_id)
[3349]	1051	self.do_timing()
[2576]	1052	if pin_password:
	1053	password = pin_parts[2]
	1054	self.makeStudentMember(student_id,password = password)
[2540]	1055	student_object.manage_setLocalRoles(student_id, ['Owner',])
[3349]	1056	self.do_timing()
[3264]	1057	#logger.info("creating %s reg_no %s" % (student_id,reg_no))
[2514]	1058	#
	1059	# application
	1060	#
[2540]	1061	student_object.invokeFactory('StudentApplication','application')
	1062	application = student_object.application
[2575]	1063	#self.portal_workflow.doActionFor(application,'open',dest_container=application)
[3350]	1064	#self.do_timing()
[2514]	1065	da = {'Title': 'Application Data'}
[3264]	1066	da['jamb_reg_no'] = reg_no
[2576]	1067
[2540]	1068	sex = 'M'
	1069	if brain.sex:
	1070	sex = 'F'
	1071	da['jamb_sex'] = sex
[2670]	1072	da['jamb_age'] = brain.jamb_age
[3341]	1073	da['app_reg_pin'] = brain.pin
[2670]	1074	da['jamb_lga'] = brain.jamb_lga
	1075	da['jamb_state'] = brain.jamb_state
[2575]	1076	da['jamb_score'] = brain.aggregate
[2514]	1077	da['app_email'] = brain.email
[2540]	1078	da['app_mobile'] = brain.phone
[3339]	1079
[3775]	1080	# entry_mode cannot be retrieved from the certtificate
	1081	# because ug_ft has two different entry modes
	1082
[3780]	1083	if brain.entry_mode: # does not happen because import_application
	1084	da['entry_mode'] = brain.entry_mode # schema has no field entry_mode
[3775]	1085	elif brain.screening_type == 'pume':
[3776]	1086	da['entry_mode'] = 'ume_ft'
[3775]	1087	elif brain.screening_type == 'pde':
[3776]	1088	da['entry_mode'] = 'de_ft'
[3775]	1089	else:
	1090	da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
	1091
	1092	#elif brain.screening_type == 'pce':
	1093	# da['entry_mode'] = 'pce'
	1094	#elif brain.screening_type == 'prence':
	1095	# da['entry_mode'] = 'prence'
	1096	#else:
	1097	# da['entry_mode'] = ''
[3776]	1098
[3339]	1099
[3681]	1100	#da['entry_session'] = entry_session
[3961]	1101	da['entry_session'] = es
[2575]	1102	da['jamb_lastname'] = brain.lastname
	1103	da['jamb_middlename'] = brain.middlenames # different field names!
[2670]	1104	da['jamb_firstname'] = brain.firstname
[2575]	1105	da['screening_application_date'] = brain.application_date
	1106	da['date_of_birth'] = brain.date_of_birth
	1107	da['jamb_first_cos'] = brain.course1
	1108	da['jamb_second_cos'] = brain.course2
	1109	da['course3'] = brain.course3
	1110	da['screening_type'] = brain.screening_type
	1111	da['screening_score'] = brain.screening_score
	1112	da['screening_date'] = brain.screening_date
	1113	da['hq_type'] = brain.hq_type
	1114	da['hq_grade'] = brain.hq_grade
	1115	da['aos'] = brain.aos
[2576]	1116
[2514]	1117	application.getContent().edit(mapping=da)
[3349]	1118	self.do_timing()
[2575]	1119	#self.portal_workflow.doActionFor(application,'close',dest_container=application)
[2514]	1120	#
	1121	# personal
	1122	#
[2540]	1123	student_object.invokeFactory('StudentPersonal','personal')
	1124	personal = student_object.personal
[2575]	1125	#self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
[3350]	1126	#self.do_timing()
[2514]	1127	dp = {'Title': 'Personal Data'}
	1128	dp['sex'] = brain.sex
	1129	dp['email'] = brain.email
[2540]	1130	dp['phone'] = brain.phone
[2514]	1131	dp['lastname'] = brain.lastname
[2575]	1132	dp['middlename'] = brain.middlenames # different field names!
[2576]	1133	dp['firstname'] = brain.firstname
[2514]	1134	personal.getContent().edit(mapping=dp)
[3349]	1135	self.do_timing()
[2514]	1136	#
	1137	# clearance
	1138	#
[2540]	1139	student_object.invokeFactory('StudentClearance','clearance')
	1140	clearance = student_object.clearance
[2575]	1141	#self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[2514]	1142	dc = {'Title': 'Clearance/Eligibility Record'}
	1143	dc['lga'] = brain.lga
[2670]	1144	dc['birthday'] = brain.date_of_birth
[2514]	1145	clearance.getContent().edit(mapping=dc)
[3349]	1146	self.do_timing()
[2575]	1147	#self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[2514]	1148	#
	1149	# study Course
	1150	#
[2540]	1151	student_object.invokeFactory('StudentStudyCourse','study_course')
	1152	studycourse = student_object.study_course
[2575]	1153	#self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
[3350]	1154	#self.do_timing()
[2514]	1155	dsc = {}
	1156	dsc['study_course'] = brain.course_admitted
	1157	dsc['current_verdict'] = ''
[3780]	1158	dsc['current_mode'] = da['entry_mode'] # no longer used
[2575]	1159	if da['entry_mode'].startswith('de'):
	1160	dsc['current_level'] = '200'
[3040]	1161	elif da['entry_mode'].startswith('pre'):
	1162	dsc['current_level'] = '000'
[5518]	1163	elif da['entry_mode'].startswith('pde'):
	1164	dsc['current_level'] = '500'
[2575]	1165	else:
	1166	dsc['current_level'] = '100'
[3961]	1167	dsc['current_session'] = es
[2514]	1168	studycourse.getContent().edit(mapping=dsc)
[3349]	1169	self.do_timing()
[2514]	1170	#
	1171	# payments folder
[2540]	1172	student_object.invokeFactory('PaymentsFolder','payments')
	1173	payments = getattr(student_object,'payments')
[3350]	1174	#self.do_timing()
[2514]	1175	dpay = {}
	1176	dpay['Title'] = 'Payments'
	1177	payments.getContent().edit(mapping=dpay)
[2540]	1178	self.portal_workflow.doActionFor(payments,'open')
[3349]	1179	self.do_timing()
[2580]	1180	#
	1181	# passport foto
	1182	app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
	1183	brain.screening_type,
	1184	brain.reg_no)
[2675]	1185	images_dir = getImagesDir(student_id)
	1186	#images_dir = os.path.join("%s" % images_base,student_id)
	1187	letter_dir,student_dir = os.path.split(images_dir)
	1188	if not os.path.exists(letter_dir):
	1189	os.mkdir(letter_dir)
[2580]	1190	if not os.path.exists(images_dir):
	1191	os.mkdir(images_dir)
	1192	image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
	1193	if os.path.exists(app_picture):
	1194	copy2(app_picture,image_name)
	1195	else:
	1196	logger.info('passport of %s/%s not found: %s' % (student_id,
	1197	brain.reg_no,
	1198	app_picture))
[2656]	1199
[3349]	1200	self.do_timing()
	1201	self.print_timing()
	1202	if with_timing:
	1203	self.timer_step = 0
	1204	self._v_timer_count += 1
	1205	self._v_total += self.total_time
[2514]	1206	return student_id
	1207	###)
	1208
[2556]	1209	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
[1194]	1210	def makeStudentLevel(self,student_id):
	1211	"create the StudyLevel for a returning Student"
	1212	#import pdb;pdb.set_trace()
[1571]	1213	logger = logging.getLogger('WAeUPTool.makeStudentLevel')
[1194]	1214	students_folder = self.portal_url.getPortalObject().campus.students
	1215	res = self.students_catalog(id=student_id)
	1216	if res:
	1217	st = res[0]
	1218	course = st.course
	1219	matric_no = st.matric_no
	1220	level = st.level
	1221	res = self.results_import(matric_no = matric_no)
	1222	if res:
	1223	results = res
[1571]	1224	logger.info('%s creating Level %s' % (student_id,level))
[1194]	1225	#
	1226	# Level
	1227	#
	1228	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1229	studycourse = getattr(student_obj,"study_course",None)
	1230	self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
	1231	l = getattr(studycourse,level,None)
	1232	if l is None:
	1233	studycourse.invokeFactory('StudentStudyLevel', level)
	1234	l = getattr(studycourse, level)
	1235	self.portal_workflow.doActionFor(l,'open',dest_container=l)
	1236	l.getContent().edit(mapping={'Title': "Level %s" % level})
	1237	###)
[3984]	1238
	1239
	1240	security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
	1241	def exportAllStudyLevels(self,student_id):
	1242	"export the StudyLevels for a student"
	1243	#import pdb;pdb.set_trace()
	1244	logger = logging.getLogger('WAeUPTool.exportAllStudyLevels')
	1245	students_folder = self.portal_url.getPortalObject().campus.students
	1246	res = self.students_catalog(id=student_id)
	1247	if res:
	1248	st = res[0]
	1249	course = st.course
	1250	matric_no = st.matric_no
	1251	level = st.level
	1252	student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1253	studycourse = getattr(student_obj,"study_course",None)
[4800]	1254	if studycourse:
	1255	levels = studycourse.objectIds()
	1256	else:
	1257	levels = []
[3984]	1258	stool = getToolByName(self, 'portal_schemas')
	1259	schema = stool._getOb('student_study_level')
	1260	fields = ['student_id','level']
	1261	fields.extend(schema.keys())
	1262	format = '"%(' + ')s","%('.join(fields) + ')s"'
	1263	export_file = "%s/export/study_levels_removed.csv" % (i_home)
	1264
[3989]	1265	if not os.path.exists(export_file):
	1266	file_handler = open(export_file,"a")
	1267	headline = ','.join(fields)
	1268	file_handler.write(headline +'\n')
	1269	else:
	1270	file_handler = open(export_file,"a")
	1271
[3984]	1272	for level in levels:
	1273	level_object = getattr(studycourse,level)
	1274	level_content = level_object.getContent()
	1275	d = {'student_id':student_id,'level':level}
	1276	for field in schema.keys():
	1277	d[field] = getattr(level_content,field,'')
	1278	line = format % d
	1279	file_handler.write(line +'\n')
	1280
	1281	###)
[1194]	1282
[2000]	1283	security.declarePublic('getHallInfo') ###(
	1284	def getHallInfo(self,bed):
	1285	"""return Hall Info"""
[828]	1286	info = {}
[3406]	1287	bedsplit = bed.split('_')
	1288	if len(bedsplit) == 4:
	1289	hall,block,room,letter = bed.split('_')
	1290	else:
	1291	info['maintenance_code'] = 'None'
	1292	return info
[1846]	1293	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
[828]	1294	if res and len(res) == 1:
	1295	hall_brain = res[0]
	1296	hall_doc = hall_brain.getObject().getContent()
	1297	else:
	1298	return info
	1299	info['hall_title'] = hall_brain.Title
	1300	info['maintenance_code'] = hall_doc.maintenance_code
[1846]	1301	res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
[828]	1302	batch_doc = None
[5168]	1303	# Pick the first pin batch which the right prefix. This may cause problems if old maintenance fee batches exist.
	1304	# They must be removed first.
[828]	1305	for brain in res:
	1306	if brain.id.startswith(info['maintenance_code']):
	1307	batch_doc = brain.getObject().getContent()
	1308	break
	1309	if batch_doc is None:
[2913]	1310	info['maintenance_fee'] = ''
[828]	1311	else:
	1312	info['maintenance_fee'] = batch_doc.cost
	1313	return info
[1151]	1314	###)
[828]	1315
[2396]	1316	security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
	1317	def removePictureFolder(self,student_id):
	1318	"""remove picture_folder by renaming it"""
[3807]	1319	picture_path = getImagesDir(student_id)
	1320	dest_path = os.path.join("%s" % images_base,'removed',student_id)
	1321	dest_path = dest_path + "_removed"
	1322	if os.path.exists(dest_path) or not os.path.exists(picture_path):
[2396]	1323	return False
[3807]	1324	os.rename(picture_path,dest_path)
[2396]	1325	return True
	1326	###)
	1327
	1328	security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
	1329	def restorePictureFolder(self,student_id):
	1330	"""restore picture_folder by renaming it"""
[3807]	1331	picture_path = getImagesDir(student_id)
	1332	orig_path = os.path.join("%s" % images_base,'removed',student_id)
	1333	orig_path = orig_path + "_removed"
	1334	if os.path.exists(picture_path) or not os.path.exists(orig_path):
	1335	return False
[4558]	1336	os.rename(orig_path,picture_path)
[2396]	1337	return True
	1338	###)
	1339
[2363]	1340	security.declarePublic('picturesExist') ###(
[2420]	1341	def picturesExist(self, ids,student_id=None):
[2363]	1342	"""check if pictures exist in the filesystem"""
[2420]	1343	if student_id is None:
	1344	student_id = self.getStudentId()
	1345	if student_id is None:
	1346	return False
[2675]	1347	picture_path = getImagesDir(student_id)
	1348	#picture_path = os.path.join(images_base,student_id)
[2363]	1349	if not os.path.exists(picture_path):
	1350	return False
	1351	pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1352	return set(ids).issubset(set(pictures))
	1353	###)
[3805]	1354
[3812]	1355	security.declarePublic('picturePathExists') ###(
	1356	def picturePathExists(self, student_id=None):
	1357	"""check if picture path exists in the filesystem"""
	1358	if student_id is None:
	1359	student_id = self.getStudentId()
	1360	if student_id is None:
	1361	return False
	1362	picture_path = getImagesDir(student_id)
	1363	if os.path.exists(picture_path):
	1364	return True
	1365	return False
	1366	###)
[3805]	1367
[3812]	1368
[3807]	1369	security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
	1370	def removeUnusedImageFolders(self):
[3806]	1371	"""check if an unused image folders exists in the filesystem"""
[3807]	1372	mtool = self.portal_membership
	1373	member = mtool.getAuthenticatedMember()
	1374	member_id = str(member)
	1375	logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
[3806]	1376	abc = os.listdir(images_base)
[3805]	1377	ifolders = []
[3806]	1378	for i in abc:
[3805]	1379	picture_path = os.path.join(images_base,i)
	1380	ifolders.extend(os.listdir(picture_path))
	1381	unused_ids = []
	1382	for id in ifolders:
	1383	res = self.students_catalog(id=id)
	1384	if not res:
	1385	unused_ids.append(id)
[3807]	1386	#import pdb;pdb.set_trace()
	1387	if not id.endswith('removed'):
	1388	removed = self.waeup_tool.removePictureFolder(id)
	1389	if removed:
	1390	logger.info('%s: image folder %s successfully removed' % (member_id,id))
	1391	else:
	1392	logger.info('%s: image folder %s could not be removed' % (member_id,id))
	1393	return
[3805]	1394
	1395	###)
	1396
[2364]	1397	security.declarePublic('picturesList') ###(
	1398	def picturesList(self):
	1399	"""check if pictures exist in the filesystem"""
	1400	path = 'images'
	1401	student_id = self.getStudentId()
[2677]	1402	#picture_path = os.path.join(i_home,path,student_id)
	1403	picture_path = getImagesDir(student_id)
[2364]	1404	if not os.path.exists(picture_path):
	1405	return []
	1406	return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
	1407	###)
	1408
[2114]	1409	security.declarePublic('showFsPicture') ###(
	1410	def showFsPicture(self,path):
	1411	"""return a picture from the filesystem"""
[2675]	1412	#picture_path = os.path.join(i_home,path)
	1413	picture_path = os.path.join(images_base,path)
[2335]	1414	response = self.REQUEST.RESPONSE
[2342]	1415	#import pdb;pdb.set_trace()
[2354]	1416	registry = getToolByName(self, 'mimetypes_registry')
	1417	mimetype = str(registry.lookupExtension(path.lower()) or
	1418	registry.lookupExtension('file.bin'))
[2114]	1419	if os.path.exists(picture_path):
[2354]	1420	response.setHeader('Content-type',mimetype)
[2114]	1421	return open(picture_path).read()
[2335]	1422	picture_path = os.path.join(i_home,'import',path)
	1423	if os.path.exists(picture_path):
	1424	return open(picture_path).read()
[2114]	1425	###)
	1426
[1151]	1427	security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
	1428	def deleteAllCourses(self,department="All"):
	1429	''' delete the courses'''
	1430	pm = self.portal_membership
	1431	member = pm.getAuthenticatedMember()
[1160]	1432
[1151]	1433	if str(member) not in ("henrik","joachim"):
	1434	return "not possible"
	1435	if department == "All":
	1436	res = self.portal_catalog({'meta_type': 'Department'})
	1437	if len(res) < 1:
	1438	return "No Departments found"
[1160]	1439
[1151]	1440	deleted = []
	1441	for dep in res:
	1442	cf = dep.getObject().courses
	1443	if cf:
	1444	cf.manage_delObjects(ids=cf.objectIds())
	1445	deleted.append("deleted Courses in %s" % dep.getId)
	1446	return "\r".join(deleted)
[1160]	1447	###)
[1151]	1448
[1572]	1449	security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
	1450	def getLogfileLines(self,filename="event.log",numlines=20):
	1451	"""Get last NUMLINES lines of logfile FILENAME.
[1160]	1452
[1572]	1453	Return last lines' of a file in the instances logfile directory as
	1454	a list. The number of returned lines equals `numlines' or less. If
	1455	less than `numlines' lines are available, the whole file ist
	1456	returned. If the file can not be opened or some other error
	1457	occurs, empty list is returend.
	1458	"""
	1459	result = []
	1460	lines_hit = 0
	1461
	1462	# We only handle files in instances' log directory...
	1463	logpath = os.path.join(i_home, "log")
	1464	filename = str(os.path.abspath( os.path.join( logpath, filename )))
	1465	if not filename.startswith( logpath ):
	1466	# Attempt to access file outside log-dir...
	1467	return []
	1468
	1469	try:
	1470	fd = file( filename, "rb" )
	1471	except IOError:
	1472	return []
	1473	if not fd:
	1474	return []
	1475
	1476	if os.linesep == None:
	1477	linesep = '\n'
	1478	else:
	1479	linesep = os.linesep
	1480
	1481	# Try to find 'numlines' times a lineseparator, searching from end
	1482	# and moving to the beginning of file...
	1483	fd.seek( 0, 2) # Move to end of file...
	1484	while lines_hit < numlines:
	1485	if fd.read(1) == linesep[-1]: # This moves filedescriptor
	1486	# one step forward...
	1487	lines_hit += 1
	1488	try:
	1489	fd.seek( -2, 1) # Go two bytes back from current pos...
	1490	except IOError:
	1491	# We cannot go back two bytes. Maybe the file is too small...
	1492	break
	1493	fd.seek(2,1)
	1494
	1495	# Read all lines from current position...
	1496	result = fd.readlines()
	1497	# Remove line endings...
	1498	result = [x.strip() for x in result]
	1499	fd.close()
	1500	return result
	1501	###)
	1502
[1700]	1503	security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
[1665]	1504	def getCallbacksFromLog(self,filename):
	1505	"""fix Online Payment Transactions from Z2.log entries"""
	1506	import transaction
	1507	import random
	1508	from cgi import parse_qs
	1509	from urlparse import urlparse
	1510	#from pdb import set_trace
	1511	wftool = self.portal_workflow
	1512	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1513	students_folder = self.portal_url.getPortalObject().campus.students
	1514	s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
	1515	s += r'(?P<date>.)\] "(?P<get>.)" (?P<codes>\d+ \d+) "'
	1516	s += r'(?P<intersw>.)" "(?P<agent>.)"'
	1517	data = re.compile(s)
	1518	start = True
	1519	tr_count = 1
	1520	total = 0
	1521	#name = 'pume_results'
	1522	#name = 'epaymentsuccessful_z2log2'
	1523	name = filename
	1524	no_import = []
	1525	imported = []
	1526	logger = logging.getLogger('WAeUPTool.getFailedTransactions')
	1527	try:
	1528	transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
	1529	except:
	1530	logger.error('Error reading %s' % name)
	1531	return
	1532	tas = []
	1533	for line in transactions:
	1534	dict = {}
	1535	items = data.search(line)
	1536	dict['idict'] = idict = items.groupdict()
	1537	#print idict
	1538	#from pdb import set_trace;set_trace()
	1539	urlparsed = urlparse(idict['get'][4:])
	1540	#print urlparsed
	1541	path = urlparsed[2].split('/')
	1542	dict['student_id'] = student_id = path[8]
	1543	dict['payment_id'] = payment_id = path[10]
	1544	dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
	1545	tas.append(dict)
	1546	tr_count += 1
	1547	return tas
	1548	###)
	1549
[1620]	1550	security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
	1551	def importOnlinePaymentTransactions(self):
	1552	"""load Online Payment Transactions from CSV values"""
	1553	import transaction
	1554	import random
	1555	#from pdb import set_trace
	1556	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
[1625]	1557	opt = self.online_payments_import
[1620]	1558	students_folder = self.portal_url.getPortalObject().campus.students
	1559	start = True
	1560	tr_count = 1
	1561	total = 0
	1562	#name = 'pume_results'
	1563	name = 'OnlineTransactions'
	1564	no_import = []
	1565	imported = []
	1566	logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
	1567	try:
	1568	transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1569	except:
	1570	logger.error('Error reading %s.csv' % name)
	1571	return
	1572	for pay_transaction in transactions:
	1573	if start:
	1574	start = False
	1575	logger.info('Start loading from %s.csv' % name)
	1576	s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
	1577	no_import.append('%s,"Error"' % s)
	1578	format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
	1579	format_error = format + ',"%(Error)s"'
	1580	data = {}
[1644]	1581
[1781]	1582	# format of the first file sent by Tayo
[1643]	1583	#data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
	1584	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1585	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1586	#data['response_code'] = response_code = pay_transaction['Resp Code']
	1587	#data['amount'] = amount = pay_transaction['Amount']
[1644]	1588
[1781]	1589	# format of the second file sent by Tayo
[1798]	1590	#data['datetime'] = date = 0
	1591	#data['student_id'] = student_id = pay_transaction['Payer ID']
	1592	#data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
	1593	#data['response_code'] = response_code = '00'
	1594	#data['amount'] = amount = pay_transaction['Amount']
[1813]	1595
[1798]	1596	# format of the third file sent by Kehinde
[1643]	1597	data['datetime'] = date = 0
[1798]	1598	data['student_id'] = student_id = pay_transaction['customer_id']
	1599	data['order_id'] = order_id = pay_transaction['merchant_reference']
[1644]	1600	data['response_code'] = response_code = '00'
[1813]	1601	data['amount'] = amount = pay_transaction['Amount']
[1644]	1602
[1620]	1603	dup = False
	1604	if response_code == "12":
	1605	continue
	1606	try:
	1607	opt.addRecord(**data)
	1608	except ValueError:
	1609	dup = True
	1610	#from pdb import set_trace;set_trace()
	1611	if dup:
	1612	if response_code == "00":
[1798]	1613	try:
	1614	opt.modifyRecord(**data)
	1615	except:
	1616	logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
	1617	continue
[1620]	1618	else:
[1674]	1619	pay_transaction['Error'] = "Duplicate order_id"
[1620]	1620	no_import.append( format_error % pay_transaction)
[1674]	1621	logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
[1620]	1622	continue
	1623	tr_count += 1
	1624	if tr_count > 1000:
	1625	if len(no_import) > 0:
	1626	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1627	'\n'.join(no_import) + '\n')
	1628	no_import = []
[1645]	1629	em = '%d transactions committed\n' % (tr_count)
[1620]	1630	transaction.commit()
	1631	regs = []
	1632	logger.info(em)
	1633	total += tr_count
	1634	tr_count = 0
	1635	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1636	'\n'.join(no_import))
	1637	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1638	###)
[2237]	1639
[3172]	1640	security.declareProtected(ModifyPortalContent,"importData")###(
	1641	def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
	1642	"""load data from CSV values"""
	1643	import transaction
	1644	import random
	1645	students_folder = self.portal_url.getPortalObject().campus.students
[3292]	1646	uploads_folder = self.portal_url.getPortalObject().campus.uploads
[3172]	1647	pending_only = False
[3177]	1648	pend_str = '--'
[3172]	1649	elapse = time.time()
	1650	#
	1651	# preparations
	1652	#
[3177]	1653	if filename == pend_str:
[3172]	1654	pending_only = True
	1655	importer_name = ''.join([part.capitalize() for part in name.split('_')])
[3191]	1656	importer = eval("%sImport" % importer_name)(self)
[3172]	1657	logger = importer.logger
	1658	if importer.init_errors:
[3191]	1659	logger.info(importer.init_errors)
	1660	return importer.init_errors
[3172]	1661	member = importer.member
[3320]	1662	#current = importer.current
[3172]	1663	import_date = importer.import_date
	1664	#
	1665	# not_imported
	1666	#
	1667	info = importer.info
	1668	data_keys = importer.data_keys
[3181]	1669	csv_keys = importer.csv_keys
[3172]	1670	#csv_keys.extend(info.keys())
	1671	headline_mapping = dict((k,k) for k in csv_keys)
	1672	#
	1673	# pending
	1674	#
	1675	pending_path = importer.pending_path
	1676	pending_tmp = importer.pending_tmp
	1677	pending_backup = importer.pending_backup
	1678	pending_fn = importer.pending_fn
	1679	imported_path = importer.imported_path
	1680	imported_fn = importer.imported_fn
[3180]	1681	commit_after = importer.commit_after
[3172]	1682	pending = []
	1683	pending_digests = []
	1684	#total_added_to_pending = 0
	1685	if not pending_only:
	1686	pending,pending_digests = importer.makeIdLists()
	1687	pending_at_start = len(pending)
	1688	datafile = open(pending_tmp,"w")
	1689	pending_csv_writer = csv.DictWriter(datafile,
	1690	csv_keys,
	1691	extrasaction='ignore')
	1692	pending_csv_writer.writerow(headline_mapping)
	1693	datafile.close()
	1694	#
	1695	# imported
	1696	#
	1697	if not os.path.exists(imported_path):
	1698	datafile = open(imported_path,"w")
	1699	imported_csv_writer = csv.DictWriter(datafile,
	1700	csv_keys,
	1701	extrasaction='ignore')
	1702	imported_csv_writer.writerow(headline_mapping)
	1703	datafile.close()
	1704	start = True
	1705	tr_count = 0
	1706	total = 0
	1707	total_added_to_imported = 0
	1708	total_pending = 0
[3292]	1709	import_source_done = ""
[3172]	1710	if pending_only:
	1711	import_source_path = pending_path
[2193]	1712	else:
[3172]	1713	import_source_path = "%s/import/%s.csv" % (i_home,filename)
[3292]	1714	import_source_done = "%s/import/%s.done" % (i_home,filename)
[3172]	1715	if not os.path.exists(import_source_path):
[3195]	1716	fn = os.path.split(import_source_path)[1]
[3268]	1717	em = 'no import file %(fn)s' % vars()
[3172]	1718	return em
	1719	import_source_fn = os.path.split(import_source_path)[1]
[3267]	1720	if not pending_only:
	1721	info['imported_from'] = import_source_fn
[3277]	1722	headline = csv.reader(open(import_source_path,"rb")).next()
	1723	if "import_mode" not in headline:
[3191]	1724	msg = 'import_mode must be in heading'
	1725	return msg
[3277]	1726	invalid_keys = importer.checkHeadline(headline)
	1727	if invalid_keys:
	1728	return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
[3201]	1729
[3300]	1730	import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
	1731	or k.strip() in info.keys())]
[3277]	1732	# diff2schema = set(import_keys).difference(set(importer.schema.keys()))
	1733	# diff2layout = set(import_keys).difference(set(importer.layout.keys()))
	1734	# if diff2schema and diff2schema != set(['id',]):
	1735	# msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
	1736	# return msg
[3172]	1737	#
	1738	# start importing
	1739	#
[2193]	1740	try:
[3172]	1741	reader = csv.DictReader(open(import_source_path,"rb"))
	1742	except:
	1743	msg = 'Error reading %s.csv' % filename
	1744	logger.error(msg)
	1745	return msg
	1746	items = [item for item in reader]
	1747	total_to_import = len(items)
[3181]	1748	tti_float = float(total_to_import)
[3172]	1749	if pending_only:
	1750	pending_at_start = total_to_import
	1751	count = 0
	1752	imported = []
[3391]	1753	old_commit_count = 0
[3204]	1754	error_count = imported_count = 0
[3172]	1755	already_in = 0
	1756	for record in items:
	1757	item = {}
[3198]	1758	empty_value_keys = []
[3172]	1759	for k,v in record.items():
[3346]	1760	if k is None:
	1761	continue
[3172]	1762	if v:
[3198]	1763	if v == EMPTY:
	1764	empty_value_keys += k,
	1765	v = ''
[3172]	1766	item[k.strip()] = v.strip()
	1767	count += 1
	1768	if start:
	1769	start = False
	1770	adapters = [MappingStorageAdapter(importer.schema, item)]
[3191]	1771	logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
[3172]	1772	dm = DataModel(item, adapters,context=self)
	1773	ds = DataStructure(data=item,datamodel=dm)
	1774	error_string = ""
	1775	total += 1
[3271]	1776	import_mode = item.get('import_mode','')
[3191]	1777	import_method = getattr(importer, '%(import_mode)s' % vars(),None )
	1778	if import_method is None:
[3271]	1779	error_string += "import_mode '%(import_mode)s' is invalid" % vars()
[3251]	1780	elif (import_mode in importer.required_modes and
[3248]	1781	not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
	1782	diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
	1783	error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
[3191]	1784	else:
	1785	for k in import_keys:
	1786	if k not in item.keys() or k not in importer.validators.keys():
	1787	continue
	1788	if not importer.validators[k](ds,mode=import_mode):
[3201]	1789	error_string += ' ++ '
	1790	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
[3172]	1791	ds.getErrorMapping(k)))
	1792	if error_string:
	1793	error = error_string
	1794	id = ''
	1795	mapping = item
[2699]	1796	else:
[3172]	1797	temp_item = item.copy()
	1798	temp_item.update(dm)
[3185]	1799	results = import_method(temp_item)
[3172]	1800	id = results[0]
	1801	error = results[1]
	1802	mapping = results[2]
[3267]	1803	#
	1804	# restore some values
	1805	#
[3198]	1806	for k in empty_value_keys:
	1807	mapping[k] = EMPTY
[3186]	1808	if mapping.has_key('sex'):
	1809	#import pdb;pdb.set_trace()
[3267]	1810	if mapping['sex'] == True:
[3186]	1811	mapping['sex'] = 'F'
[3267]	1812	elif mapping['sex'] == False:
[3186]	1813	mapping['sex'] = 'M'
[3267]	1814	if pending_only:
	1815	info['imported_from'] = item['imported_from']
[3181]	1816	data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
[3172]	1817	info['error'] = error
[3326]	1818	info['import_record_no'] = count + 1
[3172]	1819	mapping.update(info)
[3178]	1820	log_list = []
[3172]	1821	if error:
[3204]	1822	error_count += 1
[3172]	1823	digest = makeDigest(mapping,data_keys)
	1824	if digest not in pending_digests:
	1825	pending_digests += digest,
	1826	pending.append(mapping)
	1827	if not pending_only:
[3251]	1828	log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
[3172]	1829	else:
	1830	already_in += 1
	1831	pass
	1832	else:
	1833	imported_count += 1
	1834	imported += mapping,
[3251]	1835	log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
[3178]	1836	if log_list:
	1837	time_till_now = time.time() - elapse
[3204]	1838	percent_finished = (error_count + imported_count)/tti_float*100
[3181]	1839	log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
[3178]	1840	logger.info(' '.join(log_list))
[3172]	1841	finished = count > total_to_import - 1
[3277]	1842	must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
	1843	if must_commit:
	1844	old_commit_count = imported_count
[3181]	1845
[3172]	1846	if must_commit or finished:
	1847	if len(imported):
	1848	transaction.commit()
	1849	datafile = open(imported_path,"a")
	1850	writer = csv.DictWriter(datafile,
	1851	csv_keys,
	1852	extrasaction='ignore')
	1853	writer.writerows(imported)
	1854	datafile.close()
	1855	total_added_to_imported += len(imported)
	1856	imported = []
	1857	if len(pending) > 0:
	1858	datafile = open(pending_tmp,"a")
	1859	writer = csv.DictWriter(datafile,
	1860	csv_keys,
	1861	extrasaction='ignore')
	1862	writer.writerows(pending)
	1863	datafile.close()
	1864	total_pending += len(pending)
	1865	#total_added_to_pending += len(pending)
	1866	pending = []
	1867	if not finished:
[3335]	1868	msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
[3172]	1869	logger.info(msg)
	1870	elapse = time.time() - elapse
[3271]	1871	if os.path.exists(pending_path):
	1872	copy2(pending_path,pending_backup)
[3172]	1873	copy2(pending_tmp,pending_path)
[3191]	1874	msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
[3181]	1875	msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
[3172]	1876	if pending_only:
	1877	removed_pending = pending_at_start - total_pending
[3184]	1878	msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
[2699]	1879	else:
[3181]	1880	added_pending = total_pending - pending_at_start
	1881	msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
	1882	#msg += "%(total_pending)d totally written" % vars() # this line does not make any sense
[3172]	1883	logger.info(msg)
[3292]	1884	if import_source_done:
	1885	copy(import_source_path,import_source_done)
	1886	os.remove(import_source_path)
	1887	upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
	1888	if upload is not None:
	1889	upload_doc = upload.getContent()
	1890	mapping = {}
[3308]	1891	#mapping['import_date'] = DateTime.DateTime()
	1892	mapping['import_date'] = import_date
[3327]	1893	mapping['imported_by'] = importer.imported_by
[3292]	1894	mapping['import_message'] = msg
	1895	upload_doc.edit(mapping = mapping)
[3178]	1896	os.remove(pending_tmp)
[3172]	1897	return msg
[2514]	1898	###)
	1899
[2292]	1900	security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
	1901	def moveImagesToFS(self,student_id="O738726"):
	1902	"move the images to the filesystem"
[2675]	1903	images_dir = getImagesDir(student_id)
	1904	#images_dir = os.path.join("%s" % images_base,student_id)
[2292]	1905	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1906	stool = getToolByName(self, 'portal_schemas')
	1907	schemas = ['student_application',
	1908	'student_clearance',
	1909	]
	1910	created = False
	1911	for schema_id in schemas:
	1912	schema = stool._getOb(schema_id)
	1913	object = getattr(student_folder,schema_id[len('student_'):],None)
	1914	if object is None:
	1915	continue
	1916	doc = object.getContent()
	1917	for key in schema.keys():
	1918	if schema[key].meta_type != "CPS Image Field":
	1919	continue
	1920	#import pdb;pdb.set_trace()
[2351]	1921	image = getattr(doc,key,None)
	1922	if not image or not hasattr(image,"data"):
[2292]	1923	continue
	1924	if not created:
	1925	if not os.path.exists(images_dir):
	1926	os.mkdir(images_dir)
	1927	created = True
	1928	filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
	1929	open(filename,"wb").write(str(image.data))
[2500]	1930	###)
[2292]	1931
[2300]	1932	security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
	1933	def movePassportToFS(self,student_id="O738726"):
	1934	"move the passports to the filesystem"
	1935	images_dir = os.path.join("%s" % i_home,'passports')
	1936	student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
	1937	stool = getToolByName(self, 'portal_schemas')
	1938	schemas = ['student_application',
	1939	#'student_clearance',
	1940	]
	1941	created = False
	1942	for schema_id in schemas:
	1943	schema = stool._getOb(schema_id)
	1944	object = getattr(student_folder,schema_id[len('student_'):],None)
	1945	if object is None:
	1946	continue
	1947	doc = object.getContent()
	1948	for key in schema.keys():
	1949	if schema[key].meta_type != "CPS Image Field":
	1950	continue
	1951	#import pdb;pdb.set_trace()
	1952	image = getattr(doc,key)
	1953	if not hasattr(image,"data"):
	1954	continue
	1955	if not created:
	1956	if not os.path.exists(images_dir):
	1957	os.mkdir(images_dir)
	1958	created = True
	1959	filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
	1960	open(filename,"wb").write(str(image.data))
[2500]	1961	###)
[3824]	1962
	1963
	1964	security.declarePublic('getConfigParams')
	1965	def getConfigParams(self,conf_id="configuration"):
	1966	conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
	1967	conf_obj = conf.getContent()
	1968	stool = getToolByName(self, 'portal_schemas')
	1969	schema = stool._getOb('configuration')
	1970	d = {}
	1971	for key in schema.keys():
	1972	d[key] = getattr(conf_obj,key,None)
	1973	return d
[2300]	1974
[3824]	1975
[5223]	1976	security.declarePublic("changeWorkflowState")
[5183]	1977	def changeWorkflowState(self, content, state_id, acquire_permissions=False,
	1978	portal_workflow=None, **kw):
[5223]	1979	#Change the workflow state of an object
	1980	#@param content: Content obj which state will be changed
	1981	#@param state_id: name of the state to put on content
	1982	#@param acquire_permissions: True->All permissions unchecked and on riles and
	1983	# acquired
	1984	# False->Applies new state security map
	1985	#@param portal_workflow: Provide workflow tool (optimisation) if known
	1986	#@param kw: change the values of same name of the state mapping
	1987	#@return: None
	1988
[5183]	1989
	1990	if portal_workflow is None:
	1991	portal_workflow = getToolByName(content, 'portal_workflow')
	1992
	1993	# Might raise IndexError if no workflow is associated to this type
	1994
	1995	wf_def = portal_workflow.getWorkflowsFor(content)[0]
	1996	wf_id= wf_def.getId()
	1997
	1998	wf_state = {
	1999	'action': None,
	2000	'actor': None,
	2001	'comments': "Setting state to %s" % state_id,
	2002	'review_state': state_id,
	2003	'time': DateTime.DateTime(),
	2004	}
	2005
	2006	# Updating wf_state from keyword args
	2007	for k in kw.keys():
	2008	# Remove unknown items
	2009	if not wf_state.has_key(k):
	2010	del kw[k]
	2011	if kw.has_key('review_state'):
	2012	del kw['review_state']
	2013	wf_state.update(kw)
	2014
	2015	portal_workflow.setStatusOf(wf_id, content, wf_state)
	2016
	2017	if acquire_permissions:
	2018	# Acquire all permissions
	2019	for permission in content.possible_permissions():
	2020	content.manage_permission(permission, acquire=1)
	2021	else:
	2022	# Setting new state permissions
	2023	wf_def.updateRoleMappingsFor(content)
	2024
	2025	# Map changes to the catalogs
	2026	content.reindexObject(idxs=['allowedRolesAndUsers', 'review_state'])
	2027	return
	2028
	2029
	2030
	2031	security.declareProtected(ModifyPortalContent,"changeStudentState")
	2032	def changeStudentState(self, content, state_id, acquire_permissions=False,
	2033	portal_workflow=None, **kw):
	2034	"""Change the student state of an object
	2035
	2036	ChangeWorkflowState + indexing students_catalog and check for registration state
	2037
	2038	"""
	2039
	2040	if portal_workflow is None:
	2041	portal_workflow = getToolByName(content, 'portal_workflow')
	2042
	2043	# Might raise IndexError if no workflow is associated to this type
	2044
	2045	wf_def = portal_workflow.getWorkflowsFor(content)[0]
	2046	wf_id= wf_def.getId()
	2047	wf = portal_workflow.getWorkflowById(wf_id)
	2048
	2049	if not wf.states.get(state_id):
	2050	return 'reg state does not exist'
	2051
	2052	wf_state = {
	2053	'action': None,
	2054	'actor': None,
	2055	'comments': "Setting state to %s" % state_id,
	2056	'review_state': state_id,
	2057	'time': DateTime.DateTime(),
	2058	}
	2059
	2060	# Updating wf_state from keyword args
	2061	for k in kw.keys():
	2062	# Remove unknown items
	2063	if not wf_state.has_key(k):
	2064	del kw[k]
	2065	if kw.has_key('review_state'):
	2066	del kw['review_state']
	2067	wf_state.update(kw)
	2068
	2069	portal_workflow.setStatusOf(wf_id, content, wf_state)
	2070
	2071	if acquire_permissions:
	2072	# Acquire all permissions
	2073	for permission in content.possible_permissions():
	2074	content.manage_permission(permission, acquire=1)
	2075	else:
	2076	# Setting new state permissions
	2077	wf_def.updateRoleMappingsFor(content)
	2078
	2079	# Map changes to the catalogs
	2080	content.reindexObject(idxs=['allowedRolesAndUsers', 'review_state'])
	2081	data = {}
	2082	#import pdb;pdb.set_trace()
	2083	student_id = content.getId()
	2084	res = self.students_catalog(id=student_id)
	2085	data['id'] = student_id
	2086	data['review_state'] = state_id
	2087	self.students_catalog.modifyRecord(**data)
	2088
	2089	return
	2090
	2091
[5191]	2092	security.declarePublic('web_sniffer') ###(
	2093	def web_sniffer(self,domain=None,params=None):
	2094	"web sniffer based on http://python-dev.blogspot.com/2008/06/python-websniffer.html"
	2095
	2096	import httplib
	2097	import string
	2098	import os
	2099
	2100	opera = {"User-Agent":"Opera/9.20 (Windows NT 6.0; U; en)", "Accept-Charset":"ISO-8859-1,UTF-8;q=0.7,*;q=0.7", "Cache-Control":"no", "Accept-Language":"de,en;q=0.7,en-us;q=0.3", "Referer":"http://fake_referer.net"}
	2101
	2102	path = '/'
	2103	method = 'GET'
	2104	profile = opera
	2105	#import pdb;pdb.set_trace()
	2106	conn = httplib.HTTPSConnection(domain)
	2107	conn.request(method, path, params, profile)
	2108	stat = conn.getresponse()
	2109	headers = stat.getheaders()
	2110	version = stat.version
	2111	status = stat.status
	2112	reason = stat.reason
	2113	data = stat.read()
	2114	conn.close()
	2115
	2116	#file_name = domain.replace(".", "-")
	2117	#file_object = open(file_name, "w")
	2118	#file_object.write(data)
	2119	#file_object.close()
	2120
	2121	print "\n"
	2122	print "WEBSNIFFER 0.1"
	2123	print "+++++++++++++++++"
	2124	print "\n"
	2125	print "Header (Client)"
	2126	print "-----------------"
	2127	for key, value in profile.iteritems():
	2128	print "%s \t %s" % (key, value)
	2129	print "-----------------"
	2130	print "\n"
	2131	print "Header (Server)"
	2132	print "-----------------"
	2133	print "HTTP \t %s" % (version)
	2134	print "Status \t %s (%s)" % (status, reason)
	2135	for key, value in headers:
	2136	print "%s \t %s" % (key, value)
	2137	print "-----------------"
	2138	print "\n"
	2139	#print "Quelltext"
	2140	#print "-----------------"
	2141	#print "Quelltext geschrieben in File:"
	2142	#print file_name
	2143	#print "-----------------"
	2144	#print "\n"
	2145
	2146
	2147
[828]	2148	InitializeClass(WAeUPTool)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: