Context navigation

unused_methods.txt @ 3371

Last change on this file since 3371 was 3251, checked in by Henrik Bettermann, 17 years ago

improve logging messages.

This is the actual logging history for imports:

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? admin starts import from test.csv

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? 100.000 % done in 0.01s, record from test.csv added to verdicts.pending, import_mode: edit, current_session: 06, matric_no: ENG0206078, current_level: 700, current_verdict: L, id: G151316, student G151316: imported level 700 does not match current_level 400

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? finished importing from test.csv in 0.01 seconds, 1 records totally read, 0 added to verdicts.imported, 1 added to verdicts.pending, 0 already in verdicts.pending

2008-02-28 10:47:06 INFO WAeUPImport.VerdictsImport? admin starts import from verdicts.pending

2008-02-28 10:47:07 INFO WAeUPImport.VerdictsImport? 100.000 % done in 0.99s, record imported and added to verdicts.imported from verdicts.pending, error: student G151316: imported level 700 does not match current_level 400, import_mode: edit, matric_no: ENG0206078, current_level: 400, imported_by: admin, import_date: 28/02/08 10:45:39, current_session: 06, current_verdict: L, id: G151316, imported_from: test.csv

2008-02-28 10:47:07 INFO WAeUPImport.VerdictsImport? finished importing from verdicts.pending in 1.17 seconds, 1 records totally read, 1 added to verdicts.imported, 1 removed from verdicts.pending

File size: 120.7 KB

Rev	Line
[3251]	1	From Students.py:
	2
	3	security.declareProtected(ModifyPortalContent,"loadFullTimeStudentsFromCSV")###(
	4	def loadFullTimeStudentsFromCSV(self):
	5	"""load Fulltime Studentdata from CSV values"""
	6	import transaction
	7	import random
	8	tr_count = 0
	9	name = 'short_full_time'
	10	no_import = False
	11	if not no_import:
	12	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
	13	no_import.write('"MatricNo","EntryRegNo","CurrentSession","StudentLevel","fullname","FirstName","MiddleName","Lastname","FormerSurname","Sex","Nationality","State","LGA","PermanentAddress","PermanentAddressCity","CampusAddress","PhoneNumber","Emailaddress","Mode","CourseMajor","Faculty","Dept"\n')
	14	logger = logging.getLogger('Import.%s' % name)
	15	logger.info('Start loading from %s.csv' % name)
	16	pwlist = []
	17	pwlist.append('"student_id","firstname","middlename","lastname","matric_no","jamb_reg_no","access_code"')
	18	pwl_template = Template('"$student_id","$firstname","$middlename","$lastname","$matric_no","$jamb_reg_no","$access_code"')
	19	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	20	try:
	21	students = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	22	except:
	23	logger.error('Error reading %s.csv' % name)
	24	return
	25	l = self.portal_catalog({'meta_type': "StudentClearance",})
	26	matrics = []
	27	for s in l:
	28	matrics.append(s.getObject().getContent().matric_no)
	29	print matrics
	30	l = self.portal_catalog({'meta_type': "Certificate"})
	31	certs = {}
	32	for c in l:
	33	ca,ac,fa,dep_id,co,certcode = c.relative_path.split('/')
	34	cid = "%(dep_id)s_%(certcode)s" % vars()
	35	certs[cid] = c.getObject()
	36	for student in students:
	37	logger.info('processing "%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
	38	sid = student.get('MatricNo')
	39	if sid == "":
	40	em = 'Empty MatricNo\n'
	41	logger.info(em)
	42	no_import.write(em)
	43	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
	44	continue
	45	certcode = makeCertificateCode(student.get('CourseMajor'))
	46	dep_id = student.get('Dept')
	47	fac_id = student.get('Faculty')
	48	cid = "%(dep_id)s_%(certcode)s" % vars()
	49	if cid not in certs.keys():
	50	em = 'Certificate with ID %s %s not found\n' % (certcode, student.get('CourseMajor'))
	51	logger.info(em)
	52	no_import.write(em)
	53	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
	54	continue
	55	certificate_doc = certs[cid].getContent()
	56	level = student.get('StudentLevel')
	57	try:
	58	int(level)
	59	except:
	60	em = 'Student with ID %(MatricNo)s StudentLevel is empty\n' % student
	61	logger.info(em)
	62	no_import.write(em)
	63	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
	64	continue
	65	matric_no = student.get('MatricNo')
	66	if matric_no not in matrics:
	67	matrics.append(matric_no)
	68	sid = self.generateStudentId(student.get('Lastname')[0])
	69	#self.log('Creating Faculty %(id)s = %(Title)s' % faculty)
	70	students_folder.invokeFactory('Student', sid)
	71	logger.info('%(tr_count)s: Creating Student with ID %(sid)s Matric_no %(matric_no)s ' % vars())
	72	student_obj = getattr(self,sid)
	73	access_code = "%d" % random.randint(1000000000,9999999999)
	74	student_obj.getContent().makeStudentMember(sid,access_code,)
	75	pwl_dict = {'student_id': sid,'access_code':access_code}
	76	student_obj.invokeFactory('StudentApplication','application')
	77	application = student_obj.application
	78	da = {'Title': 'Application Data'}
	79	student_obj.invokeFactory('StudentPersonal','personal')
	80	da['jamb_reg_no'] = student.get('EntryRegNo')
	81	personal = student_obj.personal
	82	dp = {'Title': 'Personal Data'}
	83	student_obj.invokeFactory('StudentClearance','clearance')
	84	clearance = student_obj.clearance
	85	dc = {'Title': 'Clearance/Eligibility Record'}
	86	dc['matric_no'] = matric_no
	87	state = student.get('State')
	88	lga = student.get('LGA')
	89	if state and lga:
	90	lga = state + ' / ' + lga
	91	else:
	92	lga = "None"
	93	dc['lga'] = lga
	94	dc['nationality'] = student.get('Nationality')
	95	dc['email'] = student.get('Emailaddress')
	96	dp['firstname'] = student.get('FirstName')
	97	dp['middlename'] = student.get('MiddleName')
	98	dp['lastname'] = student.get('Lastname')
	99	dp['former_surname'] = student.get('FormerSurname')
	100	dp['sex'] = student.get('Sex') == 'F'
	101	dp['perm_address'] = student.get('PermanentAddress')
	102	dp['perm_city'] = student.get('PermanentAddressCity')
	103	dp['campus_address'] = student.get('CampusAddress')
	104	dp['phone'] = student.get('PhoneNumber')
	105	application.getContent().edit(mapping=da)
	106	personal.getContent().edit(mapping=dp)
	107	clearance.getContent().edit(mapping=dc)
	108	#
	109	# Study Course
	110	#
	111	student_obj.invokeFactory('StudentStudyCourse','study_course')
	112	studycourse = student_obj.study_course
	113	dsc = {}
	114	from_certificate = ['title',
	115	'max_elect',
	116	'max_pass',
	117	'n_core',
	118	'nr_years',
	119	'probation_credits',
	120	'promotion_credits',
	121	'start_level',
	122	]
	123	for f in from_certificate:
	124	dsc[f] = getattr(certificate_doc,f)
	125	dsc['faculty'] = fac_id
	126	dsc['department'] = dep_id
	127	dsc['study_course'] = certcode
	128	css = student.get('CurrentSession') or '2004-2005'
	129	cs = int(css.split('-')[0]) - 2000
	130	cl = int(student.get('StudentLevel') or '100')/100
	131	dsc['entry_session'] = "200%s" % (cs - cl)
	132	dsc['clr_ac_pin'] = access_code
	133	studycourse.getContent().edit(mapping=dsc)
	134	#
	135	# Level
	136	#
	137	## l = getattr(studycourse,level,None)
	138	## if 0 and l is None:
	139	## #self.log('Creating Department %(DeptCode)s = %(Description)s' % dep)
	140	## logger.info('Creating Level %(StudentLevel)s for %(fullname)s' % student)
	141	## studycourse.invokeFactory('StudentStudyLevel', level)
	142	## l = getattr(studycourse, level)
	143	## certificate = certs[certcode]
	144	## cert_level = getattr(certificate,level,None)
	145	## if cert_level is None:
	146	## logger.info('Level %(level)s not in %(certcode)s' % vars())
	147	## l.getContent().edit(mapping={'Title': "Level %s" % level})
	148	else:
	149	em = 'Student with ID %(MatricNo)s %(fullname)s already exists\n' % student
	150	logger.info(em)
	151	no_import.write(em)
	152	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
	153	continue
	154	if tr_count > MAX_TRANS:
	155	transaction.commit()
	156	em = 'Transaction commited\n' % student
	157	logger.info(em)
	158	tr_count = 0
	159	tr_count += 1
	160	pwl_dict.update(dc)
	161	pwl_dict.update(da)
	162	pwl_dict.update(dp)
	163	wftool = self.portal_workflow
	164	pwlist.append(pwl_template.substitute(pwl_dict))
	165	wftool.doActionFor(student_obj,'clear_and_validate')
	166	student_obj.manage_setLocalRoles(sid, ['Owner',])
	167	wftool.doActionFor(application,'close')
	168	application.manage_setLocalRoles(sid, ['Owner',])
	169	wftool.doActionFor(clearance,'close')
	170	clearance.manage_setLocalRoles(sid, ['Owner',])
	171	wftool.doActionFor(personal,'close')
	172	personal.manage_setLocalRoles(sid, ['Owner',])
	173	wftool.doActionFor(studycourse,'close_for_edit')
	174	studycourse.manage_setLocalRoles(sid, ['Owner',])
	175	open("%s/import/pwlist-%s.csv" % (i_home,name),"w+").write('\n'.join(pwlist))
	176	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	177	###)
	178
	179	security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
	180	def loadPumeResultsFromCSV(self):
	181	"""load Fulltime Studentdata from CSV values into pumeresults catalog"""
	182	import transaction
	183	import random
	184	## csv_d = {'jamb_reg_no': "RegNumber", ###(
	185	## 'status': "Admission Status",
	186	## 'name': "Name",
	187	## 'score': "Score",
	188	## 'sex': "Sex",
	189	## 'faculty': "Faculty",
	190	## 'department': "Dept",
	191	## 'course': "Course",
	192	## 'course_code_org': "Course Code",
	193	## }
	194	###)
	195	csv_d = {'jamb_reg_no': "JAMBRegno",
	196	'name': "Name",
	197	'score': "Score",
	198	'sex': "Sex",
	199	'course': "Course",
	200	'faculty': "Faculty",
	201	'department': "Dept",
	202	'course_code_org': "Course Code",
	203	'status': "Admission Status",
	204	'result_type': None,
	205	}
	206	csv_fields = [f[1] for f in csv_d.items() if f[1]]
	207	tr_count = 0
	208	total = 0
	209	#name = 'pup_new'
	210	name = 'pup_update'
	211	update = name.endswith('update')
	212	no_import = []
	213	ok_import = []
	214	ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
	215	no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
	216	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	217	ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
	218	#open(ok_import_name,"w").write('\n'.join(no_import))
	219	no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
	220	#open(no_import_name,"w").write('\n'.join(no_import))
	221	logger = logging.getLogger('Import.%s' % name)
	222	starttime = DateTime.now()
	223	logger.info('Start loading from %s.csv' % name)
	224	try:
	225	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	226	except:
	227	logger.error('Error reading %s.csv' % name)
	228	return
	229	pume = self.portal_pumeresults
	230	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	231	import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
	232	eduplicate = '%s,"duplicate"' % format
	233	einvalidjamb = '%s,"invalid JambRegNo"' % format
	234	added = 'added ,%s' % format
	235	#from pdb import set_trace;set_trace()
	236	for jamb in result:
	237	dict = {}
	238	for f,fn in csv_d.items():
	239	dict[f] = jamb.get(csv_d[f])
	240	dict['result_type'] = 'DE'
	241	jnr = jamb.get(csv_d['jamb_reg_no'])
	242	if not checkJambNo(jnr):
	243	logger.info(einvalidjamb % jamb)
	244	dd = {}
	245	for f,fn in csv_d.items():
	246	dd[fn] = getattr(data,f)
	247	no_import.append(eduplicate % dd)
	248	no_import.append(eduplicate % jamb)
	249	continue
	250	res = pume(jamb_reg_no=jnr)
	251	if len(res) > 0:
	252	if update:
	253	try:
	254	pume.modifyRecord(**dict)
	255	except ValueError:
	256	logger.info(eduplicate % jamb)
	257	continue
	258	except KeyError:
	259	pume.addRecord(**dict)
	260	logger.info(added % jamb)
	261	continue
	262	else:
	263	data = res[0]
	264	if data.name != jamb.get(csv_d['name']):
	265	#set_trace()
	266	logger.info(eduplicate % jamb)
	267	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
	268	#logger.info(em)
	269	dd = {}
	270	for f,fn in csv_d.items():
	271	dd[fn] = getattr(data,f)
	272	no_import.append(eduplicate % dd)
	273	no_import.append(eduplicate % jamb)
	274	continue
	275	try:
	276	pume.addRecord(**dict)
	277	ok_import.append(import_format % dict)
	278	except ValueError:
	279	logger.info(eduplicate % jamb)
	280	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
	281	#logger.info(em)
	282	no_import.append(eduplicate % jamb)
	283	logger.info('End loading from %s.csv' % name)
	284	if len(no_import) > 1:
	285	open(no_import_name,"w+").write('\n'.join(no_import))
	286	open(ok_import_name,"w+").write('\n'.join(ok_import))
	287	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	288	###)
	289
	290
	291	security.declareProtected(ModifyPortalContent,"OLDloadPumeResultsFromCSV")###(
	292	def OLDloadPumeResultsFromCSV(self):
	293	"""load Fulltime Studentdata from CSV values"""
	294	import transaction
	295	import random
	296	wftool = self.portal_workflow
	297	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	298	csv_d = {'jamb_reg_no': "JAMBRegno",
	299	'jamb_lastname': "Name",
	300	'pume_options': "Options",
	301	'session': "Session",
	302	'days': "Days",
	303	'response': "Responce",
	304	'wrong': "Wrong",
	305	'pume_eng_score': "EngScore",
	306	'pume_gen_score': "GenScore",
	307	'pume_tot_score': "Score",
	308	'batch': "Batch",
	309	'serial': "SerialNo",
	310	'jamb_score': "JambScore",
	311	'omitted':"Omitted",
	312	'search_key': "SearchKey",
	313	'jamb_sex': "Sex",
	314	'fac1': "Fac1",
	315	'fac2': "Fac2",
	316	'jamb_first_cos': "CourseofStudy",
	317	'stud_status':"StudStatus",
	318	'registered': "Registered",
	319	'jamb_state': "State",
	320	'eng_fail': "EngFail",
	321	'gen_fail': "GenFail",
	322	'un_ans_eng': "UnAnsEng",
	323	'un_ans_eng': "UnAnsGen",
	324	'total_ans': "TotalUnAns",
	325	'dept': "Dept",
	326	'jamb_second_cos': "Course2",
	327	'jamb_third_cos': "course3",
	328	}
	329	csv_fields = [f[1] for f in csv_d.items()]
	330	tr_count = 0
	331	name = 'pume_results'
	332	no_import = []
	333	s = ','.join(['"%s"' % fn for fn in csv_fields])
	334	no_import.append('%s\n' % s)
	335	logger = logging.getLogger('Import.%s' % name)
	336	logger.info('Start loading from %s.csv' % name)
	337	try:
	338	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	339	except:
	340	logger.error('Error reading %s.csv' % name)
	341	return
	342	for jamb in result:
	343	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	344	processing = "processing %s" % format
	345	logger.info(processing % jamb)
	346	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
	347	#import pdb;pdb.set_trace()
	348	res = self.portal_catalog({'portal_type': "StudentApplication",
	349	'jamb_reg_no': jamb_reg_no })
	350	if res:
	351	em = 'Student with REG-NO %s already exists\n' % jamb_reg_no
	352	logger.info(em)
	353	no_import.append(em)
	354	no_import.append(format % jamb)
	355	continue
	356	cert_id = jamb.get(csv_d['jamb_first_cos']).upper()
	357	res = self.portal_catalog({'portal_type': "Certificate",
	358	'id': cert_id })
	359	if len(res) < 1:
	360	em = 'No Certificate with ID %s \n' % cert_id
	361	logger.info(em)
	362	no_import.append(em)
	363	no_import.append(format % jamb)
	364	continue
	365	cert = res[0].getObject()
	366	cert_path = res[0].getPath()
	367	cert_doc = cert.getContent()
	368	jamb_name = jamb.get(csv_d['jamb_lastname'])
	369	jamb_name.replace('>','')
	370	names = jamb_name.split()
	371	letter = names[-1][0].upper()
	372	sid = self.generateStudentId(letter)
	373	not_created = True
	374	while not_created:
	375	try:
	376	students_folder.invokeFactory('Student', sid)
	377	not_created = False
	378	except BadRequest:
	379	sid = self.generateStudentId(letter)
	380	logger.info('%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
	381	student = getattr(self,sid)
	382	student.manage_setLocalRoles(sid, ['Owner',])
	383	student.invokeFactory('StudentClearance','clearance')
	384	#wftool.doActionFor(student.clearance,'open')
	385	dp = {'Title': 'Clearance/Eligibility Record'}
	386	student.clearance.manage_setLocalRoles(sid, ['Owner',])
	387	student.invokeFactory('StudentPume','pume')
	388	dp = {'Title': 'Pume Data'}
	389	student.invokeFactory('StudentApplication','application')
	390	da = {'Title': 'Application Data'}
	391	da["jamb_lastname"] = jamb_name
	392	da_fields = ('jamb_reg_no',
	393	'jamb_sex',
	394	'jamb_state',
	395	'jamb_score',
	396	'jamb_first_cos',
	397	'jamb_sex',
	398	'jamb_state',
	399	'jamb_first_cos',
	400	'jamb_second_cos',
	401	)
	402	for f in da_fields:
	403	da[f] = jamb.get(csv_d[f])
	404	app = student.application
	405	app.getContent().edit(mapping=da)
	406	app.manage_setLocalRoles(sid, ['Owner',])
	407	#wftool.doActionFor(app,'close')
	408	dp_fields = (
	409	'pume_eng_score',
	410	'pume_gen_score',
	411	'pume_tot_score',
	412	)
	413	for f in dp_fields:
	414	dp[f] = float(jamb.get(csv_d[f]))
	415	pume = student.pume
	416	pume.getContent().edit(mapping=dp)
	417	#wftool.doActionFor(pume,'close')
	418	pume.manage_setLocalRoles(sid, ['Owner',])
	419	#
	420	# Study Course
	421	#
	422	student.invokeFactory('StudentStudyCourse','study_course')
	423	study_course = student.study_course
	424	dsc = {}
	425	from_certificate = ['title',
	426	'max_elect',
	427	'max_pass',
	428	'n_core',
	429	'nr_years',
	430	'probation_credits',
	431	'promotion_credits',
	432	'start_level',
	433	]
	434	for f in from_certificate:
	435	dsc[f] = getattr(cert_doc,f)
	436	cpl = cert_path.split('/')
	437	dsc['faculty'] = cpl[-4]
	438	dsc['department'] = cpl[-3]
	439	dsc['study_course'] = cert_id
	440	dsc['entry_session'] = jamb.get(csv_d['session'])
	441	study_course.getContent().edit(mapping=dsc)
	442	student.getContent().createSubObjects()
	443	if dp['pume_tot_score']>49:
	444	wftool.doActionFor(student,'pume_pass')
	445	wftool.doActionFor(student,'admit')
	446	else:
	447	wftool.doActionFor(student,'pume_fail')
	448	wftool.doActionFor(student,'reject_admission')
	449	if len(no_import) > 1:
	450	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
	451	'\n'.join(no_import))
	452	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	453	###)
	454
	455	security.declareProtected(ModifyPortalContent,"loadFullTimeStudentsResultsFromCSV") ###(
	456	def loadFullTimeStudentsResultsFromCSV(self):
	457	"""load Fulltime Studentdata from CSV values"""
	458	#return
	459	level_wf_actions = {}
	460	level_wf_actions["SUCCESSFUL STUDENT"] = "pass_A"
	461	level_wf_actions["STUDENT WITH CARRYOVER COURSES"] = "pass_B"
	462	level_wf_actions["STUDENT FOR PROBATION"] = "probate_C"
	463	level_wf_actions["STUDENT ON PROBATION/TRANSFER"] = "reject_D"
	464	import transaction
	465	wftool = self.portal_workflow
	466	tr_count = 0
	467	name = 'short_full_time_results_2004_2005'
	468	no_import = False
	469	if not no_import:
	470	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
	471	no_import.write('"Matnumber","CosCode","Ansbook","CosStuatus","Session","Mat_Cos","Score","CarryLevel","Grade","Weight","Semster","Verdict","Level","id","GPA"\n')
	472	logger = logging.getLogger('import.%s' % name)
	473	logger.info('Start loading from %s.csv' % name)
	474	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	475	try:
	476	results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	477	except:
	478	logger.error('Error reading %s.csv' % name)
	479	return
	480	l = self.portal_catalog({'meta_type': "Course"})
	481	courses = {}
	482	for c in l:
	483	courses[c.id] = c.getObject()
	484	level_changed = False
	485	student_changed = False
	486	sid = ''
	487	#import pdb;pdb.set_trace()
	488	for result in results:
	489	temp_sid = result.get('Matnumber')
	490	if temp_sid != sid:
	491	student_changed = True
	492	res = self.portal_catalog({'meta_type': "StudentClearance",
	493	'SearchableText': temp_sid })
	494	if not res:
	495	em = 'Student with ID %(Matnumber)s not found\n' % result
	496	logger.info(em)
	497	no_import.write(em)
	498	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
	499	continue
	500	elif len(res) > 1:
	501	em = 'More than one Student with ID %(Matnumber)s found\n' % result
	502	logger.info(em)
	503	no_import.write(em)
	504	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
	505	continue
	506	sid = temp_sid
	507	sf = res[0].getObject().aq_parent
	508	sc = getattr(sf,'study_course')
	509	level = ''
	510	else:
	511	student_changed = False
	512	course = result.get('CosCode')
	513	if course not in courses.keys():
	514	em = 'Course with ID %(CosCode)s not found\n' % result
	515	logger.info(em)
	516	no_import.write(em)
	517	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
	518	continue
	519	course_doc = courses[course].getContent()
	520	temp_level = result.get('Level')
	521	student_id = sf.getId()
	522	result['StudentId'] = student_id
	523	if temp_level != level:
	524	try:
	525	int(temp_level)
	526	except:
	527	em = 'Result with ID %(Matnumber)s Course %(CosCode)s Level is empty\n' % result
	528	logger.info(em)
	529	no_import.write(em)
	530	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
	531	continue
	532	level_changed = True
	533	if 'dlev' in vars().keys():
	534	wftool.doActionFor(l,level_wf_actions[dlev['verdict']])
	535	level = temp_level
	536	l = getattr(sc,level,None)
	537	if l is None:
	538	logger.info('Creating Level %(Level)s for %(StudentId)s %(Matnumber)s' % result)
	539	sc.invokeFactory('StudentStudyLevel', level)
	540	l = getattr(sc, level)
	541	l.manage_setLocalRoles(student_id, ['Owner',])
	542	else:
	543	level_changed = False
	544	cr = getattr(l,course,None)
	545	if cr is None:
	546	logger.info('Creating Course %(CosCode)s for %(StudentId)s %(Matnumber)s in Level %(Level)s' % result)
	547	l.invokeFactory('StudentCourseResult',course)
	548	cr = getattr(l,course)
	549	dcr = {}
	550	from_course = ['title',
	551	'credits',
	552	'passmark',
	553	]
	554	for f in from_course:
	555	dcr[f] = getattr(course_doc,f)
	556	dlev = {}
	557	dcr['ansbook'] = result.get('Ansbook')
	558	dcr['semester'] = getInt(result.get('Semster'))
	559	dcr['status'] = result.get('CosStuatus')
	560	dcr['score'] = getInt(result.get('Score'))
	561	dlev['session'] = result.get('Session')
	562	dcr['carry_level'] = result.get('CarryLevel')
	563	dcr['grade'] = result.get('Grade')
	564	dcr['weight'] = result.get('Weight')
	565	dlev['verdict'] = result.get('Verdict')
	566	dcr['import_id'] = result.get('id')
	567	gpa = result.get('GPA').replace(',','.')
	568	dlev['imported_gpa'] = getFloat(gpa)
	569	cr.getContent().edit(mapping = dcr)
	570	cr.manage_setLocalRoles(student_id, ['Owner',])
	571	l.getContent().edit(mapping = dlev)
	572	if tr_count > MAX_TRANS:
	573	transaction.commit()
	574	tr_count = 0
	575	tr_count += 1
	576	wftool.doActionFor(cr,'close')
	577	wftool.doActionFor(l,level_wf_actions[dlev['verdict']])
	578	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	579
	580	###)
	581
	582	security.declareProtected(ModifyPortalContent,"loadJAMBFromCSV")###(
	583	def loadJAMBFromCSV(self):
	584	"""load JAMB data from CSV values"""
	585	#return
	586	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	587	import transaction
	588	tr_count = 0
	589	name = 'SampleJAMBDataII'
	590	wftool = self.portal_workflow
	591	no_import = False
	592	if not no_import:
	593	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
	594	no_import.write('REG-NO,NAME,SEX,STATE,LGA,ENG-SCORE,SUBJ1,SUBJ1-SCORE,SUBJ2,SUBJ2-SCORE,SUBJ3,SUBJ3-SCORE,AGGREGATE,UNIV1,FACULTY1,COURSE1,UNIV2,FACULTY2,COURSE2')
	595	logger = logging.getLogger('Import.%s' % name)
	596	logger.info('Start loading from %s.csv' % name)
	597	try:
	598	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	599	except:
	600	logger.error('Error reading %s.csv' % name)
	601	return
	602	for jamb in result:
	603	logger.info('processing %(REG-NO)s,%(NAME)s,%(SEX)s,%(STATE)s,%(LGA)s,%(ENG-SCORE)s,%(SUBJ1)s,%(SUBJ1-SCORE)s,%(SUBJ2)s,%(SUBJ2-SCORE)s,%(SUBJ3)s,%(SUBJ3-SCORE)s,%(AGGREGATE)s,%(UNIV1)s,%(FACULTY1)s,%(COURSE1)s,%(UNIV2)s,%(FACULTY2)s,%(COURSE2)s\n' % jamb)
	604	jamb_reg_no = jamb.get('REG-NO')
	605	res = self.portal_catalog({'meta_type': "StudentApplication",
	606	'jamb_reg_no': jamb_reg_no })
	607	if res:
	608	em = 'Student with REG-NO %(REG-NO)s already exists\n' % jamb
	609	logger.info(em)
	610	no_import.write(em)
	611	no_import.write('%(REG-NO)s,%(NAME)s,%(SEX)s,%(STATE)s,%(LGA)s,%(ENG-SCORE)s,%(SUBJ1)s,%(SUBJ1-SCORE)s,%(SUBJ2)s,%(SUBJ2-SCORE)s,%(SUBJ3)s,%(SUBJ3-SCORE)s,%(AGGREGATE)s,%(UNIV1)s,%(FACULTY1)s,%(COURSE1)s,%(UNIV2)s,%(FACULTY2)s,%(COURSE2)s\n' % jamb)
	612	continue
	613	jamb_name = jamb.get("NAME")
	614	jamb_name.replace('>','')
	615	names = jamb_name.split()
	616	letter = names[-1][0].upper()
	617	sid = self.generateStudentId(letter)
	618	not_created = True
	619	while not_created:
	620	try:
	621	students_folder.invokeFactory('Student', sid)
	622	not_created = False
	623	except BadRequest:
	624	sid = self.generateStudentId(letter)
	625	logger.info('%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
	626	student = getattr(self,sid)
	627	student.manage_setLocalRoles(sid, ['Owner',])
	628	student.invokeFactory('StudentApplication','application')
	629	da = {'Title': 'Application Data'}
	630	da["jamb_reg_no"] = jamb.get("REG-NO")
	631	da["jamb_lastname"] = jamb_name
	632	da["jamb_sex"] = jamb.get("SEX")
	633	da["jamb_state"] = jamb.get("STATE")
	634	da["jamb_lga"] = jamb.get("LGA")
	635	da["jamb_score"] = jamb.get("AGGREGATE")
	636	da["jamb_first_cos"] = jamb.get("COURSE1")
	637	da["jamb_second_cos"] = jamb.get("COURSE2")
	638	da["jamb_first_uni"] = jamb.get("UNIV1")
	639	da["jamb_second_uni"] = jamb.get("UNIV2")
	640	app = student.application
	641	app_doc = app.getContent()
	642	app_doc.edit(mapping=da)
	643	#wftool.doActionFor(app,'open',dest_container=app)
	644	app.manage_setLocalRoles(sid, ['Owner',])
	645	student.getContent().createSubObjects()
	646	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	647	###)
	648
	649
	650	security.declareProtected(ModifyPortalContent,"importPreviousSessionStudents")###(
	651	def importPreviousSessionStudents(self):
	652	"""load and create previous session students from CSV"""
	653	import transaction
	654	import random
	655	wftool = self.portal_workflow
	656	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	657	students_folder = self.portal_url.getPortalObject().campus.students
	658	tr_count = 1
	659	total = 0
	660	#name = 'pume_results'
	661	name = 'Previous'
	662	keys = self.portal_schemas.import_student.keys()
	663	keys += self.portal_schemas.import_student_level_data.keys()
	664	not_imported = []
	665	imported = []
	666	certificates = {}
	667	logger = logging.getLogger('Students.StudentsFolder.importPreviousSessionStudents')
	668	try:
	669	records = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	670	except:
	671	logger.error('Error reading %s.csv' % name)
	672	return
	673	start = True
	674	for record in records:
	675	if start:
	676	start = False
	677	logger.info('Start loading from %s.csv' % name)
	678	false_keys = [k for k in record.keys() if k not in keys]
	679	right_keys = [k for k in record.keys() if k in keys]
	680	if false_keys:
	681	logger.info('Fields %s not in schema' % false_keys)
	682	s = ','.join(['"%s"' % k for k in right_keys])
	683	imported.append(s)
	684	not_imported.append('%s,"error"' % s)
	685	format = ','.join(['"%%(%s)s"' % k for k in right_keys])
	686	format_error = format + ',"%(error)s"'
	687	study_course = makeCertificateCode(record.get('study_course'))
	688	matric_no = record.get('matric_no')
	689	student_res = self.students_catalog(matric_no = matric_no)
	690	if student_res:
	691	record['error'] = "Student exists"
	692	not_imported.append(format_error % record)
	693	continue
	694	if study_course not in certificates.keys():
	695	cert_res = self.portal_catalog(portal_type='Certificate',
	696	id = study_course)
	697	if not cert_res:
	698	record['error'] = "No such studycourse"
	699	not_imported.append(format_error % record)
	700	continue
	701	certificates[cert_res[0].id] = cert_res[0]
	702	sid = self.generateStudentId('x',students_folder)
	703	students_folder.invokeFactory('Student', sid)
	704	#from pdb import set_trace;set_trace()
	705	record['student_id'] = sid
	706	student = getattr(self,sid)
	707	student.manage_setLocalRoles(sid, ['Owner',])
	708	student.invokeFactory('StudentApplication','application')
	709	app = student.application
	710	app_doc = app.getContent()
	711	dict = {'Title': 'Application Data'}
	712	dict["jamb_lastname"] = "%(firstname)s %(lastname)s %(middlename)s" % record
	713	r2d = [
	714	('entry_mode','entry_mode'),
	715	('sex','jamb_sex'),
	716	('jamb_score','jamb_score'),
	717	('jamb_reg_no','jamb_reg_no'),
	718	]
	719	for r,d in r2d:
	720	dict[d] = record[r]
	721	app_doc.edit(mapping=dict)
	722	app.manage_setLocalRoles(sid, ['Owner',])
	723	wftool.doActionFor(app,'close')
	724	record['sex'] = record['sex'] == 'F'
	725	student.invokeFactory('StudentPersonal','personal')
	726	dict = {}
	727	r2d = [('firstname','firstname'),
	728	('middlename','middlename'),
	729	('lastname','lastname'),
	730	('sex','sex'),
	731	('email','email'),
	732	('phone','phone'),
	733	('address','perm_address'),
	734	]
	735	for r,d in r2d:
	736	dict[d] = record[r]
	737	per = student.personal
	738	per_doc = per.getContent()
	739	per_doc.edit(mapping = dict)
	740	per.manage_setLocalRoles(sid, ['Owner',])
	741	#
	742	# Clearance
	743	#
	744	student.invokeFactory('StudentClearance','clearance')
	745	#wftool.doActionFor(student.clearance,'open')
	746	clearance = getattr(student,'clearance')
	747	dict = {'Title': 'Clearance/Eligibility Record'}
	748	clearance.getContent().edit(mapping = dict)
	749	student.clearance.manage_setLocalRoles(sid, ['Owner',])
	750	#
	751	# Study Course
	752	#
	753	student.invokeFactory('StudentStudyCourse','study_course')
	754	study_course = student.study_course
	755	dict = {}
	756	r2d = [('session','current_session'),
	757	('level','current_level'),
	758	('verdict','current_verdict'),
	759	('study_course','study_course'),
	760	]
	761	for r,d in r2d:
	762	dict[d] = record[r]
	763	study_course.getContent().edit(mapping=dict)
	764	#
	765	# Study Level
	766	#
	767	level = record['level']
	768	study_course.invokeFactory('StudentStudyLevel',level)
	769	study_level = getattr(study_course,level)
	770	dict = {}
	771	r2d = [('session','session'),
	772	('level','code'),
	773	('verdict','verdict'),
	774	]
	775	for r,d in r2d:
	776	dict[d] = record[r]
	777	study_level.getContent().edit(mapping=dict)
	778	wftool.doActionFor(student,'return')
	779	imported.append(format % record)
	780	tr_count += 1
	781	record['tr_count'] = tr_count
	782	record['total'] = total
	783	logger.info('%(total)s+%(tr_count)s: Creating Student %(student_id)s %(matric_no)s %(jamb_reg_no)s' % record)
	784	if tr_count > 1000:
	785	if len(not_imported) > 0:
	786	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	787	'\n'.join(not_imported)+'\n')
	788	not_imported = []
	789	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	790	'\n'.join(imported) + '\n')
	791	imported = []
	792	em = '%d transactions commited total %s' % (tr_count,total)
	793	transaction.commit()
	794	logger.info(em)
	795	regs = []
	796	total += tr_count
	797	tr_count = 0
	798	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	799	'\n'.join(imported))
	800	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	801	'\n'.join(not_imported))
	802	em = '%d transactions commited total %d' % (tr_count,total)
	803	logger.info(em)
	804	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	805	###)
	806
	807
	808	security.declareProtected(ModifyPortalContent,"createDEStudents")###(
	809	def createDEStudents(self):
	810	"""load Fulltime Studentdata from CSV values"""
	811	import transaction
	812	import random
	813	#from pdb import set_trace
	814	wftool = self.portal_workflow
	815	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	816	csv_d = {'jamb_reg_no': "RegNumber",
	817	'jamb_lastname': "Name",
	818	'session': "Session",
	819	'pume_tot_score': "PUDE SCORE",
	820	##'jamb_score': "JambScore",
	821	'entry_mode': "EntryMode",
	822	'jamb_sex': "Sex",
	823	'jamb_state': "State",
	824	'jamb_first_cos': "AdminCourse",
	825	'faculty': "AdminFaculty",
	826	'course_code': "AdmitCoscode",
	827	'stud_status':"AdmitStatus",
	828	'department': "AdmitDept",
	829	'jamb_lga': "LGA",
	830	'app_email': "email",
	831	'app_mobile': "PhoneNumbers",
	832	}
	833	csv_fields = [f[1] for f in csv_d.items()]
	834	tr_count = 0
	835	total = 0
	836	#name = 'pume_results'
	837	name = 'DE_Admitted'
	838	no_import = []
	839	s = ','.join(['"%s"' % fn for fn in csv_fields])
	840	no_import.append('"Error",%s' % s)
	841	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	842	no_certificate = "no certificate %s" % format
	843	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
	844	logger = logging.getLogger('Students.StudentsFolder.createDEStudents')
	845	logger.info('Start loading from %s.csv' % name)
	846	l = self.portal_catalog({'meta_type': "Certificate"})
	847	certs = {}
	848	cert_docs = {}
	849	for f in l:
	850	certs[f.getId] = f.getObject().getContent()
	851	try:
	852	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	853	except:
	854	logger.error('Error reading %s.csv' % name)
	855	return
	856	for jamb in result:
	857	jamb['Error'] = "Processing"
	858	logger.info(format % jamb)
	859	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
	860	res = self.portal_catalog({'portal_type': "StudentApplication",
	861	'SearchableText': jamb_reg_no })
	862	if res:
	863	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
	864	logger.info(em)
	865	jamb['Error'] = "Student exists"
	866	no_import.append(format % jamb)
	867	continue
	868	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
	869	if cert_id not in certs.keys():
	870	em = 'No Certificate with ID %s \n' % cert_id
	871	logger.info(em)
	872	jamb['Error'] = "No Certificate %s" % cert_id
	873	no_import.append( format % jamb)
	874	continue
	875	jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
	876	cert_doc = certs[cert_id]
	877	catalog_entry = {}
	878	catalog_entry['jamb_reg_no'] = jamb_reg_no
	879	jamb_name = jamb.get(csv_d['jamb_lastname'])
	880	jamb_name.replace('>','')
	881	jamb_name.replace('<','')
	882	names = jamb_name.split()
	883	letter = names[-1][0].upper()
	884	sid = self.generateStudentId(letter)
	885	not_created = True
	886	while not_created:
	887	try:
	888	students_folder.invokeFactory('Student', sid)
	889	not_created = False
	890	except BadRequest:
	891	sid = self.generateStudentId(letter)
	892	catalog_entry['id'] = sid
	893	tr_count += 1
	894	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
	895	student = getattr(self,sid)
	896	student.manage_setLocalRoles(sid, ['Owner',])
	897	student.invokeFactory('StudentPume','pume')
	898	dp = {'Title': 'Pume Data'}
	899	student.invokeFactory('StudentApplication','application')
	900	da = {'Title': 'Application Data'}
	901	da["jamb_lastname"] = jamb_name
	902	da_fields = ('jamb_reg_no',
	903	'jamb_sex',
	904	'entry_mode',
	905	#'jamb_score',
	906	'jamb_first_cos',
	907	'jamb_sex',
	908	'jamb_state',
	909	'jamb_lga',
	910	'app_email',
	911	'app_mobile',
	912	)
	913	for f in da_fields:
	914	da[f] = jamb.get(csv_d[f])
	915	catalog_entry['email'] = jamb.get(csv_d['app_email'])
	916	app = student.application
	917	app_doc = app.getContent()
	918	picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
	919	#import pdb;pdb.set_trace()
	920	if os.path.exists(picture):
	921	file = open(picture)
	922	if False:
	923	img = PIL.Image.open(file)
	924	img.thumbnail((150,200),
	925	resample=PIL.Image.ANTIALIAS)
	926	# We now need a buffer to write to. It can't be the same
	927	# as the inbuffer as the PNG writer will write over itself.
	928	outfile = StringIO()
	929	img.save(outfile, format=img.format)
	930	else:
	931	outfile = file.read()
	932	app_doc.manage_addFile('passport',
	933	file=outfile,
	934	title="%s.jpg" % jamb_reg_no)
	935	app.getContent().edit(mapping=da)
	936	app.manage_setLocalRoles(sid, ['Owner',])
	937	#wftool.doActionFor(app,'close')
	938	dp_fields = (
	939	#'pume_eng_score',
	940	#'pume_gen_score',
	941	'pume_tot_score',
	942	)
	943	dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
	944	pume = student.pume
	945	pume.getContent().edit(mapping=dp)
	946	#wftool.doActionFor(pume,'close')
	947	pume.manage_setLocalRoles(sid, ['Owner',])
	948	#student.getContent().createSubObjects()
	949	dp = {}
	950	if len(names) == 3:
	951	dp['firstname'] = names[0].capitalize()
	952	dp['middlename'] = names[1].capitalize()
	953	dp['lastname'] = names[2].capitalize()
	954	elif len(names) == 2:
	955	dp['firstname'] = names[0].capitalize()
	956	dp['middlename'] = ''
	957	dp['lastname'] = names[1].capitalize()
	958	else:
	959	dp['firstname'] = ''
	960	dp['middlename'] = ''
	961	dp['lastname'] = jamb_name
	962	dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
	963	catalog_entry['sex'] = dp['sex']
	964	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
	965	student.invokeFactory('StudentPersonal','personal')
	966	per = student.personal
	967	per_doc = per.getContent()
	968	per_doc.edit(mapping = dp)
	969	per.manage_setLocalRoles(sid, ['Owner',])
	970	if jamb.get(csv_d['stud_status']) == "Admitted":
	971	wftool.doActionFor(student,'pume_pass')
	972	wftool.doActionFor(student,'admit')
	973	else:
	974	wftool.doActionFor(student,'pume_fail')
	975	wftool.doActionFor(student,'reject_admission')
	976	continue
	977	#
	978	# Clearance
	979	#
	980	student.invokeFactory('StudentClearance','clearance')
	981	#wftool.doActionFor(student.clearance,'open')
	982	dp = {'Title': 'Clearance/Eligibility Record'}
	983	student.clearance.manage_setLocalRoles(sid, ['Owner',])
	984	#
	985	# Study Course
	986	#
	987	student.invokeFactory('StudentStudyCourse','study_course')
	988	study_course = student.study_course
	989	dsc = {}
	990	#from_certificate = ['title',
	991	# 'max_elect',
	992	# 'max_pass',
	993	# 'n_core',
	994	# 'nr_years',
	995	# 'probation_credits',
	996	# 'promotion_credits',
	997	# 'start_level',
	998	# ]
	999	#for f in from_certificate:
	1000	# dsc[f] = getattr(cert_doc,f)
	1001	#dsc['faculty'] = jamb.get(csv_d['faculty'])
	1002	#dsc['department'] = jamb.get(csv_d['department'])
	1003	catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
	1004	catalog_entry['department'] = jamb.get(csv_d['department'])
	1005	catalog_entry['course'] = cert_id
	1006	#catalog_entry['level'] = getattr(cert_doc,'start_level')
	1007	catalog_entry['level'] = '200'
	1008	dsc['study_course'] = cert_id
	1009	dsc['current_level'] = '200'
	1010	#dsc['entry_session'] = jamb.get(csv_d['session'])
	1011	study_course.getContent().edit(mapping=dsc)
	1012	self.students_catalog.addRecord(**catalog_entry)
	1013	if tr_count > 10:
	1014	if len(no_import) > 1:
	1015	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1016	'\n'.join(no_import)+'\n')
	1017	no_import = []
	1018	em = '%d transactions commited\n' % tr_count
	1019	transaction.commit()
	1020	logger.info(em)
	1021	total += tr_count
	1022	tr_count = 0
	1023	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1024	'\n'.join(no_import))
	1025	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1026	###)
	1027
	1028	security.declareProtected(ModifyPortalContent,"createNewStudents")###(
	1029	def createNewStudents(self):
	1030	"""load Fulltime Studentdata from CSV values"""
	1031	import transaction
	1032	import random
	1033	#from pdb import set_trace
	1034	wftool = self.portal_workflow
	1035	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	1036	csv_d = {'jamb_reg_no': "RegNumber",
	1037	'jamb_lastname': "Name",
	1038	'session': "Session",
	1039	'pume_tot_score': "PUME SCORE",
	1040	'jamb_score': "JambScore",
	1041	'jamb_sex': "Sex",
	1042	'jamb_state': "State",
	1043	## 'jamb_first_cos': "AdminCourse",
	1044	'faculty': "AdminFaculty",
	1045	'course_code': "AdmitCoscode",
	1046	'stud_status':"AdmitStatus",
	1047	'department': "AdmitDept",
	1048	'jamb_lga': "LGA",
	1049	'app_email': "email",
	1050	'app_mobile': "PhoneNumbers",
	1051	}
	1052	csv_fields = [f[1] for f in csv_d.items()]
	1053	tr_count = 0
	1054	total = 0
	1055	#name = 'pume_results'
	1056	name = 'Admitted'
	1057	no_import = []
	1058	s = ','.join(['"%s"' % fn for fn in csv_fields])
	1059	no_import.append('"Error",%s' % s)
	1060	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	1061	no_certificate = "no certificate %s" % format
	1062	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
	1063	logger = logging.getLogger('Students.StudentsFolder.createNewStudents')
	1064	logger.info('Start loading from %s.csv' % name)
	1065	l = self.portal_catalog({'meta_type': "Certificate"})
	1066	certs = {}
	1067	cert_docs = {}
	1068	for f in l:
	1069	certs[f.getId] = f.getObject().getContent()
	1070	try:
	1071	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1072	except:
	1073	logger.error('Error reading %s.csv' % name)
	1074	return
	1075	for jamb in result:
	1076	jamb['Error'] = "Processing "
	1077	logger.info(format % jamb)
	1078	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
	1079	res = self.portal_catalog({'portal_type': "StudentApplication",
	1080	'SearchableText': jamb_reg_no })
	1081	if res:
	1082	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
	1083	logger.info(em)
	1084	jamb['Error'] = "Student exists"
	1085	no_import.append(format % jamb)
	1086	continue
	1087	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
	1088	if cert_id not in certs.keys():
	1089	em = 'No Certificate with ID %s \n' % cert_id
	1090	logger.info(em)
	1091	jamb['Error'] = "No Certificate %s" % cert_id
	1092	no_import.append( format % jamb)
	1093	continue
	1094	res = self.portal_pumeresults(jamb_reg_no = jamb_reg_no)
	1095	if len(res) == 1:
	1096	self.portal_pumeresults.modifyRecord(jamb_reg_no = jamb_reg_no,
	1097	status = jamb.get(csv_d['stud_status']),
	1098	)
	1099	jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
	1100	cert_doc = certs[cert_id]
	1101	catalog_entry = {}
	1102	catalog_entry['jamb_reg_no'] = jamb_reg_no
	1103	jamb_name = jamb.get(csv_d['jamb_lastname'])
	1104	jamb_name.replace('>','')
	1105	jamb_name.replace('<','')
	1106	names = jamb_name.split()
	1107	letter = names[-1][0].upper()
	1108	sid = self.generateStudentId(letter)
	1109	not_created = True
	1110	while not_created:
	1111	try:
	1112	students_folder.invokeFactory('Student', sid)
	1113	not_created = False
	1114	except BadRequest:
	1115	sid = self.generateStudentId(letter)
	1116	catalog_entry['id'] = sid
	1117	tr_count += 1
	1118	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
	1119	student = getattr(self,sid)
	1120	student.manage_setLocalRoles(sid, ['Owner',])
	1121	student.invokeFactory('StudentPume','pume')
	1122	dp = {'Title': 'Pume Data'}
	1123	student.invokeFactory('StudentApplication','application')
	1124	da = {'Title': 'Application Data'}
	1125	da["jamb_lastname"] = jamb_name
	1126	da_fields = ('jamb_reg_no',
	1127	'jamb_sex',
	1128	#'jamb_state',
	1129	'jamb_score',
	1130	## 'jamb_first_cos',
	1131	'jamb_sex',
	1132	'jamb_state',
	1133	'jamb_lga',
	1134	'app_email',
	1135	'app_mobile',
	1136	)
	1137	for f in da_fields:
	1138	da[f] = jamb.get(csv_d[f])
	1139	catalog_entry['email'] = jamb.get(csv_d['app_email'])
	1140	app = student.application
	1141	app_doc = app.getContent()
	1142	#import pdb;pdb.set_trace()
	1143	picture ="%s/import/pictures/%s.jpg" % (i_home,picture_id)
	1144	if os.path.exists(picture):
	1145	file = open(picture)
	1146	if False:
	1147	img = PIL.Image.open(file)
	1148	img.thumbnail((150,200),
	1149	resample=PIL.Image.ANTIALIAS)
	1150	# We now need a buffer to write to. It can't be the same
	1151	# as the inbuffer as the PNG writer will write over itself.
	1152	outfile = StringIO()
	1153	img.save(outfile, format=img.format)
	1154	else:
	1155	outfile = file.read()
	1156	app_doc.manage_addFile('passport',
	1157	file=outfile,
	1158	title="%s.jpg" % jamb_reg_no)
	1159	app.getContent().edit(mapping=da)
	1160	app.manage_setLocalRoles(sid, ['Owner',])
	1161	#wftool.doActionFor(app,'close')
	1162	dp_fields = (
	1163	#'pume_eng_score',
	1164	#'pume_gen_score',
	1165	'pume_tot_score',
	1166	)
	1167	dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
	1168	pume = student.pume
	1169	pume.getContent().edit(mapping=dp)
	1170	#wftool.doActionFor(pume,'close')
	1171	pume.manage_setLocalRoles(sid, ['Owner',])
	1172	#student.getContent().createSubObjects()
	1173	dp = {}
	1174	if len(names) == 3:
	1175	dp['firstname'] = names[0].capitalize()
	1176	dp['middlename'] = names[1].capitalize()
	1177	dp['lastname'] = names[2].capitalize()
	1178	elif len(names) == 2:
	1179	dp['firstname'] = names[0].capitalize()
	1180	dp['middlename'] = ''
	1181	dp['lastname'] = names[1].capitalize()
	1182	else:
	1183	dp['firstname'] = ''
	1184	dp['middlename'] = ''
	1185	dp['lastname'] = jamb_name
	1186	dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
	1187	catalog_entry['sex'] = dp['sex']
	1188	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
	1189	student.invokeFactory('StudentPersonal','personal')
	1190	per = student.personal
	1191	per_doc = per.getContent()
	1192	per_doc.edit(mapping = dp)
	1193	per.manage_setLocalRoles(sid, ['Owner',])
	1194	if jamb.get(csv_d['stud_status']) == "Admitted":
	1195	wftool.doActionFor(student,'pume_pass')
	1196	wftool.doActionFor(student,'admit')
	1197	else:
	1198	wftool.doActionFor(student,'pume_fail')
	1199	wftool.doActionFor(student,'reject_admission')
	1200	continue
	1201	#
	1202	# Clearance
	1203	#
	1204	student.invokeFactory('StudentClearance','clearance')
	1205	#wftool.doActionFor(student.clearance,'open')
	1206	dp = {'Title': 'Clearance/Eligibility Record'}
	1207	student.clearance.manage_setLocalRoles(sid, ['Owner',])
	1208	#
	1209	# Study Course
	1210	#
	1211	student.invokeFactory('StudentStudyCourse','study_course')
	1212	study_course = student.study_course
	1213	dsc = {}
	1214	#from_certificate = ['title',
	1215	# 'max_elect',
	1216	# 'max_pass',
	1217	# 'n_core',
	1218	# 'nr_years',
	1219	# 'probation_credits',
	1220	# 'promotion_credits',
	1221	# 'start_level',
	1222	# ]
	1223	#for f in from_certificate:
	1224	# dsc[f] = getattr(cert_doc,f)
	1225	#dsc['faculty'] = jamb.get(csv_d['faculty'])
	1226	#dsc['department'] = jamb.get(csv_d['department'])
	1227	catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
	1228	catalog_entry['department'] = jamb.get(csv_d['department'])
	1229	catalog_entry['course'] = cert_id
	1230	#catalog_entry['level'] = getattr(cert_doc,'start_level')
	1231	catalog_entry['level'] = '100'
	1232	dsc['study_course'] = cert_id
	1233	#dsc['entry_level'] = '100'
	1234	#dsc['entry_session'] = jamb.get(csv_d['session'])
	1235	study_course.getContent().edit(mapping=dsc)
	1236	self.students_catalog.addRecord(**catalog_entry)
	1237	if tr_count > 10:
	1238	if len(no_import) > 0:
	1239	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1240	'\n'.join(no_import) + "\n")
	1241	no_import = []
	1242	em = '%d transactions commited\n' % tr_count
	1243	transaction.commit()
	1244	logger.info(em)
	1245	total += tr_count
	1246	tr_count = 0
	1247	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1248	'\n'.join(no_import))
	1249	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1250	###)
	1251
	1252
	1253	security.declareProtected(ModifyPortalContent,"fixVerdicts")###(
	1254	def fixVerdicts(self,csv_file=None):
	1255	"""fix wrong uploaded verdicts"""
	1256	import transaction
	1257	import random
	1258	wftool = self.portal_workflow
	1259	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1260	students_folder = self.portal_url.getPortalObject().campus.students
	1261	tr_count = 1
	1262	total = 0
	1263	if csv_file is None:
	1264	name = 'Verdicts'
	1265	else:
	1266	name = csv_file
	1267	st_cat = self.students_catalog
	1268	no_import = []
	1269	verdicts_voc = self.portal_vocabularies.verdicts
	1270	rverdicts = {}
	1271	for k,v in verdicts_voc.items():
	1272	rverdicts[v.upper()] = k
	1273	rverdicts['STUDENT ON PROBATION'] = 'C'
	1274	logger = logging.getLogger('Students.StudentsFolder.fixVerdicts')
	1275	try:
	1276	verdicts = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1277	except:
	1278	logger.error('Error reading %s.csv' % name)
	1279	return
	1280	start = True
	1281	#import pdb;pdb.set_trace()
	1282	for verdict in verdicts:
	1283	if start:
	1284	start = False
	1285	logger.info('Start loading from %s.csv' % name)
	1286	s = ','.join(['"%s"' % fn for fn in verdict.keys()])
	1287	no_import.append('%s,"Error"' % s)
	1288	format = ','.join(['"%%(%s)s"' % fn for fn in verdict.keys()])
	1289	format_error = format + ',"%(Error)s"'
	1290	matric_no = verdict.get('MAT NO')
	1291	if not matric_no:
	1292	continue
	1293	matric_no = matric_no.upper()
	1294	if matric_no == '':
	1295	continue
	1296	verdict_code = rverdicts.get(verdict.get('CATEGORY'),None)
	1297	if verdict_code is None:
	1298	continue
	1299	sres = st_cat(matric_no = matric_no)
	1300	if sres:
	1301	student_id = sres[0].id
	1302	student_obj = getattr(students_folder,student_id,None)
	1303	if student_obj:
	1304	study_course = getattr(student_obj,'study_course',None)
	1305	if study_course is None:
	1306	verdict['Error'] = "Student did not yet log in"
	1307	no_import.append( format_error % verdict)
	1308	continue
	1309	st_cat.modifyRecord(id = student_id,
	1310	verdict=verdict_code)
	1311
	1312	dsc = {}
	1313	dsc['current_verdict'] = verdict_code
	1314	study_course.getContent().edit(mapping=dsc)
	1315	else:
	1316	verdict['Error'] = "Not found in students_catalog"
	1317	no_import.append( format_error % verdict)
	1318	continue
	1319	tr_count += 1
	1320	if tr_count > 1000:
	1321	if len(no_import) > 0:
	1322	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1323	'\n'.join(no_import) + '\n')
	1324	no_import = []
	1325	em = '%d transactions commited total %s\n' % (tr_count,total)
	1326	transaction.commit()
	1327	regs = []
	1328	logger.info(em)
	1329	total += tr_count
	1330	tr_count = 0
	1331	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1332	'\n'.join(no_import))
	1333	total += tr_count
	1334	em = '%d total transactions commited' % (total)
	1335	logger.info(em)
	1336	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1337	###)
	1338
	1339
	1340	security.declareProtected(ModifyPortalContent,"fixAllEntryModeForReturning")###(
	1341	def fixAllEntryModeForReturning(self):
	1342	"read all Returning*.csv"
	1343	ipath = "%s/import/" % i_home
	1344	names = os.listdir(ipath)
	1345	for name in names:
	1346	head,tail = os.path.splitext(name)
	1347	if head.startswith('Returning')\
	1348	and tail == '.csv'\
	1349	and name.find('imported') < 0:
	1350	self.fixEntryModeForReturning(csv_file=head)
	1351	###)
	1352
	1353	security.declareProtected(ModifyPortalContent,"fixEntryModeForReturning")###(
	1354	def fixEntryModeForReturning(self,csv_file=None):
	1355	"""load Returning Studentdata from CSV values"""
	1356	import transaction
	1357	import random
	1358	wftool = self.portal_workflow
	1359	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1360	students_folder = self.portal_url.getPortalObject().campus.students
	1361	tr_count = 1
	1362	total = 0
	1363	if csv_file is None:
	1364	name = 'Returning'
	1365	else:
	1366	name = csv_file
	1367	table = self.returning_import
	1368	st_cat = self.students_catalog
	1369	no_import = []
	1370	logger = logging.getLogger('Students.StudentsFolder.fixEntryModeForReturning')
	1371	try:
	1372	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1373	except:
	1374	logger.error('Error reading %s.csv' % name)
	1375	return
	1376	start = True
	1377	for student in returning:
	1378	if start:
	1379	start = False
	1380	logger.info('Start loading from %s.csv' % name)
	1381	s = ','.join(['"%s"' % fn for fn in student.keys()])
	1382	no_import.append('%s,"Error"' % s)
	1383	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
	1384	format_error = format + ',"%(Error)s"'
	1385	matric_no = student.get('matric_no')
	1386	if not matric_no:
	1387	continue
	1388	matric_no = matric_no.upper()
	1389	student['matric_no'] = matric_no
	1390	if matric_no == '':
	1391	continue
	1392	if not table(matric_no = matric_no):
	1393	student['Error'] = "Not imported yet"
	1394	no_import.append( format_error % student)
	1395	continue
	1396	student_id = None
	1397	app = None
	1398	per = None
	1399	if st_cat(matric_no = matric_no):
	1400	student_id = st_cat(matric_no = matric_no)[0].id
	1401	student_obj = getattr(students_folder,student_id,None)
	1402	if student_obj:
	1403	app = getattr(student_obj,'application',None)
	1404	if app is not None:
	1405	app_doc = app.getContent()
	1406	per = getattr(student_obj,'personal',None)
	1407	if per is not None:
	1408	per_doc = per.getContent()
	1409	student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
	1410	student['Permanent_Address'] = perm_address = student.get('Permanent Address')
	1411	#import pdb;pdb.set_trace()
	1412	if not entry_mode:
	1413	student['Error'] = "'Mode of Entry' empty"
	1414	no_import.append( format_error % student)
	1415	continue
	1416	try:
	1417	table.modifyRecord(matric_no = matric_no,
	1418	Mode_of_Entry = entry_mode,
	1419	Permanent_Address = perm_address)
	1420	except KeyError:
	1421	student['Error'] = "Not found in returning_import"
	1422	no_import.append( format_error % student)
	1423	continue
	1424	if student_id is not None:
	1425	try:
	1426	st_cat.modifyRecord(id = student_id,
	1427	entry_mode=entry_mode)
	1428	except KeyError:
	1429	student['Error'] = "Not found in students_catalog"
	1430	no_import.append( format_error % student)
	1431	continue
	1432	if app is not None:
	1433	da = {}
	1434	da['entry_mode'] = entry_mode
	1435	app_doc.edit(mapping=da)
	1436	if per is not None:
	1437	dp = {}
	1438	dp['perm_address'] = perm_address
	1439	per_doc.edit(mapping=dp)
	1440	tr_count += 1
	1441	if tr_count > 1000:
	1442	if len(no_import) > 0:
	1443	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1444	'\n'.join(no_import) + '\n')
	1445	no_import = []
	1446	em = '%d transactions commited total %s\n' % (tr_count,total)
	1447	transaction.commit()
	1448	regs = []
	1449	logger.info(em)
	1450	total += tr_count
	1451	tr_count = 0
	1452	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1453	'\n'.join(no_import))
	1454	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1455	###)
	1456
	1457	security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
	1458	def loadPumeResultsFromCSV(self):
	1459	"""load Fulltime Studentdata from CSV values into pumeresults catalog"""
	1460	import transaction
	1461	import random
	1462	## csv_d = {'jamb_reg_no': "RegNumber", ###(
	1463	## 'status': "Admission Status",
	1464	## 'name': "Name",
	1465	## 'score': "Score",
	1466	## 'sex': "Sex",
	1467	## 'faculty': "Faculty",
	1468	## 'department': "Dept",
	1469	## 'course': "Course",
	1470	## 'course_code_org': "Course Code",
	1471	## }
	1472	###)
	1473	## csv_d = {'jamb_reg_no': "JAMBRegno",
	1474	## 'name': "Name",
	1475	## 'score': "Score",
	1476	## 'sex': "Sex",
	1477	## 'course': "Course",
	1478	## 'faculty': "Faculty",
	1479	## 'department': "Dept",
	1480	## 'course_code_org': "Course Code",
	1481	## 'status': "Admission Status",
	1482	## 'result_type': None,
	1483	## }
	1484
	1485	csv_d = {'jamb_reg_no': "reg_no",
	1486	'name': "fullname",
	1487	'score': "pume_score",
	1488	'sex': "sex",
	1489	'course': "study_course",
	1490	'course_code_org': "study_course",
	1491	'status': "admission_status",
	1492	'result_type': "entry_mode",
	1493	}
	1494
	1495	csv_fields = [f[1] for f in csv_d.items() if f[1]]
	1496	tr_count = 0
	1497	total = 0
	1498	#name = 'pup_new'
	1499	#name = 'pup_update'
	1500	name = 'Admitted_update'
	1501	update = name.endswith('update')
	1502	no_import = []
	1503	ok_import = []
	1504	ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
	1505	no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
	1506	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1507	ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
	1508	#open(ok_import_name,"w").write('\n'.join(no_import))
	1509	no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
	1510	#open(no_import_name,"w").write('\n'.join(no_import))
	1511	logger = logging.getLogger('Students.loadPumeResultsFromCSV')
	1512	starttime = DateTime.now()
	1513	logger.info('Start loading from %s.csv' % name)
	1514	try:
	1515	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1516	except:
	1517	logger.error('Error reading %s.csv' % name)
	1518	return
	1519	pume = self.portal_pumeresults
	1520	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	1521	import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
	1522	eduplicate = '%s,"duplicate"' % format
	1523	eoriginal = '%s,"original"' % format
	1524	einvalidjamb = '%s,"invalid JambRegNo"' % format
	1525	added = 'added ,%s' % format
	1526	#from pdb import set_trace;set_trace()
	1527	for jamb in result:
	1528	if not jamb.get(csv_d['score']):
	1529	logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no']))
	1530	continue
	1531	dict = {}
	1532	for f,fn in csv_d.items():
	1533	dict[f] = jamb.get(csv_d[f])
	1534	dict['result_type'] = 'CEST'
	1535	jnr = jamb.get(csv_d['jamb_reg_no'])
	1536	#if not checkJambNo(jnr):
	1537	# logger.info(einvalidjamb % jamb)
	1538	# dd = {}
	1539	# for f,fn in csv_d.items():
	1540	# dd[fn] = getattr(data,f)
	1541	# no_import.append(eduplicate % dd)
	1542	# no_import.append(eduplicate % jamb)
	1543	# continue
	1544	res = pume(jamb_reg_no=jnr)
	1545	if len(res) > 0:
	1546	if update:
	1547	try:
	1548	pume.modifyRecord(**dict)
	1549	# Can not happen, but anyway...
	1550	except ValueError:
	1551	logger.info(eduplicate % jamb)
	1552	continue
	1553	# Can not happen, but anyway...
	1554	except KeyError:
	1555	pume.addRecord(**dict)
	1556	logger.info(added % jamb)
	1557	continue
	1558	else:
	1559	data = res[0]
	1560	if data.name != jamb.get(csv_d['name']):
	1561	#set_trace()
	1562	logger.info(eduplicate % jamb)
	1563	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
	1564	#logger.info(em)
	1565	dd = {}
	1566	for f,fn in csv_d.items():
	1567	dd[fn] = getattr(data,f)
	1568	no_import.append(eoriginal % dd)
	1569	no_import.append(eduplicate % jamb)
	1570	continue
	1571	else:
	1572	try:
	1573	pume.addRecord(**dict)
	1574	ok_import.append(import_format % dict)
	1575	except ValueError:
	1576	logger.info(eduplicate % jamb)
	1577	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
	1578	#logger.info(em)
	1579	no_import.append(eduplicate % jamb)
	1580	logger.info('End loading from %s.csv' % name)
	1581	if len(no_import) > 1:
	1582	open(no_import_name,"w+").write('\n'.join(no_import))
	1583	open(ok_import_name,"w+").write('\n'.join(ok_import))
	1584	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1585	###)
	1586
	1587	security.declareProtected(ModifyPortalContent,"createStudents")###(
	1588	def createStudents(self):
	1589	"""
	1590	load addmitted Studentdata from CSV values and create Studentobjects.
	1591	This is the current method to create new addmitted Students.
	1592	Before running the eventservice for the students_catalog must be disabled.
	1593	"""
	1594	import transaction
	1595	import random
	1596	#from pdb import set_trace
	1597	wftool = self.portal_workflow
	1598	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	1599	students_folder = self.portal_url.getPortalObject().campus.students
	1600	levels = {'ume_ft':'100',
	1601	'de_ft': '200',
	1602	'ug_pt': '100',
	1603	'de_pt': '200',
	1604	'pg_ft': '700',
	1605	'pg_pt': '700',
	1606	'dp_pt': '100',
	1607	'dp_ft': '100',
	1608	}
	1609	csv_d = {'jamb_reg_no': "reg_no",
	1610	'entry_mode': 'entry_mode',
	1611	'jamb_firstname': "firstname",
	1612	'jamb_middlename': "middlename",
	1613	'jamb_lastname': "lastname",
	1614	'jamb_sex': "sex",
	1615	'jamb_state': "state",
	1616	'birthday': "date_of_birth",
	1617	'app_email': "email",
	1618	'study_course': "study_course",
	1619	'perm_address': "address",
	1620	'admission_status': "admission_status",
	1621	}
	1622	csv_fields = [f[1] for f in csv_d.items()]
	1623	tr_count = 0
	1624	total = 0
	1625	#name = 'pume_results'
	1626	name = 'Admitted_update'
	1627	no_import = []
	1628	s = ','.join(['"%s"' % fn for fn in csv_fields])
	1629	no_import.append('"Error",%s' % s)
	1630	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	1631	no_certificate = "no certificate %s" % format
	1632	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
	1633	logger = logging.getLogger('Students.StudentsFolder.createStudents')
	1634	logger.info('Start loading from %s.csv' % name)
	1635	certs = {}
	1636	try:
	1637	results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1638	except:
	1639	logger.error('Error reading %s.csv' % name)
	1640	return
	1641	for result in results:
	1642	if not result.get(csv_d['admission_status']).startswith('Admitted'):
	1643	continue
	1644	#result['Error'] = "Processing "
	1645	#logger.info(format % result)
	1646	jamb_reg_no = result.get(csv_d['jamb_reg_no'])
	1647	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
	1648	if res:
	1649	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
	1650	logger.info(em)
	1651	result['Error'] = "Student exists"
	1652	no_import.append(format % result)
	1653	continue
	1654	cert_id = makeCertificateCode(result.get(csv_d['study_course']))
	1655	if cert_id not in certs.keys():
	1656	res = self.portal_catalog(meta_type = "Certificate",id = cert_id)
	1657	if not res:
	1658	em = 'No Certificate with ID %s \n' % cert_id
	1659	logger.info(em)
	1660	result['Error'] = "No Certificate %s" % cert_id
	1661	no_import.append( format % result)
	1662	continue
	1663	cert = res[0]
	1664	cert_path = cert.getPath().split('/')
	1665	certificate = certs[cert_id] = {'faculty': cert_path[-4],
	1666	'department': cert_path[-3]}
	1667	cert_doc = certs[cert_id]
	1668	catalog_entry = {}
	1669	catalog_entry['jamb_reg_no'] = jamb_reg_no
	1670	firstname = result.get(csv_d['jamb_firstname'])
	1671	middlename = result.get(csv_d['jamb_middlename'])
	1672	lastname = result.get(csv_d['jamb_lastname'])
	1673	if len(firstname) < 3\
	1674	and len(middlename) < 3\
	1675	and len(lastname) < 3:
	1676	em = 'Student Names to short \n'
	1677	logger.info(em)
	1678	result['Error'] = "Names to short"
	1679	no_import.append( format % result)
	1680	continue
	1681	perm_address = result.get(csv_d['perm_address'])
	1682	sid = self.generateStudentId('x')
	1683	students_folder.invokeFactory('Student', sid)
	1684	catalog_entry['id'] = sid
	1685	tr_count += 1
	1686	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
	1687	student = getattr(self,sid)
	1688	student.manage_setLocalRoles(sid, ['Owner',])
	1689	student.invokeFactory('StudentApplication','application')
	1690	da = {'Title': 'Application Data'}
	1691	da["jamb_firstname"] = firstname
	1692	da["jamb_middlename"] = middlename
	1693	da["jamb_lastname"] = lastname
	1694	catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0]
	1695	catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')
	1696	da_fields = ('jamb_reg_no',
	1697	'jamb_sex',
	1698	'jamb_state',
	1699	'entry_mode',
	1700	'app_email',
	1701	)
	1702	for f in da_fields:
	1703	da[f] = result.get(csv_d[f])
	1704	catalog_entry['email'] = da['app_email']
	1705	catalog_entry['entry_mode'] = da['entry_mode']
	1706	app = student.application
	1707	app_doc = app.getContent()
	1708	app.getContent().edit(mapping=da)
	1709	picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
	1710	app.manage_setLocalRoles(sid, ['Owner',])
	1711
	1712	picture_id = da['jamb_reg_no'].replace('/','_')
	1713	file = None
	1714	for ext in ('jpg','JPG'):
	1715	picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)
	1716	if os.path.exists(picture):
	1717	file = open(picture)
	1718	break
	1719	if file is not None:
	1720
	1721	## file conversion does not work
	1722	#img = PIL.Image.open(file)
	1723	#img.thumbnail((150,200),
	1724	# resample=PIL.Image.ANTIALIAS)
	1725	#outfile = StringIO()
	1726	#img.save(outfile, format=img.format)
	1727
	1728	outfile = file.read()
	1729	app_doc.manage_addFile('passport',
	1730	file=outfile,
	1731	title="%s.jpg" % jamb_reg_no)
	1732	#wftool.doActionFor(app,'close')
	1733	dp = {}
	1734	dp['firstname'] = firstname
	1735	dp['middlename'] = middlename
	1736	dp['lastname'] = lastname
	1737	dp['email'] = da['app_email']
	1738	dp['sex'] = sex
	1739	dp['perm_address'] = perm_address
	1740	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
	1741	student.invokeFactory('StudentPersonal','personal')
	1742	per = student.personal
	1743	per_doc = per.getContent()
	1744	per_doc.edit(mapping = dp)
	1745	per.manage_setLocalRoles(sid, ['Owner',])
	1746	wftool.doActionFor(student,'pume_pass')
	1747	wftool.doActionFor(student,'admit')
	1748	#
	1749	# Clearance
	1750	#
	1751	student.invokeFactory('StudentClearance','clearance')
	1752	#wftool.doActionFor(student.clearance,'open')
	1753	clearance = student.clearance
	1754	dc = {'Title': 'Clearance/Eligibility Record'}
	1755	clearance = student.clearance
	1756	date_str = result.get(csv_d['birthday'])
	1757	try:
	1758	date = DateTime.DateTime(date_str)
	1759	except:
	1760	#import pdb;pdb.set_trace()
	1761	date = None
	1762	dc['birthday'] = date
	1763	clearance.getContent().edit(mapping=dc)
	1764	clearance.manage_setLocalRoles(sid, ['Owner',])
	1765	#
	1766	# Study Course
	1767	#
	1768	student.invokeFactory('StudentStudyCourse','study_course')
	1769	study_course = student.study_course
	1770	dsc = {}
	1771	catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100')
	1772	#catalog_entry['level'] = dsc['current_level'] = '100' # Attention: not for DE students
	1773	catalog_entry['session'] = dsc['current_session'] = da['entry_session']
	1774	catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']
	1775	catalog_entry['course'] = dsc['study_course'] = cert_id
	1776	catalog_entry['faculty'] = certificate['faculty']
	1777	catalog_entry['department'] = certificate['department']
	1778	catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'
	1779	catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)
	1780	study_course.getContent().edit(mapping=dsc)
	1781	#import pdb;pdb.set_trace()
	1782	self.students_catalog.addRecord(**catalog_entry)
	1783	if tr_count > 100:
	1784	if len(no_import) > 0:
	1785	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1786	'\n'.join(no_import) + "\n")
	1787	no_import = []
	1788	em = '%d transactions commited\n' % tr_count
	1789	transaction.commit()
	1790	logger.info(em)
	1791	total += tr_count
	1792	tr_count = 0
	1793	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
	1794	'\n'.join(no_import))
	1795	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1796	###)
	1797
	1798
	1799	security.declareProtected(ModifyPortalContent,"importReturningStudents")###(
	1800	def importReturningStudents(self):
	1801	"""load Returning Studentdata from CSV values"""
	1802	import transaction
	1803	import random
	1804	#from pdb import set_trace
	1805	wftool = self.portal_workflow
	1806	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1807	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	1808	students_folder = self.portal_url.getPortalObject().campus.students
	1809	tr_count = 1
	1810	total = 0
	1811	#name = 'pume_results'
	1812	name = 'Returning'
	1813	table = self.returning_import
	1814	no_import = []
	1815	imported = []
	1816	logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')
	1817	try:
	1818	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	1819	except:
	1820	logger.error('Error reading %s.csv' % name)
	1821	return
	1822	l = self.portal_catalog({'meta_type': "Certificate"})
	1823	certs = {}
	1824	cert_docs = {}
	1825	for f in l:
	1826	certs[f.getId] = f.getObject().getContent()
	1827	start = True
	1828	res = table()
	1829	regs = []
	1830	if len(res) > 0:
	1831	regs = [s.matric_no for s in res]
	1832	#import pdb;pdb.set_trace()
	1833	for student in returning:
	1834	if start:
	1835	start = False
	1836	logger.info('Start loading from %s.csv' % name)
	1837	s = ','.join(['"%s"' % fn for fn in student.keys()])
	1838	imported.append(s)
	1839	no_import.append('%s,"Error"' % s)
	1840	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
	1841	format_error = format + ',"%(Error)s"'
	1842	no_certificate = "no certificate %s" % format
	1843	student['matric_no'] = matric_no = student.get('matric_no').upper()
	1844	student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
	1845	student['Permanent_Address'] = perm_address = student.get('Permanent Address')
	1846	if matric_no == '':
	1847	student['Error'] = "Empty matric_no"
	1848	no_import.append( format_error % student)
	1849	continue
	1850	if matric_no in regs or self.returning_import(matric_no = matric_no):
	1851	student['Error'] = "Duplicate"
	1852	no_import.append( format_error % student)
	1853	continue
	1854	cert_id = makeCertificateCode(student.get('Coursemajorcode'))
	1855	if cert_id not in certs.keys():
	1856	student['Error'] = "No Certificate %s" % cert_id
	1857	no_import.append( format_error % student)
	1858	continue
	1859	try:
	1860	table.addRecord(**student)
	1861	except ValueError:
	1862	student['Error'] = "Duplicate"
	1863	no_import.append( format_error % student)
	1864	continue
	1865	regs.append(student.get('matric_no'))
	1866	imported.append(format % student)
	1867	tr_count += 1
	1868	if tr_count > 1000:
	1869	if len(no_import) > 0:
	1870	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1871	'\n'.join(no_import) + '\n')
	1872	no_import = []
	1873	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	1874	'\n'.join(no_import) + "\n")
	1875	imported = []
	1876	em = '%d transactions commited total %s\n' % (tr_count,total)
	1877	transaction.commit()
	1878	regs = []
	1879	logger.info(em)
	1880	total += tr_count
	1881	tr_count = 0
	1882	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	1883	'\n'.join(imported))
	1884	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	1885	'\n'.join(no_import))
	1886	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	1887	###)
	1888
	1889	security.declareProtected(ModifyPortalContent,"fixAllNames")###(
	1890	def fixAllNames(self):
	1891	"fix all students names"
	1892	import transaction
	1893	response = self.REQUEST.RESPONSE
	1894	logger = logging.getLogger('fixAllNames')
	1895	logger.info('Start')
	1896	students = self.portal_catalog(portal_type='Student')
	1897	count = 0
	1898	total = 0
	1899	for student in students:
	1900	scat_res = self.students_catalog(id = student.getId)
	1901	if not scat_res:
	1902	self.students_catalog.addRecord(id = student.getId)
	1903	scat_res = self.students_catalog(id = student.getId)
	1904	student_entry = scat_res[0]
	1905	old_new = self.fixName(student,student_entry)
	1906	count += 1
	1907	response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))
	1908	if count > 2000:
	1909	transaction.commit()
	1910	logger.info("%d transactions commited" % count)
	1911	total += count
	1912	count = 0
	1913	###)
	1914
	1915	security.declareProtected(ModifyPortalContent,"fixName")###(
	1916	def fixName(self,student_brain, student_entry):
	1917	"fix the name of a student"
	1918	fix = "first"
	1919	if student_entry.get('name_fixed',None) == fix:
	1920	return "Name already fixed"
	1921	student_id = student_entry.id
	1922	new_student = student_entry.jamb_reg_no.startswith('6')
	1923	student_obj = student_brain.getObject()
	1924	personal = getattr(student_obj,'personal',None)
	1925	invalid = ''
	1926	if personal is None:
	1927	return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")
	1928	per_doc = personal.getContent()
	1929	old_first = per_doc.firstname
	1930	old_middle = per_doc.middlename
	1931	old_last = per_doc.lastname
	1932	new_first = ''
	1933	new_middle = ''
	1934	new_last = ''
	1935	if new_student:
	1936	if not old_first and not old_middle and old_last:
	1937	new_names = [n.capitalize() for n in old_last.split()]
	1938	if len(new_names) > 1:
	1939	old_first = new_names[0]
	1940	old_last = new_names[-1]
	1941	old_middle = ' '.join(new_names[1:-1])
	1942	else:
	1943	old_last = new_names[0]
	1944	old_first = ''
	1945	old_middle = ''
	1946	if old_first:
	1947	new_first = old_first
	1948	if old_middle:
	1949	new_middle = old_middle
	1950	if old_last:
	1951	new_last = old_last
	1952	if old_first.find('<') != -1 or\
	1953	old_first.find('>') != -1 or\
	1954	old_middle.find('<') != -1 or\
	1955	old_middle.find('>') != -1 or\
	1956	old_last.find('<') != -1 or\
	1957	old_last.find('>') != -1:
	1958	invalid = "invalid characters"
	1959	else:
	1960	new_first = old_first
	1961	if new_first.strip() == '-':
	1962	new_first = ''
	1963	new_middle = old_middle
	1964	if new_middle.strip() == '-':
	1965	new_middle = ''
	1966	new_last = old_last
	1967	if new_last.strip() == '-':
	1968	new_last = ''
	1969	name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()
	1970	if new_student:
	1971	text = "New"
	1972	else:
	1973	text = "Returning"
	1974	old_new = '"%s","%s","%s","%s"' % (invalid,text,
	1975	student_entry.name,
	1976	name)
	1977	if not invalid:
	1978	self.students_catalog.modifyRecord(id = student_id,
	1979	name_fixed = fix,
	1980	name = name)
	1981	per_doc.edit(mapping = {'firstname' : new_first,
	1982	'middlename' : new_middle,
	1983	'lastname' : new_last,
	1984	})
	1985	return old_new
	1986	###)
	1987
	1988	security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(
	1989	def updateReturningStudents(self):
	1990	"""load and overwrite Returning Student Data from CSV values"""
	1991	import transaction
	1992	import random
	1993	#from pdb import set_trace
	1994	wftool = self.portal_workflow
	1995	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	1996	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	1997	students_folder = self.portal_url.getPortalObject().campus.students
	1998	tr_count = 1
	1999	total = 0
	2000	#name = 'pume_results'
	2001	name = 'Returning_update'
	2002	table = self.returning_import
	2003	no_import = []
	2004	imported = []
	2005	logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')
	2006	try:
	2007	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	2008	except:
	2009	logger.error('Error reading %s.csv' % name)
	2010	return
	2011	l = self.portal_catalog({'meta_type': "Certificate"})
	2012	certs = {}
	2013	cert_docs = {}
	2014	for f in l:
	2015	certs[f.getId] = f.getObject().getContent()
	2016	start = True
	2017	res = table()
	2018	regs = []
	2019	if len(res) > 0:
	2020	regs = [s.matric_no for s in res]
	2021	for student in returning:
	2022	if start:
	2023	start = False
	2024	logger.info('Start loading from %s.csv' % name)
	2025	s = ','.join(['"%s"' % fn for fn in student.keys()])
	2026	imported.append(s)
	2027	no_import.append('%s,"Error"' % s)
	2028	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
	2029	format_error = format + ',"%(Error)s"'
	2030	no_certificate = "no certificate %s" % format
	2031	matric_no = student.get('matric_no').upper()
	2032	student['matric_no'] = matric_no
	2033	if matric_no == '':
	2034	student['Error'] = "Empty matric_no"
	2035	no_import.append( format_error % student)
	2036	continue
	2037	# if matric_no in regs or self.returning_import(matric_no = matric_no):
	2038	# student['Error'] = "Duplicate"
	2039	# no_import.append( format_error % student)
	2040	# continue
	2041	# cert_id = makeCertificateCode(student.get('Coursemajorcode'))
	2042	# if cert_id not in certs.keys():
	2043	# student['Error'] = "No Certificate %s" % cert_id
	2044	# no_import.append( format_error % student)
	2045	# continue
	2046	try:
	2047	table.modifyRecord(**student)
	2048	except KeyError:
	2049	#import pdb;pdb.set_trace()
	2050	student['Error'] = "no Student found to update"
	2051	no_import.append( format_error % student)
	2052	continue
	2053	#s = self.students_catalog(matric_no=matric_no)
	2054	#if s:
	2055	# level = "%s" % (int(student.get('Level')) + 100)
	2056	# self.students_catalog.modifyRecord(id = s[0].id,
	2057	# level=level)
	2058
	2059	regs.append(student.get('matric_no'))
	2060	imported.append(format % student)
	2061	tr_count += 1
	2062	if tr_count > 1000:
	2063	if len(no_import) > 0:
	2064	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	2065	'\n'.join(no_import) + '\n')
	2066	no_import = []
	2067	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	2068	'\n'.join(no_import) + "\n")
	2069	imported = []
	2070	em = '%d transactions commited total %s\n' % (tr_count,total)
	2071	transaction.commit()
	2072	regs = []
	2073	logger.info(em)
	2074	total += tr_count
	2075	tr_count = 0
	2076	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
	2077	'\n'.join(imported))
	2078	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
	2079	'\n'.join(no_import))
	2080	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	2081	###)
	2082
	2083	security.declareProtected(ModifyPortalContent,"exportStudents")###(
	2084	def exportStudents(self):
	2085	"""export Studentsdata to a file"""
	2086	member = self.portal_membership.getAuthenticatedMember()
	2087	logger = logging.getLogger('Students.exportStudents')
	2088	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	2089	students_folder = self.portal_url.getPortalObject().campus.students
	2090	export_file = "%s/export/students_%s.csv" % (i_home,current)
	2091
	2092	from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp
	2093	aq_students = self.students_catalog.evalAdvancedQuery
	2094	toexport = {'students_catalog': ("id",
	2095	"matric_no",
	2096	"jamb_reg_no",
	2097	"name",
	2098	"review_state",
	2099	"entry_session",
	2100	"session",
	2101	"entry_mode",
	2102	"mode",
	2103	"verdict",
	2104	"sex",
	2105	"email",
	2106	"phone",
	2107	"faculty",
	2108	"department",
	2109	"course",
	2110	"level",
	2111	),
	2112	'personal': ('firstname',
	2113	'middlename',
	2114	'lastname',
	2115	'perm_address',
	2116	),
	2117	'clearance': ('state', # is not an attribute of clearance but is needed for splitting lga
	2118	'lga',
	2119	)
	2120	}
	2121	res_list = []
	2122	lines = []
	2123	fields = []
	2124	with_lga = False
	2125	for k in toexport.keys():
	2126	for f in toexport[k]:
	2127	if f == 'lga' :
	2128	with_lga = True
	2129	fields.append(f)
	2130	headline = ','.join(fields).replace('level','current_level')
	2131	open(export_file,"a").write(headline +'\n')
	2132	format = '"%(' + ')s","%('.join(fields) + ')s"'
	2133	query = In('review_state',('cleared_and_validated',
	2134	'clearance_requested',
	2135	'school_fee_paid',
	2136	'courses_registered',
	2137	'courses_validated'))
	2138	#query = In('review_state',('clearance_requested'))
	2139	students = aq_students(query)
	2140	nr2export = len(students)
	2141	logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))
	2142	chunk = 1000
	2143	total = 0
	2144	start = DateTime.DateTime().timeTime()
	2145	start_chunk = DateTime.DateTime().timeTime()
	2146	## alternative method slightly slower
	2147	# student_recs = {}
	2148	# for s in students:
	2149	# student_recs[s.id] = s
	2150	# catalog_recs = {}
	2151	# brains = self.portal_catalog(portal_type = 'Student')
	2152	# for cat_rec in brains:
	2153	# sid = cat_rec.getId
	2154	# catalog_recs[sid] = cat_rec
	2155	# #import pdb;pdb.set_trace()
	2156	# start = DateTime.DateTime().timeTime()
	2157	# start_chunk = DateTime.DateTime().timeTime()
	2158	# for student in students:
	2159	# if student.id not in student_recs.keys():
	2160	# continue
	2161	# not_all = False
	2162	# d = self.getFormattedStudentEntry(student_recs[student.id])
	2163	# student_obj = catalog_recs[student.id].getObject()
	2164	for student in students:
	2165	not_all = False
	2166	d = self.getFormattedStudentEntry(student)
	2167	student_obj = getattr(students_folder,student.id)
	2168	for k in toexport.keys()[1:]:
	2169	try:
	2170	object = getattr(student_obj,k)
	2171	object_doc = object.getContent()
	2172	except:
	2173	logger.info('%s %s record not found' % (student.id,k))
	2174	not_all = True
	2175	continue
	2176	for f in toexport[k]:
	2177	d[f] = getattr(object_doc,f,'')
	2178	if not_all:
	2179	continue
	2180	if with_lga:
	2181	d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas)
	2182	lines.append(format % d)
	2183	total += 1
	2184	if total and not total % chunk or total == len(students):
	2185	open(export_file,"a").write('\n'.join(lines) +'\n')
	2186	anz = len(lines)
	2187	logger.info("wrote %(anz)d total written %(total)d" % vars())
	2188	end_chunk = DateTime.DateTime().timeTime()
	2189	duration = end_chunk-start_chunk
	2190	per_record = duration/anz
	2191	till_now = end_chunk - start
	2192	avarage_per_record = till_now/total
	2193	estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
	2194	estimated_end = estimated_end.strftime("%H:%M:%S")
	2195	logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
	2196	start_chunk = DateTime.DateTime().timeTime()
	2197	lines = []
	2198	end = DateTime.DateTime().timeTime()
	2199	logger.info('total time %6.2f m' % ((end-start)/60))
	2200	filename, extension = os.path.splitext(export_file)
	2201	from subprocess import call
	2202	msg = "wrote %(total)d records to %(export_file)s" % vars()
	2203	try:
	2204	retcode = call('gzip %s' % (export_file),shell=True)
	2205	if retcode == 0:
	2206	msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
	2207	except OSError, e:
	2208	retcode = -99
	2209	logger.info("zip failed with %s" % e)
	2210	logger.info(msg)
	2211	args = {'portal_status_message': msg}
	2212	#url = self.REQUEST.get('URL1') + '?' + urlencode(args)
	2213	url = self.REQUEST.get('URL2')
	2214	return self.REQUEST.RESPONSE.redirect(url)
	2215	###)
	2216
	2217	security.declareProtected(ModifyPortalContent,"updateStudyCourse")###(
	2218	def updateStudyCourse(self):
	2219	"""update StudyCourse from CSV values"""
	2220	import transaction
	2221	import random
	2222	from pdb import set_trace
	2223	wftool = self.portal_workflow
	2224	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
	2225	students_folder = self.portal_url.getPortalObject().campus.students
	2226	csv_d = {'jamb_reg_no': "RegNumber",
	2227	'jamb_lastname': "Name",
	2228	'session': "Session",
	2229	'pume_tot_score': "PUME SCORE",
	2230	'jamb_score': "JambScore",
	2231	'jamb_sex': "Sex",
	2232	'jamb_state': "State",
	2233	## 'jamb_first_cos': "AdminCourse",
	2234	'faculty': "AdminFaculty",
	2235	'course_code': "AdmitCoscode",
	2236	'stud_status':"AdmitStatus",
	2237	'department': "AdmitDept",
	2238	'jamb_lga': "LGA",
	2239	'app_email': "email",
	2240	'app_mobile': "PhoneNumbers",
	2241	}
	2242	csv_fields = [f[1] for f in csv_d.items()]
	2243	tr_count = 0
	2244	total = 0
	2245	#name = 'pume_results'
	2246	name = 'StudyCourseChange'
	2247	no_import = []
	2248	s = ','.join(['"%s"' % fn for fn in csv_fields])
	2249	no_import.append('"Error",%s' % s)
	2250	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
	2251	no_certificate = "no certificate %s" % format
	2252	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
	2253	'\n'.join(no_import))
	2254	logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse')
	2255	logger.info('Start loading from %s.csv' % name)
	2256	l = self.portal_catalog({'meta_type': "Certificate"})
	2257	try:
	2258	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
	2259	except:
	2260	logger.error('Error reading %s.csv' % name)
	2261	return
	2262	for jamb in result:
	2263	jamb['Error'] = "Processing "
	2264	logger.info(format % jamb)
	2265	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
	2266	res = self.portal_catalog({'portal_type': "StudentApplication",
	2267	'SearchableText': jamb_reg_no })
	2268	if not res:
	2269	em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no
	2270	logger.info(em)
	2271	jamb['Error'] = "Student does not exist"
	2272	no_import.append(format % jamb)
	2273	continue
	2274	sid = res[0].getPath().split('/')[-2]
	2275	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
	2276	res = self.portal_catalog(portal_type = "Certificate", id = cert_id)
	2277	if not res:
	2278	em = 'No Certificate with ID %s \n' % cert_id
	2279	logger.info(em)
	2280	jamb['Error'] = "No Certificate %s" % cert_id
	2281	no_import.append( format % jamb)
	2282	continue
	2283	cert_brain = res[0]
	2284	catalog_entry = {}
	2285	student = getattr(self,sid)
	2286	#
	2287	# Study Course
	2288	#
	2289	study_course = student.study_course
	2290	dsc = {}
	2291	cert_pl = cert_brain.getPath().split('/')
	2292	catalog_entry['id'] = sid
	2293	catalog_entry['faculty'] = cert_pl[-4]
	2294	catalog_entry['department'] = cert_pl[-3]
	2295	catalog_entry['course'] = cert_id
	2296	dsc['study_course'] = cert_id
	2297	study_course.getContent().edit(mapping=dsc)
	2298	self.students_catalog.modifyRecord(**catalog_entry)
	2299	if tr_count > 10:
	2300	if len(no_import) > 1:
	2301	open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write(
	2302	'\n'.join(no_import))
	2303	no_import = []
	2304	em = '%d transactions commited\n' % tr_count
	2305	transaction.commit()
	2306	logger.info(em)
	2307	total += tr_count
	2308	tr_count = 0
	2309	tr_count += 1
	2310	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
	2311	###)
	2312
	2313
	2314
	2315	From WAeUP_Tool.py:
	2316
	2317
	2318	def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###(
	2319	"""load data from CSV values"""
	2320	import transaction
	2321	import random
	2322	member = self.portal_membership.getAuthenticatedMember()
	2323	logger = logging.getLogger('WAeUPTool.importData')
	2324	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
	2325	import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S")
	2326	students_folder = self.portal_url.getPortalObject().campus.students
	2327	start = True
	2328	tr_count = 0
	2329	total_imported = 0
	2330	total_not_imported = 0
	2331	total = 0
	2332	pending_only = False
	2333	pend_str = '_pending'
	2334	if name.endswith(pend_str):
	2335	pending_only = True
	2336	name = name[:-len(pend_str)]
	2337	iname = "import_%s" % name
	2338	if name in ('application','course_result',):
	2339	commit_after = 2000
	2340	else:
	2341	commit_after = 100
	2342	stool = getToolByName(self, 'portal_schemas')
	2343	ltool = getToolByName(self, 'portal_layouts')
	2344	schema = stool._getOb(iname)
	2345	if schema is None:
	2346	em = 'No such schema %s' % iname
	2347	logger.error('No such schema %s' % iname)
	2348	return em
	2349	layout = ltool._getOb(iname)
	2350	if layout is None:
	2351	em = 'No such layout %s' % iname
	2352	logger.error(em)
	2353	return em
	2354	validators = {}
	2355	for widget in layout.keys():
	2356	validators[widget] = layout[widget].validate
	2357	mode = "create"
	2358	if edit:
	2359	if filename.endswith('_toDelete'):
	2360	mode = "delete"
	2361	else:
	2362	mode = "edit"
	2363	importer_name = "mass_%(mode)s_%(name)s" % vars()
	2364	importer = getattr(self, '%s' % importer_name,None)
	2365	if importer is None:
	2366	em = 'No importer function %s' % importer_name
	2367	logger.error(em)
	2368	return em
	2369	pending_fn = "%s/import/%ss_pending.csv" % (i_home,name)
	2370	pending_imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
	2371	if pending_only:
	2372	import_source_fn = pending_fn
	2373	imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
	2374	not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current)
	2375	if not os.path.exists(pending_fn):
	2376	em = 'No %s' % os.path.split(pending_fn)
	2377	return em
	2378	else:
	2379	import_source_fn = "%s/import/%s.csv" % (i_home,filename)
	2380	imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
	2381	not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
	2382	if not os.path.exists(import_source_fn):
	2383	em = 'No %s' % os.path.split(import_soure_fn)
	2384	return em
	2385	attrs = csv.reader(open(import_source_fn,"rb")).next()
	2386	import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
	2387	or k.strip() == 'Error')]
	2388	diff2schema = set(import_keys).difference(set(schema.keys()))
	2389	diff2layout = set(import_keys).difference(set(layout.keys()))
	2390	if diff2schema and diff2schema != set(['id',]):
	2391	msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
	2392	return msg
	2393	if mode in ("create","edit"):
	2394	required_keys = [layout.getIdUnprefixed(id)
	2395	for id,widget in layout.objectItems()
	2396	if widget.is_required]
	2397	if not set(required_keys).issubset(set(import_keys)):
	2398	diff2import = set(required_keys).difference(set(import_keys))
	2399	msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
	2400	return msg
	2401	#
	2402	# not_imported
	2403	#
	2404	not_imported_keys = import_keys + ["Error",]
	2405	not_imported_file = open(not_imported_fn,"w")
	2406	not_imported_csv_writer = csv.DictWriter(not_imported_file,
	2407	not_imported_keys,
	2408	extrasaction='ignore')
	2409	mapping = dict((k,k) for k in not_imported_keys)
	2410	not_imported_csv_writer.writerow(mapping)
	2411	not_imported = []
	2412	#
	2413	# imported
	2414	#
	2415	imported_keys = import_keys[:]
	2416	if 'id' not in imported_keys:
	2417	imported_keys.insert(0,"id")
	2418	imported_file = open(imported_fn,"w")
	2419	imported_csv_writer = csv.DictWriter(imported_file,
	2420	imported_keys,
	2421	extrasaction='ignore')
	2422	mapping = dict((k,k) for k in imported_keys)
	2423	imported_csv_writer.writerow(mapping)
	2424	imported = []
	2425	#
	2426	# pending
	2427	#
	2428	pending_records = {}
	2429	imports_pending = 0
	2430	total_pending_imported = 0
	2431	total_pending = 0
	2432	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
	2433	format_error = format + ',"%(Error)s"'
	2434	#format = '"%(id)s",'+ format
	2435	imported = []
	2436	if name in ('verdict','course_result',):
	2437	#pending_keys = imported_keys[:]
	2438	sname = "%s_pending" % iname
	2439	pending_schema = stool._getOb(sname)
	2440	if schema is None:
	2441	em = 'No such pending_schema %s' % sname
	2442	logger.error('No such schema %s' % sname)
	2443	return em
	2444	pending_keys = pending_schema.keys()
	2445	pending_keys += "Error",
	2446	if 'id' not in pending_keys:
	2447	pending_keys.insert(0,'id')
	2448	pending_records = {}
	2449	if os.path.exists(pending_fn):
	2450	pending_imports_file = open(pending_fn)
	2451	pending_imports = csv.DictReader(pending_imports_file,
	2452	fieldnames=pending_keys)
	2453	if pending_imports:
	2454	pending_records = dict((row['id'],row) for row in pending_imports
	2455	if row['id'] != 'id')
	2456	pending_imports_file.close()
	2457	new_imports_pending = 0
	2458	pending_imported = []
	2459	if pending_only:
	2460	pending_imported_ids = []
	2461	pending_imported = []
	2462	for record in pending_records.values():
	2463	item = {}
	2464	for k,v in record.items():
	2465	if v:
	2466	item[k] = v
	2467	results = importer(item)
	2468	id = results[0]
	2469	error = results[1]
	2470	is_pending = len(results) == 3
	2471	if is_pending:
	2472	continue
	2473	del item['Error']
	2474	msg = ";".join(["%s : %s" % (k,v) for k,v in item.items()])
	2475	logger.info("imported from %(pending_fn)s %(msg)s" % vars())
	2476	pending_imported.append(item)
	2477	pending_imported_ids += id,
	2478	total_pending_imported = len(pending_imported_ids)
	2479	logger.info("imported %d pending imports" % len(pending_imported_ids))
	2480	for id in pending_imported_ids:
	2481	if id:
	2482	del pending_records[id]
	2483	else:
	2484	logger.info("tried to delete record with empty id")
	2485	pending_imports_file = open(pending_fn,'w')
	2486	pending_csv_writer = csv.DictWriter(pending_imports_file,
	2487	pending_keys,
	2488	extrasaction='ignore')
	2489	mapping = dict((k,k) for k in pending_keys)
	2490	pending_csv_writer.writerow(mapping)
	2491	if len(pending_imported) > 0:
	2492	pending_imported_file = open(pending_imported_fn,'w')
	2493	pending_imported_csv_writer = csv.DictWriter(pending_imported_file,
	2494	pending_keys,
	2495	extrasaction='ignore')
	2496	pending_imported_csv_writer.writerow(dict((k,k) for k in imported_keys))
	2497	pending_imported_csv_writer.writerows(pending_imported)
	2498	#
	2499	if pending_only:
	2500	items = []
	2501	else:
	2502	try:
	2503	items = csv.DictReader(open(import_source_fn,"rb"))
	2504	except:
	2505	msg = 'Error reading %s.csv' % filename
	2506	logger.error(msg)
	2507	return msg
	2508	not_imported = []
	2509	pending_keys = pending_records.keys()[:]
	2510	for item in items:
	2511	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
	2512	if start:
	2513	start = False
	2514	adapters = [MappingStorageAdapter(schema, item)]
	2515	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
	2516	dm = DataModel(item, adapters,context=self)
	2517	ds = DataStructure(data=item,datamodel=dm)
	2518	error_string = ""
	2519	total += 1
	2520	error_count = 0
	2521	for k in import_keys:
	2522	if not validators[k](ds,mode=mode):
	2523	if error_count:
	2524	error_string += ' ++ '
	2525	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
	2526	ds.getErrorMapping(k)))
	2527	error_count += 1
	2528	if error_string:
	2529	item['Error'] = error_string
	2530	not_imported.append(item)
	2531	total_not_imported += 1
	2532	continue
	2533	temp_item = item.copy()
	2534	temp_item.update(dm)
	2535	#id,error = importer(temp_item)
	2536	results = importer(temp_item)
	2537	id = results[0]
	2538	error = results[1]
	2539	is_pending = len(results) == 3
	2540	if is_pending:
	2541	temp_item = results[2]
	2542	temp_item['Error'] = error
	2543	msg = format_error % temp_item
	2544	#if id not in pending_records.keys():
	2545	if id not in pending_keys:
	2546	temp_item['id'] = id
	2547	pending_records[id] = temp_item
	2548	logger.info("%(id)s added to pending %(msg)s" % vars())
	2549	pending_keys += id,
	2550	new_imports_pending += 1
	2551	else:
	2552	logger.info("%(id)s already in pending %(msg)s" % vars())
	2553	continue
	2554	elif error:
	2555	item['Error'] = error
	2556	not_imported.append(item)
	2557	total_not_imported += 1
	2558	continue
	2559	item = temp_item
	2560	item['id'] = id
	2561	imported.append(item)
	2562	tr_count += 1
	2563	total_imported += 1
	2564	msg = format % item
	2565	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
	2566	if total and not total % commit_after:
	2567	transaction.commit()
	2568	if len(imported) > 0:
	2569	imported_csv_writer.writerows(imported)
	2570	imported = []
	2571	if len(not_imported) > 0:
	2572	not_imported_csv_writer.writerows(not_imported)
	2573	not_imported = []
	2574	if len(pending_records) > 0:
	2575	pending_csv_writer.writerows(pending_records.values())
	2576	total_pending += len(pending_records)
	2577	pending_records = {}
	2578	msg = '%d transactions committed\n' % (tr_count)
	2579	regs = []
	2580	logger.info(msg)
	2581	tr_count = 0
	2582	if len(imported) > 0:
	2583	imported_csv_writer.writerows(imported)
	2584	if len(not_imported) > 0:
	2585	not_imported_csv_writer.writerows(not_imported)
	2586	if len(pending_records) > 0:
	2587	pending_csv_writer.writerows(pending_records.values())
	2588	total_pending += len(pending_records)
	2589	import_source_fn = os.path.split(import_source_fn)[1]
	2590	pending_fn = os.path.split(pending_fn)[1]
	2591	msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
	2592	msg += "%(total_imported)d imported, %(total_not_imported)d not imported, " % vars()
	2593	if total_pending:
	2594	if pending_only:
	2595	msg += "%(new_imports_pending)d pending added, %(total_pending_imported)d pending imported " % vars()
	2596	else:
	2597	msg += "%(new_imports_pending)d pending added " % vars()
	2598	msg += "(of total %(total)d), " % vars()
	2599	msg += "%(total_pending)d total pending in %(pending_fn)s " % vars()
	2600	logger.info(msg)
	2601	return msg
	2602	###)

Note: See TracBrowser for help on using the repository browser.

Context navigation

source: WAeUP_SRP/base/unused_methods.txt @ 3371

Download in other formats: