Context navigation

unused_methods.txt @ 17718

Last change on this file since 17718 was 3251, checked in by Henrik Bettermann, 17 years ago

improve logging messages.

This is the actual logging history for imports:

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? admin starts import from test.csv

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? 100.000 % done in 0.01s, record from test.csv added to verdicts.pending, import_mode: edit, current_session: 06, matric_no: ENG0206078, current_level: 700, current_verdict: L, id: G151316, student G151316: imported level 700 does not match current_level 400

2008-02-28 10:45:39 INFO WAeUPImport.VerdictsImport? finished importing from test.csv in 0.01 seconds, 1 records totally read, 0 added to verdicts.imported, 1 added to verdicts.pending, 0 already in verdicts.pending

2008-02-28 10:47:06 INFO WAeUPImport.VerdictsImport? admin starts import from verdicts.pending

2008-02-28 10:47:07 INFO WAeUPImport.VerdictsImport? 100.000 % done in 0.99s, record imported and added to verdicts.imported from verdicts.pending, error: student G151316: imported level 700 does not match current_level 400, import_mode: edit, matric_no: ENG0206078, current_level: 400, imported_by: admin, import_date: 28/02/08 10:45:39, current_session: 06, current_verdict: L, id: G151316, imported_from: test.csv

2008-02-28 10:47:07 INFO WAeUPImport.VerdictsImport? finished importing from verdicts.pending in 1.17 seconds, 1 records totally read, 1 added to verdicts.imported, 1 removed from verdicts.pending

File size: 120.7 KB

Line
1	From Students.py:
2
3	security.declareProtected(ModifyPortalContent,"loadFullTimeStudentsFromCSV")###(
4	def loadFullTimeStudentsFromCSV(self):
5	"""load Fulltime Studentdata from CSV values"""
6	import transaction
7	import random
8	tr_count = 0
9	name = 'short_full_time'
10	no_import = False
11	if not no_import:
12	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
13	no_import.write('"MatricNo","EntryRegNo","CurrentSession","StudentLevel","fullname","FirstName","MiddleName","Lastname","FormerSurname","Sex","Nationality","State","LGA","PermanentAddress","PermanentAddressCity","CampusAddress","PhoneNumber","Emailaddress","Mode","CourseMajor","Faculty","Dept"\n')
14	logger = logging.getLogger('Import.%s' % name)
15	logger.info('Start loading from %s.csv' % name)
16	pwlist = []
17	pwlist.append('"student_id","firstname","middlename","lastname","matric_no","jamb_reg_no","access_code"')
18	pwl_template = Template('"$student_id","$firstname","$middlename","$lastname","$matric_no","$jamb_reg_no","$access_code"')
19	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
20	try:
21	students = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
22	except:
23	logger.error('Error reading %s.csv' % name)
24	return
25	l = self.portal_catalog({'meta_type': "StudentClearance",})
26	matrics = []
27	for s in l:
28	matrics.append(s.getObject().getContent().matric_no)
29	print matrics
30	l = self.portal_catalog({'meta_type': "Certificate"})
31	certs = {}
32	for c in l:
33	ca,ac,fa,dep_id,co,certcode = c.relative_path.split('/')
34	cid = "%(dep_id)s_%(certcode)s" % vars()
35	certs[cid] = c.getObject()
36	for student in students:
37	logger.info('processing "%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
38	sid = student.get('MatricNo')
39	if sid == "":
40	em = 'Empty MatricNo\n'
41	logger.info(em)
42	no_import.write(em)
43	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
44	continue
45	certcode = makeCertificateCode(student.get('CourseMajor'))
46	dep_id = student.get('Dept')
47	fac_id = student.get('Faculty')
48	cid = "%(dep_id)s_%(certcode)s" % vars()
49	if cid not in certs.keys():
50	em = 'Certificate with ID %s %s not found\n' % (certcode, student.get('CourseMajor'))
51	logger.info(em)
52	no_import.write(em)
53	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
54	continue
55	certificate_doc = certs[cid].getContent()
56	level = student.get('StudentLevel')
57	try:
58	int(level)
59	except:
60	em = 'Student with ID %(MatricNo)s StudentLevel is empty\n' % student
61	logger.info(em)
62	no_import.write(em)
63	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
64	continue
65	matric_no = student.get('MatricNo')
66	if matric_no not in matrics:
67	matrics.append(matric_no)
68	sid = self.generateStudentId(student.get('Lastname')[0])
69	#self.log('Creating Faculty %(id)s = %(Title)s' % faculty)
70	students_folder.invokeFactory('Student', sid)
71	logger.info('%(tr_count)s: Creating Student with ID %(sid)s Matric_no %(matric_no)s ' % vars())
72	student_obj = getattr(self,sid)
73	access_code = "%d" % random.randint(1000000000,9999999999)
74	student_obj.getContent().makeStudentMember(sid,access_code,)
75	pwl_dict = {'student_id': sid,'access_code':access_code}
76	student_obj.invokeFactory('StudentApplication','application')
77	application = student_obj.application
78	da = {'Title': 'Application Data'}
79	student_obj.invokeFactory('StudentPersonal','personal')
80	da['jamb_reg_no'] = student.get('EntryRegNo')
81	personal = student_obj.personal
82	dp = {'Title': 'Personal Data'}
83	student_obj.invokeFactory('StudentClearance','clearance')
84	clearance = student_obj.clearance
85	dc = {'Title': 'Clearance/Eligibility Record'}
86	dc['matric_no'] = matric_no
87	state = student.get('State')
88	lga = student.get('LGA')
89	if state and lga:
90	lga = state + ' / ' + lga
91	else:
92	lga = "None"
93	dc['lga'] = lga
94	dc['nationality'] = student.get('Nationality')
95	dc['email'] = student.get('Emailaddress')
96	dp['firstname'] = student.get('FirstName')
97	dp['middlename'] = student.get('MiddleName')
98	dp['lastname'] = student.get('Lastname')
99	dp['former_surname'] = student.get('FormerSurname')
100	dp['sex'] = student.get('Sex') == 'F'
101	dp['perm_address'] = student.get('PermanentAddress')
102	dp['perm_city'] = student.get('PermanentAddressCity')
103	dp['campus_address'] = student.get('CampusAddress')
104	dp['phone'] = student.get('PhoneNumber')
105	application.getContent().edit(mapping=da)
106	personal.getContent().edit(mapping=dp)
107	clearance.getContent().edit(mapping=dc)
108	#
109	# Study Course
110	#
111	student_obj.invokeFactory('StudentStudyCourse','study_course')
112	studycourse = student_obj.study_course
113	dsc = {}
114	from_certificate = ['title',
115	'max_elect',
116	'max_pass',
117	'n_core',
118	'nr_years',
119	'probation_credits',
120	'promotion_credits',
121	'start_level',
122	]
123	for f in from_certificate:
124	dsc[f] = getattr(certificate_doc,f)
125	dsc['faculty'] = fac_id
126	dsc['department'] = dep_id
127	dsc['study_course'] = certcode
128	css = student.get('CurrentSession') or '2004-2005'
129	cs = int(css.split('-')[0]) - 2000
130	cl = int(student.get('StudentLevel') or '100')/100
131	dsc['entry_session'] = "200%s" % (cs - cl)
132	dsc['clr_ac_pin'] = access_code
133	studycourse.getContent().edit(mapping=dsc)
134	#
135	# Level
136	#
137	## l = getattr(studycourse,level,None)
138	## if 0 and l is None:
139	## #self.log('Creating Department %(DeptCode)s = %(Description)s' % dep)
140	## logger.info('Creating Level %(StudentLevel)s for %(fullname)s' % student)
141	## studycourse.invokeFactory('StudentStudyLevel', level)
142	## l = getattr(studycourse, level)
143	## certificate = certs[certcode]
144	## cert_level = getattr(certificate,level,None)
145	## if cert_level is None:
146	## logger.info('Level %(level)s not in %(certcode)s' % vars())
147	## l.getContent().edit(mapping={'Title': "Level %s" % level})
148	else:
149	em = 'Student with ID %(MatricNo)s %(fullname)s already exists\n' % student
150	logger.info(em)
151	no_import.write(em)
152	no_import.write('"%(MatricNo)s","%(EntryRegNo)s","%(CurrentSession)s","%(StudentLevel)s","%(fullname)s","%(FirstName)s","%(MiddleName)s","%(Lastname)s","%(FormerSurname)s","%(Sex)s","%(Nationality)s","%(State)s","%(LGA)s","%(PermanentAddress)s","%(PermanentAddressCity)s","%(CampusAddress)s","%(PhoneNumber)s","%(Emailaddress)s","%(Mode)s","%(CourseMajor)s","%(Faculty)s","%(Dept)s"\n' % student)
153	continue
154	if tr_count > MAX_TRANS:
155	transaction.commit()
156	em = 'Transaction commited\n' % student
157	logger.info(em)
158	tr_count = 0
159	tr_count += 1
160	pwl_dict.update(dc)
161	pwl_dict.update(da)
162	pwl_dict.update(dp)
163	wftool = self.portal_workflow
164	pwlist.append(pwl_template.substitute(pwl_dict))
165	wftool.doActionFor(student_obj,'clear_and_validate')
166	student_obj.manage_setLocalRoles(sid, ['Owner',])
167	wftool.doActionFor(application,'close')
168	application.manage_setLocalRoles(sid, ['Owner',])
169	wftool.doActionFor(clearance,'close')
170	clearance.manage_setLocalRoles(sid, ['Owner',])
171	wftool.doActionFor(personal,'close')
172	personal.manage_setLocalRoles(sid, ['Owner',])
173	wftool.doActionFor(studycourse,'close_for_edit')
174	studycourse.manage_setLocalRoles(sid, ['Owner',])
175	open("%s/import/pwlist-%s.csv" % (i_home,name),"w+").write('\n'.join(pwlist))
176	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
177	###)
178
179	security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
180	def loadPumeResultsFromCSV(self):
181	"""load Fulltime Studentdata from CSV values into pumeresults catalog"""
182	import transaction
183	import random
184	## csv_d = {'jamb_reg_no': "RegNumber", ###(
185	## 'status': "Admission Status",
186	## 'name': "Name",
187	## 'score': "Score",
188	## 'sex': "Sex",
189	## 'faculty': "Faculty",
190	## 'department': "Dept",
191	## 'course': "Course",
192	## 'course_code_org': "Course Code",
193	## }
194	###)
195	csv_d = {'jamb_reg_no': "JAMBRegno",
196	'name': "Name",
197	'score': "Score",
198	'sex': "Sex",
199	'course': "Course",
200	'faculty': "Faculty",
201	'department': "Dept",
202	'course_code_org': "Course Code",
203	'status': "Admission Status",
204	'result_type': None,
205	}
206	csv_fields = [f[1] for f in csv_d.items() if f[1]]
207	tr_count = 0
208	total = 0
209	#name = 'pup_new'
210	name = 'pup_update'
211	update = name.endswith('update')
212	no_import = []
213	ok_import = []
214	ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
215	no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
216	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
217	ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
218	#open(ok_import_name,"w").write('\n'.join(no_import))
219	no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
220	#open(no_import_name,"w").write('\n'.join(no_import))
221	logger = logging.getLogger('Import.%s' % name)
222	starttime = DateTime.now()
223	logger.info('Start loading from %s.csv' % name)
224	try:
225	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
226	except:
227	logger.error('Error reading %s.csv' % name)
228	return
229	pume = self.portal_pumeresults
230	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
231	import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
232	eduplicate = '%s,"duplicate"' % format
233	einvalidjamb = '%s,"invalid JambRegNo"' % format
234	added = 'added ,%s' % format
235	#from pdb import set_trace;set_trace()
236	for jamb in result:
237	dict = {}
238	for f,fn in csv_d.items():
239	dict[f] = jamb.get(csv_d[f])
240	dict['result_type'] = 'DE'
241	jnr = jamb.get(csv_d['jamb_reg_no'])
242	if not checkJambNo(jnr):
243	logger.info(einvalidjamb % jamb)
244	dd = {}
245	for f,fn in csv_d.items():
246	dd[fn] = getattr(data,f)
247	no_import.append(eduplicate % dd)
248	no_import.append(eduplicate % jamb)
249	continue
250	res = pume(jamb_reg_no=jnr)
251	if len(res) > 0:
252	if update:
253	try:
254	pume.modifyRecord(**dict)
255	except ValueError:
256	logger.info(eduplicate % jamb)
257	continue
258	except KeyError:
259	pume.addRecord(**dict)
260	logger.info(added % jamb)
261	continue
262	else:
263	data = res[0]
264	if data.name != jamb.get(csv_d['name']):
265	#set_trace()
266	logger.info(eduplicate % jamb)
267	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
268	#logger.info(em)
269	dd = {}
270	for f,fn in csv_d.items():
271	dd[fn] = getattr(data,f)
272	no_import.append(eduplicate % dd)
273	no_import.append(eduplicate % jamb)
274	continue
275	try:
276	pume.addRecord(**dict)
277	ok_import.append(import_format % dict)
278	except ValueError:
279	logger.info(eduplicate % jamb)
280	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
281	#logger.info(em)
282	no_import.append(eduplicate % jamb)
283	logger.info('End loading from %s.csv' % name)
284	if len(no_import) > 1:
285	open(no_import_name,"w+").write('\n'.join(no_import))
286	open(ok_import_name,"w+").write('\n'.join(ok_import))
287	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
288	###)
289
290
291	security.declareProtected(ModifyPortalContent,"OLDloadPumeResultsFromCSV")###(
292	def OLDloadPumeResultsFromCSV(self):
293	"""load Fulltime Studentdata from CSV values"""
294	import transaction
295	import random
296	wftool = self.portal_workflow
297	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
298	csv_d = {'jamb_reg_no': "JAMBRegno",
299	'jamb_lastname': "Name",
300	'pume_options': "Options",
301	'session': "Session",
302	'days': "Days",
303	'response': "Responce",
304	'wrong': "Wrong",
305	'pume_eng_score': "EngScore",
306	'pume_gen_score': "GenScore",
307	'pume_tot_score': "Score",
308	'batch': "Batch",
309	'serial': "SerialNo",
310	'jamb_score': "JambScore",
311	'omitted':"Omitted",
312	'search_key': "SearchKey",
313	'jamb_sex': "Sex",
314	'fac1': "Fac1",
315	'fac2': "Fac2",
316	'jamb_first_cos': "CourseofStudy",
317	'stud_status':"StudStatus",
318	'registered': "Registered",
319	'jamb_state': "State",
320	'eng_fail': "EngFail",
321	'gen_fail': "GenFail",
322	'un_ans_eng': "UnAnsEng",
323	'un_ans_eng': "UnAnsGen",
324	'total_ans': "TotalUnAns",
325	'dept': "Dept",
326	'jamb_second_cos': "Course2",
327	'jamb_third_cos': "course3",
328	}
329	csv_fields = [f[1] for f in csv_d.items()]
330	tr_count = 0
331	name = 'pume_results'
332	no_import = []
333	s = ','.join(['"%s"' % fn for fn in csv_fields])
334	no_import.append('%s\n' % s)
335	logger = logging.getLogger('Import.%s' % name)
336	logger.info('Start loading from %s.csv' % name)
337	try:
338	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
339	except:
340	logger.error('Error reading %s.csv' % name)
341	return
342	for jamb in result:
343	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
344	processing = "processing %s" % format
345	logger.info(processing % jamb)
346	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
347	#import pdb;pdb.set_trace()
348	res = self.portal_catalog({'portal_type': "StudentApplication",
349	'jamb_reg_no': jamb_reg_no })
350	if res:
351	em = 'Student with REG-NO %s already exists\n' % jamb_reg_no
352	logger.info(em)
353	no_import.append(em)
354	no_import.append(format % jamb)
355	continue
356	cert_id = jamb.get(csv_d['jamb_first_cos']).upper()
357	res = self.portal_catalog({'portal_type': "Certificate",
358	'id': cert_id })
359	if len(res) < 1:
360	em = 'No Certificate with ID %s \n' % cert_id
361	logger.info(em)
362	no_import.append(em)
363	no_import.append(format % jamb)
364	continue
365	cert = res[0].getObject()
366	cert_path = res[0].getPath()
367	cert_doc = cert.getContent()
368	jamb_name = jamb.get(csv_d['jamb_lastname'])
369	jamb_name.replace('>','')
370	names = jamb_name.split()
371	letter = names[-1][0].upper()
372	sid = self.generateStudentId(letter)
373	not_created = True
374	while not_created:
375	try:
376	students_folder.invokeFactory('Student', sid)
377	not_created = False
378	except BadRequest:
379	sid = self.generateStudentId(letter)
380	logger.info('%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
381	student = getattr(self,sid)
382	student.manage_setLocalRoles(sid, ['Owner',])
383	student.invokeFactory('StudentClearance','clearance')
384	#wftool.doActionFor(student.clearance,'open')
385	dp = {'Title': 'Clearance/Eligibility Record'}
386	student.clearance.manage_setLocalRoles(sid, ['Owner',])
387	student.invokeFactory('StudentPume','pume')
388	dp = {'Title': 'Pume Data'}
389	student.invokeFactory('StudentApplication','application')
390	da = {'Title': 'Application Data'}
391	da["jamb_lastname"] = jamb_name
392	da_fields = ('jamb_reg_no',
393	'jamb_sex',
394	'jamb_state',
395	'jamb_score',
396	'jamb_first_cos',
397	'jamb_sex',
398	'jamb_state',
399	'jamb_first_cos',
400	'jamb_second_cos',
401	)
402	for f in da_fields:
403	da[f] = jamb.get(csv_d[f])
404	app = student.application
405	app.getContent().edit(mapping=da)
406	app.manage_setLocalRoles(sid, ['Owner',])
407	#wftool.doActionFor(app,'close')
408	dp_fields = (
409	'pume_eng_score',
410	'pume_gen_score',
411	'pume_tot_score',
412	)
413	for f in dp_fields:
414	dp[f] = float(jamb.get(csv_d[f]))
415	pume = student.pume
416	pume.getContent().edit(mapping=dp)
417	#wftool.doActionFor(pume,'close')
418	pume.manage_setLocalRoles(sid, ['Owner',])
419	#
420	# Study Course
421	#
422	student.invokeFactory('StudentStudyCourse','study_course')
423	study_course = student.study_course
424	dsc = {}
425	from_certificate = ['title',
426	'max_elect',
427	'max_pass',
428	'n_core',
429	'nr_years',
430	'probation_credits',
431	'promotion_credits',
432	'start_level',
433	]
434	for f in from_certificate:
435	dsc[f] = getattr(cert_doc,f)
436	cpl = cert_path.split('/')
437	dsc['faculty'] = cpl[-4]
438	dsc['department'] = cpl[-3]
439	dsc['study_course'] = cert_id
440	dsc['entry_session'] = jamb.get(csv_d['session'])
441	study_course.getContent().edit(mapping=dsc)
442	student.getContent().createSubObjects()
443	if dp['pume_tot_score']>49:
444	wftool.doActionFor(student,'pume_pass')
445	wftool.doActionFor(student,'admit')
446	else:
447	wftool.doActionFor(student,'pume_fail')
448	wftool.doActionFor(student,'reject_admission')
449	if len(no_import) > 1:
450	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
451	'\n'.join(no_import))
452	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
453	###)
454
455	security.declareProtected(ModifyPortalContent,"loadFullTimeStudentsResultsFromCSV") ###(
456	def loadFullTimeStudentsResultsFromCSV(self):
457	"""load Fulltime Studentdata from CSV values"""
458	#return
459	level_wf_actions = {}
460	level_wf_actions["SUCCESSFUL STUDENT"] = "pass_A"
461	level_wf_actions["STUDENT WITH CARRYOVER COURSES"] = "pass_B"
462	level_wf_actions["STUDENT FOR PROBATION"] = "probate_C"
463	level_wf_actions["STUDENT ON PROBATION/TRANSFER"] = "reject_D"
464	import transaction
465	wftool = self.portal_workflow
466	tr_count = 0
467	name = 'short_full_time_results_2004_2005'
468	no_import = False
469	if not no_import:
470	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
471	no_import.write('"Matnumber","CosCode","Ansbook","CosStuatus","Session","Mat_Cos","Score","CarryLevel","Grade","Weight","Semster","Verdict","Level","id","GPA"\n')
472	logger = logging.getLogger('import.%s' % name)
473	logger.info('Start loading from %s.csv' % name)
474	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
475	try:
476	results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
477	except:
478	logger.error('Error reading %s.csv' % name)
479	return
480	l = self.portal_catalog({'meta_type': "Course"})
481	courses = {}
482	for c in l:
483	courses[c.id] = c.getObject()
484	level_changed = False
485	student_changed = False
486	sid = ''
487	#import pdb;pdb.set_trace()
488	for result in results:
489	temp_sid = result.get('Matnumber')
490	if temp_sid != sid:
491	student_changed = True
492	res = self.portal_catalog({'meta_type': "StudentClearance",
493	'SearchableText': temp_sid })
494	if not res:
495	em = 'Student with ID %(Matnumber)s not found\n' % result
496	logger.info(em)
497	no_import.write(em)
498	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
499	continue
500	elif len(res) > 1:
501	em = 'More than one Student with ID %(Matnumber)s found\n' % result
502	logger.info(em)
503	no_import.write(em)
504	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
505	continue
506	sid = temp_sid
507	sf = res[0].getObject().aq_parent
508	sc = getattr(sf,'study_course')
509	level = ''
510	else:
511	student_changed = False
512	course = result.get('CosCode')
513	if course not in courses.keys():
514	em = 'Course with ID %(CosCode)s not found\n' % result
515	logger.info(em)
516	no_import.write(em)
517	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
518	continue
519	course_doc = courses[course].getContent()
520	temp_level = result.get('Level')
521	student_id = sf.getId()
522	result['StudentId'] = student_id
523	if temp_level != level:
524	try:
525	int(temp_level)
526	except:
527	em = 'Result with ID %(Matnumber)s Course %(CosCode)s Level is empty\n' % result
528	logger.info(em)
529	no_import.write(em)
530	no_import.write('"%(Matnumber)s","%(CosCode)s","%(Ansbook)s","%(CosStuatus)s","%(Session)s","%(Mat_Cos)s","%(Score)s","%(CarryLevel)s","%(Grade)s","%(Weight)s","%(Semster)s","%(Verdict)s","%(Level)s","%(id)s","%(GPA)s"\n' % result)
531	continue
532	level_changed = True
533	if 'dlev' in vars().keys():
534	wftool.doActionFor(l,level_wf_actions[dlev['verdict']])
535	level = temp_level
536	l = getattr(sc,level,None)
537	if l is None:
538	logger.info('Creating Level %(Level)s for %(StudentId)s %(Matnumber)s' % result)
539	sc.invokeFactory('StudentStudyLevel', level)
540	l = getattr(sc, level)
541	l.manage_setLocalRoles(student_id, ['Owner',])
542	else:
543	level_changed = False
544	cr = getattr(l,course,None)
545	if cr is None:
546	logger.info('Creating Course %(CosCode)s for %(StudentId)s %(Matnumber)s in Level %(Level)s' % result)
547	l.invokeFactory('StudentCourseResult',course)
548	cr = getattr(l,course)
549	dcr = {}
550	from_course = ['title',
551	'credits',
552	'passmark',
553	]
554	for f in from_course:
555	dcr[f] = getattr(course_doc,f)
556	dlev = {}
557	dcr['ansbook'] = result.get('Ansbook')
558	dcr['semester'] = getInt(result.get('Semster'))
559	dcr['status'] = result.get('CosStuatus')
560	dcr['score'] = getInt(result.get('Score'))
561	dlev['session'] = result.get('Session')
562	dcr['carry_level'] = result.get('CarryLevel')
563	dcr['grade'] = result.get('Grade')
564	dcr['weight'] = result.get('Weight')
565	dlev['verdict'] = result.get('Verdict')
566	dcr['import_id'] = result.get('id')
567	gpa = result.get('GPA').replace(',','.')
568	dlev['imported_gpa'] = getFloat(gpa)
569	cr.getContent().edit(mapping = dcr)
570	cr.manage_setLocalRoles(student_id, ['Owner',])
571	l.getContent().edit(mapping = dlev)
572	if tr_count > MAX_TRANS:
573	transaction.commit()
574	tr_count = 0
575	tr_count += 1
576	wftool.doActionFor(cr,'close')
577	wftool.doActionFor(l,level_wf_actions[dlev['verdict']])
578	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
579
580	###)
581
582	security.declareProtected(ModifyPortalContent,"loadJAMBFromCSV")###(
583	def loadJAMBFromCSV(self):
584	"""load JAMB data from CSV values"""
585	#return
586	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
587	import transaction
588	tr_count = 0
589	name = 'SampleJAMBDataII'
590	wftool = self.portal_workflow
591	no_import = False
592	if not no_import:
593	no_import = open("%s/import/%s_not_imported.csv" % (i_home,name),"w")
594	no_import.write('REG-NO,NAME,SEX,STATE,LGA,ENG-SCORE,SUBJ1,SUBJ1-SCORE,SUBJ2,SUBJ2-SCORE,SUBJ3,SUBJ3-SCORE,AGGREGATE,UNIV1,FACULTY1,COURSE1,UNIV2,FACULTY2,COURSE2')
595	logger = logging.getLogger('Import.%s' % name)
596	logger.info('Start loading from %s.csv' % name)
597	try:
598	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
599	except:
600	logger.error('Error reading %s.csv' % name)
601	return
602	for jamb in result:
603	logger.info('processing %(REG-NO)s,%(NAME)s,%(SEX)s,%(STATE)s,%(LGA)s,%(ENG-SCORE)s,%(SUBJ1)s,%(SUBJ1-SCORE)s,%(SUBJ2)s,%(SUBJ2-SCORE)s,%(SUBJ3)s,%(SUBJ3-SCORE)s,%(AGGREGATE)s,%(UNIV1)s,%(FACULTY1)s,%(COURSE1)s,%(UNIV2)s,%(FACULTY2)s,%(COURSE2)s\n' % jamb)
604	jamb_reg_no = jamb.get('REG-NO')
605	res = self.portal_catalog({'meta_type': "StudentApplication",
606	'jamb_reg_no': jamb_reg_no })
607	if res:
608	em = 'Student with REG-NO %(REG-NO)s already exists\n' % jamb
609	logger.info(em)
610	no_import.write(em)
611	no_import.write('%(REG-NO)s,%(NAME)s,%(SEX)s,%(STATE)s,%(LGA)s,%(ENG-SCORE)s,%(SUBJ1)s,%(SUBJ1-SCORE)s,%(SUBJ2)s,%(SUBJ2-SCORE)s,%(SUBJ3)s,%(SUBJ3-SCORE)s,%(AGGREGATE)s,%(UNIV1)s,%(FACULTY1)s,%(COURSE1)s,%(UNIV2)s,%(FACULTY2)s,%(COURSE2)s\n' % jamb)
612	continue
613	jamb_name = jamb.get("NAME")
614	jamb_name.replace('>','')
615	names = jamb_name.split()
616	letter = names[-1][0].upper()
617	sid = self.generateStudentId(letter)
618	not_created = True
619	while not_created:
620	try:
621	students_folder.invokeFactory('Student', sid)
622	not_created = False
623	except BadRequest:
624	sid = self.generateStudentId(letter)
625	logger.info('%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
626	student = getattr(self,sid)
627	student.manage_setLocalRoles(sid, ['Owner',])
628	student.invokeFactory('StudentApplication','application')
629	da = {'Title': 'Application Data'}
630	da["jamb_reg_no"] = jamb.get("REG-NO")
631	da["jamb_lastname"] = jamb_name
632	da["jamb_sex"] = jamb.get("SEX")
633	da["jamb_state"] = jamb.get("STATE")
634	da["jamb_lga"] = jamb.get("LGA")
635	da["jamb_score"] = jamb.get("AGGREGATE")
636	da["jamb_first_cos"] = jamb.get("COURSE1")
637	da["jamb_second_cos"] = jamb.get("COURSE2")
638	da["jamb_first_uni"] = jamb.get("UNIV1")
639	da["jamb_second_uni"] = jamb.get("UNIV2")
640	app = student.application
641	app_doc = app.getContent()
642	app_doc.edit(mapping=da)
643	#wftool.doActionFor(app,'open',dest_container=app)
644	app.manage_setLocalRoles(sid, ['Owner',])
645	student.getContent().createSubObjects()
646	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
647	###)
648
649
650	security.declareProtected(ModifyPortalContent,"importPreviousSessionStudents")###(
651	def importPreviousSessionStudents(self):
652	"""load and create previous session students from CSV"""
653	import transaction
654	import random
655	wftool = self.portal_workflow
656	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
657	students_folder = self.portal_url.getPortalObject().campus.students
658	tr_count = 1
659	total = 0
660	#name = 'pume_results'
661	name = 'Previous'
662	keys = self.portal_schemas.import_student.keys()
663	keys += self.portal_schemas.import_student_level_data.keys()
664	not_imported = []
665	imported = []
666	certificates = {}
667	logger = logging.getLogger('Students.StudentsFolder.importPreviousSessionStudents')
668	try:
669	records = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
670	except:
671	logger.error('Error reading %s.csv' % name)
672	return
673	start = True
674	for record in records:
675	if start:
676	start = False
677	logger.info('Start loading from %s.csv' % name)
678	false_keys = [k for k in record.keys() if k not in keys]
679	right_keys = [k for k in record.keys() if k in keys]
680	if false_keys:
681	logger.info('Fields %s not in schema' % false_keys)
682	s = ','.join(['"%s"' % k for k in right_keys])
683	imported.append(s)
684	not_imported.append('%s,"error"' % s)
685	format = ','.join(['"%%(%s)s"' % k for k in right_keys])
686	format_error = format + ',"%(error)s"'
687	study_course = makeCertificateCode(record.get('study_course'))
688	matric_no = record.get('matric_no')
689	student_res = self.students_catalog(matric_no = matric_no)
690	if student_res:
691	record['error'] = "Student exists"
692	not_imported.append(format_error % record)
693	continue
694	if study_course not in certificates.keys():
695	cert_res = self.portal_catalog(portal_type='Certificate',
696	id = study_course)
697	if not cert_res:
698	record['error'] = "No such studycourse"
699	not_imported.append(format_error % record)
700	continue
701	certificates[cert_res[0].id] = cert_res[0]
702	sid = self.generateStudentId('x',students_folder)
703	students_folder.invokeFactory('Student', sid)
704	#from pdb import set_trace;set_trace()
705	record['student_id'] = sid
706	student = getattr(self,sid)
707	student.manage_setLocalRoles(sid, ['Owner',])
708	student.invokeFactory('StudentApplication','application')
709	app = student.application
710	app_doc = app.getContent()
711	dict = {'Title': 'Application Data'}
712	dict["jamb_lastname"] = "%(firstname)s %(lastname)s %(middlename)s" % record
713	r2d = [
714	('entry_mode','entry_mode'),
715	('sex','jamb_sex'),
716	('jamb_score','jamb_score'),
717	('jamb_reg_no','jamb_reg_no'),
718	]
719	for r,d in r2d:
720	dict[d] = record[r]
721	app_doc.edit(mapping=dict)
722	app.manage_setLocalRoles(sid, ['Owner',])
723	wftool.doActionFor(app,'close')
724	record['sex'] = record['sex'] == 'F'
725	student.invokeFactory('StudentPersonal','personal')
726	dict = {}
727	r2d = [('firstname','firstname'),
728	('middlename','middlename'),
729	('lastname','lastname'),
730	('sex','sex'),
731	('email','email'),
732	('phone','phone'),
733	('address','perm_address'),
734	]
735	for r,d in r2d:
736	dict[d] = record[r]
737	per = student.personal
738	per_doc = per.getContent()
739	per_doc.edit(mapping = dict)
740	per.manage_setLocalRoles(sid, ['Owner',])
741	#
742	# Clearance
743	#
744	student.invokeFactory('StudentClearance','clearance')
745	#wftool.doActionFor(student.clearance,'open')
746	clearance = getattr(student,'clearance')
747	dict = {'Title': 'Clearance/Eligibility Record'}
748	clearance.getContent().edit(mapping = dict)
749	student.clearance.manage_setLocalRoles(sid, ['Owner',])
750	#
751	# Study Course
752	#
753	student.invokeFactory('StudentStudyCourse','study_course')
754	study_course = student.study_course
755	dict = {}
756	r2d = [('session','current_session'),
757	('level','current_level'),
758	('verdict','current_verdict'),
759	('study_course','study_course'),
760	]
761	for r,d in r2d:
762	dict[d] = record[r]
763	study_course.getContent().edit(mapping=dict)
764	#
765	# Study Level
766	#
767	level = record['level']
768	study_course.invokeFactory('StudentStudyLevel',level)
769	study_level = getattr(study_course,level)
770	dict = {}
771	r2d = [('session','session'),
772	('level','code'),
773	('verdict','verdict'),
774	]
775	for r,d in r2d:
776	dict[d] = record[r]
777	study_level.getContent().edit(mapping=dict)
778	wftool.doActionFor(student,'return')
779	imported.append(format % record)
780	tr_count += 1
781	record['tr_count'] = tr_count
782	record['total'] = total
783	logger.info('%(total)s+%(tr_count)s: Creating Student %(student_id)s %(matric_no)s %(jamb_reg_no)s' % record)
784	if tr_count > 1000:
785	if len(not_imported) > 0:
786	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
787	'\n'.join(not_imported)+'\n')
788	not_imported = []
789	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
790	'\n'.join(imported) + '\n')
791	imported = []
792	em = '%d transactions commited total %s' % (tr_count,total)
793	transaction.commit()
794	logger.info(em)
795	regs = []
796	total += tr_count
797	tr_count = 0
798	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
799	'\n'.join(imported))
800	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
801	'\n'.join(not_imported))
802	em = '%d transactions commited total %d' % (tr_count,total)
803	logger.info(em)
804	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
805	###)
806
807
808	security.declareProtected(ModifyPortalContent,"createDEStudents")###(
809	def createDEStudents(self):
810	"""load Fulltime Studentdata from CSV values"""
811	import transaction
812	import random
813	#from pdb import set_trace
814	wftool = self.portal_workflow
815	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
816	csv_d = {'jamb_reg_no': "RegNumber",
817	'jamb_lastname': "Name",
818	'session': "Session",
819	'pume_tot_score': "PUDE SCORE",
820	##'jamb_score': "JambScore",
821	'entry_mode': "EntryMode",
822	'jamb_sex': "Sex",
823	'jamb_state': "State",
824	'jamb_first_cos': "AdminCourse",
825	'faculty': "AdminFaculty",
826	'course_code': "AdmitCoscode",
827	'stud_status':"AdmitStatus",
828	'department': "AdmitDept",
829	'jamb_lga': "LGA",
830	'app_email': "email",
831	'app_mobile': "PhoneNumbers",
832	}
833	csv_fields = [f[1] for f in csv_d.items()]
834	tr_count = 0
835	total = 0
836	#name = 'pume_results'
837	name = 'DE_Admitted'
838	no_import = []
839	s = ','.join(['"%s"' % fn for fn in csv_fields])
840	no_import.append('"Error",%s' % s)
841	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
842	no_certificate = "no certificate %s" % format
843	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
844	logger = logging.getLogger('Students.StudentsFolder.createDEStudents')
845	logger.info('Start loading from %s.csv' % name)
846	l = self.portal_catalog({'meta_type': "Certificate"})
847	certs = {}
848	cert_docs = {}
849	for f in l:
850	certs[f.getId] = f.getObject().getContent()
851	try:
852	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
853	except:
854	logger.error('Error reading %s.csv' % name)
855	return
856	for jamb in result:
857	jamb['Error'] = "Processing"
858	logger.info(format % jamb)
859	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
860	res = self.portal_catalog({'portal_type': "StudentApplication",
861	'SearchableText': jamb_reg_no })
862	if res:
863	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
864	logger.info(em)
865	jamb['Error'] = "Student exists"
866	no_import.append(format % jamb)
867	continue
868	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
869	if cert_id not in certs.keys():
870	em = 'No Certificate with ID %s \n' % cert_id
871	logger.info(em)
872	jamb['Error'] = "No Certificate %s" % cert_id
873	no_import.append( format % jamb)
874	continue
875	jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
876	cert_doc = certs[cert_id]
877	catalog_entry = {}
878	catalog_entry['jamb_reg_no'] = jamb_reg_no
879	jamb_name = jamb.get(csv_d['jamb_lastname'])
880	jamb_name.replace('>','')
881	jamb_name.replace('<','')
882	names = jamb_name.split()
883	letter = names[-1][0].upper()
884	sid = self.generateStudentId(letter)
885	not_created = True
886	while not_created:
887	try:
888	students_folder.invokeFactory('Student', sid)
889	not_created = False
890	except BadRequest:
891	sid = self.generateStudentId(letter)
892	catalog_entry['id'] = sid
893	tr_count += 1
894	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
895	student = getattr(self,sid)
896	student.manage_setLocalRoles(sid, ['Owner',])
897	student.invokeFactory('StudentPume','pume')
898	dp = {'Title': 'Pume Data'}
899	student.invokeFactory('StudentApplication','application')
900	da = {'Title': 'Application Data'}
901	da["jamb_lastname"] = jamb_name
902	da_fields = ('jamb_reg_no',
903	'jamb_sex',
904	'entry_mode',
905	#'jamb_score',
906	'jamb_first_cos',
907	'jamb_sex',
908	'jamb_state',
909	'jamb_lga',
910	'app_email',
911	'app_mobile',
912	)
913	for f in da_fields:
914	da[f] = jamb.get(csv_d[f])
915	catalog_entry['email'] = jamb.get(csv_d['app_email'])
916	app = student.application
917	app_doc = app.getContent()
918	picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
919	#import pdb;pdb.set_trace()
920	if os.path.exists(picture):
921	file = open(picture)
922	if False:
923	img = PIL.Image.open(file)
924	img.thumbnail((150,200),
925	resample=PIL.Image.ANTIALIAS)
926	# We now need a buffer to write to. It can't be the same
927	# as the inbuffer as the PNG writer will write over itself.
928	outfile = StringIO()
929	img.save(outfile, format=img.format)
930	else:
931	outfile = file.read()
932	app_doc.manage_addFile('passport',
933	file=outfile,
934	title="%s.jpg" % jamb_reg_no)
935	app.getContent().edit(mapping=da)
936	app.manage_setLocalRoles(sid, ['Owner',])
937	#wftool.doActionFor(app,'close')
938	dp_fields = (
939	#'pume_eng_score',
940	#'pume_gen_score',
941	'pume_tot_score',
942	)
943	dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
944	pume = student.pume
945	pume.getContent().edit(mapping=dp)
946	#wftool.doActionFor(pume,'close')
947	pume.manage_setLocalRoles(sid, ['Owner',])
948	#student.getContent().createSubObjects()
949	dp = {}
950	if len(names) == 3:
951	dp['firstname'] = names[0].capitalize()
952	dp['middlename'] = names[1].capitalize()
953	dp['lastname'] = names[2].capitalize()
954	elif len(names) == 2:
955	dp['firstname'] = names[0].capitalize()
956	dp['middlename'] = ''
957	dp['lastname'] = names[1].capitalize()
958	else:
959	dp['firstname'] = ''
960	dp['middlename'] = ''
961	dp['lastname'] = jamb_name
962	dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
963	catalog_entry['sex'] = dp['sex']
964	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
965	student.invokeFactory('StudentPersonal','personal')
966	per = student.personal
967	per_doc = per.getContent()
968	per_doc.edit(mapping = dp)
969	per.manage_setLocalRoles(sid, ['Owner',])
970	if jamb.get(csv_d['stud_status']) == "Admitted":
971	wftool.doActionFor(student,'pume_pass')
972	wftool.doActionFor(student,'admit')
973	else:
974	wftool.doActionFor(student,'pume_fail')
975	wftool.doActionFor(student,'reject_admission')
976	continue
977	#
978	# Clearance
979	#
980	student.invokeFactory('StudentClearance','clearance')
981	#wftool.doActionFor(student.clearance,'open')
982	dp = {'Title': 'Clearance/Eligibility Record'}
983	student.clearance.manage_setLocalRoles(sid, ['Owner',])
984	#
985	# Study Course
986	#
987	student.invokeFactory('StudentStudyCourse','study_course')
988	study_course = student.study_course
989	dsc = {}
990	#from_certificate = ['title',
991	# 'max_elect',
992	# 'max_pass',
993	# 'n_core',
994	# 'nr_years',
995	# 'probation_credits',
996	# 'promotion_credits',
997	# 'start_level',
998	# ]
999	#for f in from_certificate:
1000	# dsc[f] = getattr(cert_doc,f)
1001	#dsc['faculty'] = jamb.get(csv_d['faculty'])
1002	#dsc['department'] = jamb.get(csv_d['department'])
1003	catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
1004	catalog_entry['department'] = jamb.get(csv_d['department'])
1005	catalog_entry['course'] = cert_id
1006	#catalog_entry['level'] = getattr(cert_doc,'start_level')
1007	catalog_entry['level'] = '200'
1008	dsc['study_course'] = cert_id
1009	dsc['current_level'] = '200'
1010	#dsc['entry_session'] = jamb.get(csv_d['session'])
1011	study_course.getContent().edit(mapping=dsc)
1012	self.students_catalog.addRecord(**catalog_entry)
1013	if tr_count > 10:
1014	if len(no_import) > 1:
1015	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1016	'\n'.join(no_import)+'\n')
1017	no_import = []
1018	em = '%d transactions commited\n' % tr_count
1019	transaction.commit()
1020	logger.info(em)
1021	total += tr_count
1022	tr_count = 0
1023	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1024	'\n'.join(no_import))
1025	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1026	###)
1027
1028	security.declareProtected(ModifyPortalContent,"createNewStudents")###(
1029	def createNewStudents(self):
1030	"""load Fulltime Studentdata from CSV values"""
1031	import transaction
1032	import random
1033	#from pdb import set_trace
1034	wftool = self.portal_workflow
1035	students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
1036	csv_d = {'jamb_reg_no': "RegNumber",
1037	'jamb_lastname': "Name",
1038	'session': "Session",
1039	'pume_tot_score': "PUME SCORE",
1040	'jamb_score': "JambScore",
1041	'jamb_sex': "Sex",
1042	'jamb_state': "State",
1043	## 'jamb_first_cos': "AdminCourse",
1044	'faculty': "AdminFaculty",
1045	'course_code': "AdmitCoscode",
1046	'stud_status':"AdmitStatus",
1047	'department': "AdmitDept",
1048	'jamb_lga': "LGA",
1049	'app_email': "email",
1050	'app_mobile': "PhoneNumbers",
1051	}
1052	csv_fields = [f[1] for f in csv_d.items()]
1053	tr_count = 0
1054	total = 0
1055	#name = 'pume_results'
1056	name = 'Admitted'
1057	no_import = []
1058	s = ','.join(['"%s"' % fn for fn in csv_fields])
1059	no_import.append('"Error",%s' % s)
1060	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
1061	no_certificate = "no certificate %s" % format
1062	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
1063	logger = logging.getLogger('Students.StudentsFolder.createNewStudents')
1064	logger.info('Start loading from %s.csv' % name)
1065	l = self.portal_catalog({'meta_type': "Certificate"})
1066	certs = {}
1067	cert_docs = {}
1068	for f in l:
1069	certs[f.getId] = f.getObject().getContent()
1070	try:
1071	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1072	except:
1073	logger.error('Error reading %s.csv' % name)
1074	return
1075	for jamb in result:
1076	jamb['Error'] = "Processing "
1077	logger.info(format % jamb)
1078	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
1079	res = self.portal_catalog({'portal_type': "StudentApplication",
1080	'SearchableText': jamb_reg_no })
1081	if res:
1082	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
1083	logger.info(em)
1084	jamb['Error'] = "Student exists"
1085	no_import.append(format % jamb)
1086	continue
1087	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
1088	if cert_id not in certs.keys():
1089	em = 'No Certificate with ID %s \n' % cert_id
1090	logger.info(em)
1091	jamb['Error'] = "No Certificate %s" % cert_id
1092	no_import.append( format % jamb)
1093	continue
1094	res = self.portal_pumeresults(jamb_reg_no = jamb_reg_no)
1095	if len(res) == 1:
1096	self.portal_pumeresults.modifyRecord(jamb_reg_no = jamb_reg_no,
1097	status = jamb.get(csv_d['stud_status']),
1098	)
1099	jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
1100	cert_doc = certs[cert_id]
1101	catalog_entry = {}
1102	catalog_entry['jamb_reg_no'] = jamb_reg_no
1103	jamb_name = jamb.get(csv_d['jamb_lastname'])
1104	jamb_name.replace('>','')
1105	jamb_name.replace('<','')
1106	names = jamb_name.split()
1107	letter = names[-1][0].upper()
1108	sid = self.generateStudentId(letter)
1109	not_created = True
1110	while not_created:
1111	try:
1112	students_folder.invokeFactory('Student', sid)
1113	not_created = False
1114	except BadRequest:
1115	sid = self.generateStudentId(letter)
1116	catalog_entry['id'] = sid
1117	tr_count += 1
1118	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
1119	student = getattr(self,sid)
1120	student.manage_setLocalRoles(sid, ['Owner',])
1121	student.invokeFactory('StudentPume','pume')
1122	dp = {'Title': 'Pume Data'}
1123	student.invokeFactory('StudentApplication','application')
1124	da = {'Title': 'Application Data'}
1125	da["jamb_lastname"] = jamb_name
1126	da_fields = ('jamb_reg_no',
1127	'jamb_sex',
1128	#'jamb_state',
1129	'jamb_score',
1130	## 'jamb_first_cos',
1131	'jamb_sex',
1132	'jamb_state',
1133	'jamb_lga',
1134	'app_email',
1135	'app_mobile',
1136	)
1137	for f in da_fields:
1138	da[f] = jamb.get(csv_d[f])
1139	catalog_entry['email'] = jamb.get(csv_d['app_email'])
1140	app = student.application
1141	app_doc = app.getContent()
1142	#import pdb;pdb.set_trace()
1143	picture ="%s/import/pictures/%s.jpg" % (i_home,picture_id)
1144	if os.path.exists(picture):
1145	file = open(picture)
1146	if False:
1147	img = PIL.Image.open(file)
1148	img.thumbnail((150,200),
1149	resample=PIL.Image.ANTIALIAS)
1150	# We now need a buffer to write to. It can't be the same
1151	# as the inbuffer as the PNG writer will write over itself.
1152	outfile = StringIO()
1153	img.save(outfile, format=img.format)
1154	else:
1155	outfile = file.read()
1156	app_doc.manage_addFile('passport',
1157	file=outfile,
1158	title="%s.jpg" % jamb_reg_no)
1159	app.getContent().edit(mapping=da)
1160	app.manage_setLocalRoles(sid, ['Owner',])
1161	#wftool.doActionFor(app,'close')
1162	dp_fields = (
1163	#'pume_eng_score',
1164	#'pume_gen_score',
1165	'pume_tot_score',
1166	)
1167	dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
1168	pume = student.pume
1169	pume.getContent().edit(mapping=dp)
1170	#wftool.doActionFor(pume,'close')
1171	pume.manage_setLocalRoles(sid, ['Owner',])
1172	#student.getContent().createSubObjects()
1173	dp = {}
1174	if len(names) == 3:
1175	dp['firstname'] = names[0].capitalize()
1176	dp['middlename'] = names[1].capitalize()
1177	dp['lastname'] = names[2].capitalize()
1178	elif len(names) == 2:
1179	dp['firstname'] = names[0].capitalize()
1180	dp['middlename'] = ''
1181	dp['lastname'] = names[1].capitalize()
1182	else:
1183	dp['firstname'] = ''
1184	dp['middlename'] = ''
1185	dp['lastname'] = jamb_name
1186	dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
1187	catalog_entry['sex'] = dp['sex']
1188	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
1189	student.invokeFactory('StudentPersonal','personal')
1190	per = student.personal
1191	per_doc = per.getContent()
1192	per_doc.edit(mapping = dp)
1193	per.manage_setLocalRoles(sid, ['Owner',])
1194	if jamb.get(csv_d['stud_status']) == "Admitted":
1195	wftool.doActionFor(student,'pume_pass')
1196	wftool.doActionFor(student,'admit')
1197	else:
1198	wftool.doActionFor(student,'pume_fail')
1199	wftool.doActionFor(student,'reject_admission')
1200	continue
1201	#
1202	# Clearance
1203	#
1204	student.invokeFactory('StudentClearance','clearance')
1205	#wftool.doActionFor(student.clearance,'open')
1206	dp = {'Title': 'Clearance/Eligibility Record'}
1207	student.clearance.manage_setLocalRoles(sid, ['Owner',])
1208	#
1209	# Study Course
1210	#
1211	student.invokeFactory('StudentStudyCourse','study_course')
1212	study_course = student.study_course
1213	dsc = {}
1214	#from_certificate = ['title',
1215	# 'max_elect',
1216	# 'max_pass',
1217	# 'n_core',
1218	# 'nr_years',
1219	# 'probation_credits',
1220	# 'promotion_credits',
1221	# 'start_level',
1222	# ]
1223	#for f in from_certificate:
1224	# dsc[f] = getattr(cert_doc,f)
1225	#dsc['faculty'] = jamb.get(csv_d['faculty'])
1226	#dsc['department'] = jamb.get(csv_d['department'])
1227	catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
1228	catalog_entry['department'] = jamb.get(csv_d['department'])
1229	catalog_entry['course'] = cert_id
1230	#catalog_entry['level'] = getattr(cert_doc,'start_level')
1231	catalog_entry['level'] = '100'
1232	dsc['study_course'] = cert_id
1233	#dsc['entry_level'] = '100'
1234	#dsc['entry_session'] = jamb.get(csv_d['session'])
1235	study_course.getContent().edit(mapping=dsc)
1236	self.students_catalog.addRecord(**catalog_entry)
1237	if tr_count > 10:
1238	if len(no_import) > 0:
1239	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1240	'\n'.join(no_import) + "\n")
1241	no_import = []
1242	em = '%d transactions commited\n' % tr_count
1243	transaction.commit()
1244	logger.info(em)
1245	total += tr_count
1246	tr_count = 0
1247	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1248	'\n'.join(no_import))
1249	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1250	###)
1251
1252
1253	security.declareProtected(ModifyPortalContent,"fixVerdicts")###(
1254	def fixVerdicts(self,csv_file=None):
1255	"""fix wrong uploaded verdicts"""
1256	import transaction
1257	import random
1258	wftool = self.portal_workflow
1259	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1260	students_folder = self.portal_url.getPortalObject().campus.students
1261	tr_count = 1
1262	total = 0
1263	if csv_file is None:
1264	name = 'Verdicts'
1265	else:
1266	name = csv_file
1267	st_cat = self.students_catalog
1268	no_import = []
1269	verdicts_voc = self.portal_vocabularies.verdicts
1270	rverdicts = {}
1271	for k,v in verdicts_voc.items():
1272	rverdicts[v.upper()] = k
1273	rverdicts['STUDENT ON PROBATION'] = 'C'
1274	logger = logging.getLogger('Students.StudentsFolder.fixVerdicts')
1275	try:
1276	verdicts = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1277	except:
1278	logger.error('Error reading %s.csv' % name)
1279	return
1280	start = True
1281	#import pdb;pdb.set_trace()
1282	for verdict in verdicts:
1283	if start:
1284	start = False
1285	logger.info('Start loading from %s.csv' % name)
1286	s = ','.join(['"%s"' % fn for fn in verdict.keys()])
1287	no_import.append('%s,"Error"' % s)
1288	format = ','.join(['"%%(%s)s"' % fn for fn in verdict.keys()])
1289	format_error = format + ',"%(Error)s"'
1290	matric_no = verdict.get('MAT NO')
1291	if not matric_no:
1292	continue
1293	matric_no = matric_no.upper()
1294	if matric_no == '':
1295	continue
1296	verdict_code = rverdicts.get(verdict.get('CATEGORY'),None)
1297	if verdict_code is None:
1298	continue
1299	sres = st_cat(matric_no = matric_no)
1300	if sres:
1301	student_id = sres[0].id
1302	student_obj = getattr(students_folder,student_id,None)
1303	if student_obj:
1304	study_course = getattr(student_obj,'study_course',None)
1305	if study_course is None:
1306	verdict['Error'] = "Student did not yet log in"
1307	no_import.append( format_error % verdict)
1308	continue
1309	st_cat.modifyRecord(id = student_id,
1310	verdict=verdict_code)
1311
1312	dsc = {}
1313	dsc['current_verdict'] = verdict_code
1314	study_course.getContent().edit(mapping=dsc)
1315	else:
1316	verdict['Error'] = "Not found in students_catalog"
1317	no_import.append( format_error % verdict)
1318	continue
1319	tr_count += 1
1320	if tr_count > 1000:
1321	if len(no_import) > 0:
1322	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1323	'\n'.join(no_import) + '\n')
1324	no_import = []
1325	em = '%d transactions commited total %s\n' % (tr_count,total)
1326	transaction.commit()
1327	regs = []
1328	logger.info(em)
1329	total += tr_count
1330	tr_count = 0
1331	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1332	'\n'.join(no_import))
1333	total += tr_count
1334	em = '%d total transactions commited' % (total)
1335	logger.info(em)
1336	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1337	###)
1338
1339
1340	security.declareProtected(ModifyPortalContent,"fixAllEntryModeForReturning")###(
1341	def fixAllEntryModeForReturning(self):
1342	"read all Returning*.csv"
1343	ipath = "%s/import/" % i_home
1344	names = os.listdir(ipath)
1345	for name in names:
1346	head,tail = os.path.splitext(name)
1347	if head.startswith('Returning')\
1348	and tail == '.csv'\
1349	and name.find('imported') < 0:
1350	self.fixEntryModeForReturning(csv_file=head)
1351	###)
1352
1353	security.declareProtected(ModifyPortalContent,"fixEntryModeForReturning")###(
1354	def fixEntryModeForReturning(self,csv_file=None):
1355	"""load Returning Studentdata from CSV values"""
1356	import transaction
1357	import random
1358	wftool = self.portal_workflow
1359	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1360	students_folder = self.portal_url.getPortalObject().campus.students
1361	tr_count = 1
1362	total = 0
1363	if csv_file is None:
1364	name = 'Returning'
1365	else:
1366	name = csv_file
1367	table = self.returning_import
1368	st_cat = self.students_catalog
1369	no_import = []
1370	logger = logging.getLogger('Students.StudentsFolder.fixEntryModeForReturning')
1371	try:
1372	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1373	except:
1374	logger.error('Error reading %s.csv' % name)
1375	return
1376	start = True
1377	for student in returning:
1378	if start:
1379	start = False
1380	logger.info('Start loading from %s.csv' % name)
1381	s = ','.join(['"%s"' % fn for fn in student.keys()])
1382	no_import.append('%s,"Error"' % s)
1383	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
1384	format_error = format + ',"%(Error)s"'
1385	matric_no = student.get('matric_no')
1386	if not matric_no:
1387	continue
1388	matric_no = matric_no.upper()
1389	student['matric_no'] = matric_no
1390	if matric_no == '':
1391	continue
1392	if not table(matric_no = matric_no):
1393	student['Error'] = "Not imported yet"
1394	no_import.append( format_error % student)
1395	continue
1396	student_id = None
1397	app = None
1398	per = None
1399	if st_cat(matric_no = matric_no):
1400	student_id = st_cat(matric_no = matric_no)[0].id
1401	student_obj = getattr(students_folder,student_id,None)
1402	if student_obj:
1403	app = getattr(student_obj,'application',None)
1404	if app is not None:
1405	app_doc = app.getContent()
1406	per = getattr(student_obj,'personal',None)
1407	if per is not None:
1408	per_doc = per.getContent()
1409	student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
1410	student['Permanent_Address'] = perm_address = student.get('Permanent Address')
1411	#import pdb;pdb.set_trace()
1412	if not entry_mode:
1413	student['Error'] = "'Mode of Entry' empty"
1414	no_import.append( format_error % student)
1415	continue
1416	try:
1417	table.modifyRecord(matric_no = matric_no,
1418	Mode_of_Entry = entry_mode,
1419	Permanent_Address = perm_address)
1420	except KeyError:
1421	student['Error'] = "Not found in returning_import"
1422	no_import.append( format_error % student)
1423	continue
1424	if student_id is not None:
1425	try:
1426	st_cat.modifyRecord(id = student_id,
1427	entry_mode=entry_mode)
1428	except KeyError:
1429	student['Error'] = "Not found in students_catalog"
1430	no_import.append( format_error % student)
1431	continue
1432	if app is not None:
1433	da = {}
1434	da['entry_mode'] = entry_mode
1435	app_doc.edit(mapping=da)
1436	if per is not None:
1437	dp = {}
1438	dp['perm_address'] = perm_address
1439	per_doc.edit(mapping=dp)
1440	tr_count += 1
1441	if tr_count > 1000:
1442	if len(no_import) > 0:
1443	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1444	'\n'.join(no_import) + '\n')
1445	no_import = []
1446	em = '%d transactions commited total %s\n' % (tr_count,total)
1447	transaction.commit()
1448	regs = []
1449	logger.info(em)
1450	total += tr_count
1451	tr_count = 0
1452	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1453	'\n'.join(no_import))
1454	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1455	###)
1456
1457	security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
1458	def loadPumeResultsFromCSV(self):
1459	"""load Fulltime Studentdata from CSV values into pumeresults catalog"""
1460	import transaction
1461	import random
1462	## csv_d = {'jamb_reg_no': "RegNumber", ###(
1463	## 'status': "Admission Status",
1464	## 'name': "Name",
1465	## 'score': "Score",
1466	## 'sex': "Sex",
1467	## 'faculty': "Faculty",
1468	## 'department': "Dept",
1469	## 'course': "Course",
1470	## 'course_code_org': "Course Code",
1471	## }
1472	###)
1473	## csv_d = {'jamb_reg_no': "JAMBRegno",
1474	## 'name': "Name",
1475	## 'score': "Score",
1476	## 'sex': "Sex",
1477	## 'course': "Course",
1478	## 'faculty': "Faculty",
1479	## 'department': "Dept",
1480	## 'course_code_org': "Course Code",
1481	## 'status': "Admission Status",
1482	## 'result_type': None,
1483	## }
1484
1485	csv_d = {'jamb_reg_no': "reg_no",
1486	'name': "fullname",
1487	'score': "pume_score",
1488	'sex': "sex",
1489	'course': "study_course",
1490	'course_code_org': "study_course",
1491	'status': "admission_status",
1492	'result_type': "entry_mode",
1493	}
1494
1495	csv_fields = [f[1] for f in csv_d.items() if f[1]]
1496	tr_count = 0
1497	total = 0
1498	#name = 'pup_new'
1499	#name = 'pup_update'
1500	name = 'Admitted_update'
1501	update = name.endswith('update')
1502	no_import = []
1503	ok_import = []
1504	ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
1505	no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
1506	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1507	ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
1508	#open(ok_import_name,"w").write('\n'.join(no_import))
1509	no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
1510	#open(no_import_name,"w").write('\n'.join(no_import))
1511	logger = logging.getLogger('Students.loadPumeResultsFromCSV')
1512	starttime = DateTime.now()
1513	logger.info('Start loading from %s.csv' % name)
1514	try:
1515	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1516	except:
1517	logger.error('Error reading %s.csv' % name)
1518	return
1519	pume = self.portal_pumeresults
1520	format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
1521	import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
1522	eduplicate = '%s,"duplicate"' % format
1523	eoriginal = '%s,"original"' % format
1524	einvalidjamb = '%s,"invalid JambRegNo"' % format
1525	added = 'added ,%s' % format
1526	#from pdb import set_trace;set_trace()
1527	for jamb in result:
1528	if not jamb.get(csv_d['score']):
1529	logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no']))
1530	continue
1531	dict = {}
1532	for f,fn in csv_d.items():
1533	dict[f] = jamb.get(csv_d[f])
1534	dict['result_type'] = 'CEST'
1535	jnr = jamb.get(csv_d['jamb_reg_no'])
1536	#if not checkJambNo(jnr):
1537	# logger.info(einvalidjamb % jamb)
1538	# dd = {}
1539	# for f,fn in csv_d.items():
1540	# dd[fn] = getattr(data,f)
1541	# no_import.append(eduplicate % dd)
1542	# no_import.append(eduplicate % jamb)
1543	# continue
1544	res = pume(jamb_reg_no=jnr)
1545	if len(res) > 0:
1546	if update:
1547	try:
1548	pume.modifyRecord(**dict)
1549	# Can not happen, but anyway...
1550	except ValueError:
1551	logger.info(eduplicate % jamb)
1552	continue
1553	# Can not happen, but anyway...
1554	except KeyError:
1555	pume.addRecord(**dict)
1556	logger.info(added % jamb)
1557	continue
1558	else:
1559	data = res[0]
1560	if data.name != jamb.get(csv_d['name']):
1561	#set_trace()
1562	logger.info(eduplicate % jamb)
1563	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
1564	#logger.info(em)
1565	dd = {}
1566	for f,fn in csv_d.items():
1567	dd[fn] = getattr(data,f)
1568	no_import.append(eoriginal % dd)
1569	no_import.append(eduplicate % jamb)
1570	continue
1571	else:
1572	try:
1573	pume.addRecord(**dict)
1574	ok_import.append(import_format % dict)
1575	except ValueError:
1576	logger.info(eduplicate % jamb)
1577	#em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
1578	#logger.info(em)
1579	no_import.append(eduplicate % jamb)
1580	logger.info('End loading from %s.csv' % name)
1581	if len(no_import) > 1:
1582	open(no_import_name,"w+").write('\n'.join(no_import))
1583	open(ok_import_name,"w+").write('\n'.join(ok_import))
1584	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1585	###)
1586
1587	security.declareProtected(ModifyPortalContent,"createStudents")###(
1588	def createStudents(self):
1589	"""
1590	load addmitted Studentdata from CSV values and create Studentobjects.
1591	This is the current method to create new addmitted Students.
1592	Before running the eventservice for the students_catalog must be disabled.
1593	"""
1594	import transaction
1595	import random
1596	#from pdb import set_trace
1597	wftool = self.portal_workflow
1598	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
1599	students_folder = self.portal_url.getPortalObject().campus.students
1600	levels = {'ume_ft':'100',
1601	'de_ft': '200',
1602	'ug_pt': '100',
1603	'de_pt': '200',
1604	'pg_ft': '700',
1605	'pg_pt': '700',
1606	'dp_pt': '100',
1607	'dp_ft': '100',
1608	}
1609	csv_d = {'jamb_reg_no': "reg_no",
1610	'entry_mode': 'entry_mode',
1611	'jamb_firstname': "firstname",
1612	'jamb_middlename': "middlename",
1613	'jamb_lastname': "lastname",
1614	'jamb_sex': "sex",
1615	'jamb_state': "state",
1616	'birthday': "date_of_birth",
1617	'app_email': "email",
1618	'study_course': "study_course",
1619	'perm_address': "address",
1620	'admission_status': "admission_status",
1621	}
1622	csv_fields = [f[1] for f in csv_d.items()]
1623	tr_count = 0
1624	total = 0
1625	#name = 'pume_results'
1626	name = 'Admitted_update'
1627	no_import = []
1628	s = ','.join(['"%s"' % fn for fn in csv_fields])
1629	no_import.append('"Error",%s' % s)
1630	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
1631	no_certificate = "no certificate %s" % format
1632	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
1633	logger = logging.getLogger('Students.StudentsFolder.createStudents')
1634	logger.info('Start loading from %s.csv' % name)
1635	certs = {}
1636	try:
1637	results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1638	except:
1639	logger.error('Error reading %s.csv' % name)
1640	return
1641	for result in results:
1642	if not result.get(csv_d['admission_status']).startswith('Admitted'):
1643	continue
1644	#result['Error'] = "Processing "
1645	#logger.info(format % result)
1646	jamb_reg_no = result.get(csv_d['jamb_reg_no'])
1647	res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1648	if res:
1649	em = 'Student with RegNo %s already exists\n' % jamb_reg_no
1650	logger.info(em)
1651	result['Error'] = "Student exists"
1652	no_import.append(format % result)
1653	continue
1654	cert_id = makeCertificateCode(result.get(csv_d['study_course']))
1655	if cert_id not in certs.keys():
1656	res = self.portal_catalog(meta_type = "Certificate",id = cert_id)
1657	if not res:
1658	em = 'No Certificate with ID %s \n' % cert_id
1659	logger.info(em)
1660	result['Error'] = "No Certificate %s" % cert_id
1661	no_import.append( format % result)
1662	continue
1663	cert = res[0]
1664	cert_path = cert.getPath().split('/')
1665	certificate = certs[cert_id] = {'faculty': cert_path[-4],
1666	'department': cert_path[-3]}
1667	cert_doc = certs[cert_id]
1668	catalog_entry = {}
1669	catalog_entry['jamb_reg_no'] = jamb_reg_no
1670	firstname = result.get(csv_d['jamb_firstname'])
1671	middlename = result.get(csv_d['jamb_middlename'])
1672	lastname = result.get(csv_d['jamb_lastname'])
1673	if len(firstname) < 3\
1674	and len(middlename) < 3\
1675	and len(lastname) < 3:
1676	em = 'Student Names to short \n'
1677	logger.info(em)
1678	result['Error'] = "Names to short"
1679	no_import.append( format % result)
1680	continue
1681	perm_address = result.get(csv_d['perm_address'])
1682	sid = self.generateStudentId('x')
1683	students_folder.invokeFactory('Student', sid)
1684	catalog_entry['id'] = sid
1685	tr_count += 1
1686	logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
1687	student = getattr(self,sid)
1688	student.manage_setLocalRoles(sid, ['Owner',])
1689	student.invokeFactory('StudentApplication','application')
1690	da = {'Title': 'Application Data'}
1691	da["jamb_firstname"] = firstname
1692	da["jamb_middlename"] = middlename
1693	da["jamb_lastname"] = lastname
1694	catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0]
1695	catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')
1696	da_fields = ('jamb_reg_no',
1697	'jamb_sex',
1698	'jamb_state',
1699	'entry_mode',
1700	'app_email',
1701	)
1702	for f in da_fields:
1703	da[f] = result.get(csv_d[f])
1704	catalog_entry['email'] = da['app_email']
1705	catalog_entry['entry_mode'] = da['entry_mode']
1706	app = student.application
1707	app_doc = app.getContent()
1708	app.getContent().edit(mapping=da)
1709	picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
1710	app.manage_setLocalRoles(sid, ['Owner',])
1711
1712	picture_id = da['jamb_reg_no'].replace('/','_')
1713	file = None
1714	for ext in ('jpg','JPG'):
1715	picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)
1716	if os.path.exists(picture):
1717	file = open(picture)
1718	break
1719	if file is not None:
1720
1721	## file conversion does not work
1722	#img = PIL.Image.open(file)
1723	#img.thumbnail((150,200),
1724	# resample=PIL.Image.ANTIALIAS)
1725	#outfile = StringIO()
1726	#img.save(outfile, format=img.format)
1727
1728	outfile = file.read()
1729	app_doc.manage_addFile('passport',
1730	file=outfile,
1731	title="%s.jpg" % jamb_reg_no)
1732	#wftool.doActionFor(app,'close')
1733	dp = {}
1734	dp['firstname'] = firstname
1735	dp['middlename'] = middlename
1736	dp['lastname'] = lastname
1737	dp['email'] = da['app_email']
1738	dp['sex'] = sex
1739	dp['perm_address'] = perm_address
1740	catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
1741	student.invokeFactory('StudentPersonal','personal')
1742	per = student.personal
1743	per_doc = per.getContent()
1744	per_doc.edit(mapping = dp)
1745	per.manage_setLocalRoles(sid, ['Owner',])
1746	wftool.doActionFor(student,'pume_pass')
1747	wftool.doActionFor(student,'admit')
1748	#
1749	# Clearance
1750	#
1751	student.invokeFactory('StudentClearance','clearance')
1752	#wftool.doActionFor(student.clearance,'open')
1753	clearance = student.clearance
1754	dc = {'Title': 'Clearance/Eligibility Record'}
1755	clearance = student.clearance
1756	date_str = result.get(csv_d['birthday'])
1757	try:
1758	date = DateTime.DateTime(date_str)
1759	except:
1760	#import pdb;pdb.set_trace()
1761	date = None
1762	dc['birthday'] = date
1763	clearance.getContent().edit(mapping=dc)
1764	clearance.manage_setLocalRoles(sid, ['Owner',])
1765	#
1766	# Study Course
1767	#
1768	student.invokeFactory('StudentStudyCourse','study_course')
1769	study_course = student.study_course
1770	dsc = {}
1771	catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100')
1772	#catalog_entry['level'] = dsc['current_level'] = '100' # Attention: not for DE students
1773	catalog_entry['session'] = dsc['current_session'] = da['entry_session']
1774	catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']
1775	catalog_entry['course'] = dsc['study_course'] = cert_id
1776	catalog_entry['faculty'] = certificate['faculty']
1777	catalog_entry['department'] = certificate['department']
1778	catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'
1779	catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)
1780	study_course.getContent().edit(mapping=dsc)
1781	#import pdb;pdb.set_trace()
1782	self.students_catalog.addRecord(**catalog_entry)
1783	if tr_count > 100:
1784	if len(no_import) > 0:
1785	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1786	'\n'.join(no_import) + "\n")
1787	no_import = []
1788	em = '%d transactions commited\n' % tr_count
1789	transaction.commit()
1790	logger.info(em)
1791	total += tr_count
1792	tr_count = 0
1793	open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
1794	'\n'.join(no_import))
1795	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1796	###)
1797
1798
1799	security.declareProtected(ModifyPortalContent,"importReturningStudents")###(
1800	def importReturningStudents(self):
1801	"""load Returning Studentdata from CSV values"""
1802	import transaction
1803	import random
1804	#from pdb import set_trace
1805	wftool = self.portal_workflow
1806	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1807	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
1808	students_folder = self.portal_url.getPortalObject().campus.students
1809	tr_count = 1
1810	total = 0
1811	#name = 'pume_results'
1812	name = 'Returning'
1813	table = self.returning_import
1814	no_import = []
1815	imported = []
1816	logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')
1817	try:
1818	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1819	except:
1820	logger.error('Error reading %s.csv' % name)
1821	return
1822	l = self.portal_catalog({'meta_type': "Certificate"})
1823	certs = {}
1824	cert_docs = {}
1825	for f in l:
1826	certs[f.getId] = f.getObject().getContent()
1827	start = True
1828	res = table()
1829	regs = []
1830	if len(res) > 0:
1831	regs = [s.matric_no for s in res]
1832	#import pdb;pdb.set_trace()
1833	for student in returning:
1834	if start:
1835	start = False
1836	logger.info('Start loading from %s.csv' % name)
1837	s = ','.join(['"%s"' % fn for fn in student.keys()])
1838	imported.append(s)
1839	no_import.append('%s,"Error"' % s)
1840	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
1841	format_error = format + ',"%(Error)s"'
1842	no_certificate = "no certificate %s" % format
1843	student['matric_no'] = matric_no = student.get('matric_no').upper()
1844	student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
1845	student['Permanent_Address'] = perm_address = student.get('Permanent Address')
1846	if matric_no == '':
1847	student['Error'] = "Empty matric_no"
1848	no_import.append( format_error % student)
1849	continue
1850	if matric_no in regs or self.returning_import(matric_no = matric_no):
1851	student['Error'] = "Duplicate"
1852	no_import.append( format_error % student)
1853	continue
1854	cert_id = makeCertificateCode(student.get('Coursemajorcode'))
1855	if cert_id not in certs.keys():
1856	student['Error'] = "No Certificate %s" % cert_id
1857	no_import.append( format_error % student)
1858	continue
1859	try:
1860	table.addRecord(**student)
1861	except ValueError:
1862	student['Error'] = "Duplicate"
1863	no_import.append( format_error % student)
1864	continue
1865	regs.append(student.get('matric_no'))
1866	imported.append(format % student)
1867	tr_count += 1
1868	if tr_count > 1000:
1869	if len(no_import) > 0:
1870	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1871	'\n'.join(no_import) + '\n')
1872	no_import = []
1873	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
1874	'\n'.join(no_import) + "\n")
1875	imported = []
1876	em = '%d transactions commited total %s\n' % (tr_count,total)
1877	transaction.commit()
1878	regs = []
1879	logger.info(em)
1880	total += tr_count
1881	tr_count = 0
1882	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
1883	'\n'.join(imported))
1884	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1885	'\n'.join(no_import))
1886	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1887	###)
1888
1889	security.declareProtected(ModifyPortalContent,"fixAllNames")###(
1890	def fixAllNames(self):
1891	"fix all students names"
1892	import transaction
1893	response = self.REQUEST.RESPONSE
1894	logger = logging.getLogger('fixAllNames')
1895	logger.info('Start')
1896	students = self.portal_catalog(portal_type='Student')
1897	count = 0
1898	total = 0
1899	for student in students:
1900	scat_res = self.students_catalog(id = student.getId)
1901	if not scat_res:
1902	self.students_catalog.addRecord(id = student.getId)
1903	scat_res = self.students_catalog(id = student.getId)
1904	student_entry = scat_res[0]
1905	old_new = self.fixName(student,student_entry)
1906	count += 1
1907	response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))
1908	if count > 2000:
1909	transaction.commit()
1910	logger.info("%d transactions commited" % count)
1911	total += count
1912	count = 0
1913	###)
1914
1915	security.declareProtected(ModifyPortalContent,"fixName")###(
1916	def fixName(self,student_brain, student_entry):
1917	"fix the name of a student"
1918	fix = "first"
1919	if student_entry.get('name_fixed',None) == fix:
1920	return "Name already fixed"
1921	student_id = student_entry.id
1922	new_student = student_entry.jamb_reg_no.startswith('6')
1923	student_obj = student_brain.getObject()
1924	personal = getattr(student_obj,'personal',None)
1925	invalid = ''
1926	if personal is None:
1927	return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")
1928	per_doc = personal.getContent()
1929	old_first = per_doc.firstname
1930	old_middle = per_doc.middlename
1931	old_last = per_doc.lastname
1932	new_first = ''
1933	new_middle = ''
1934	new_last = ''
1935	if new_student:
1936	if not old_first and not old_middle and old_last:
1937	new_names = [n.capitalize() for n in old_last.split()]
1938	if len(new_names) > 1:
1939	old_first = new_names[0]
1940	old_last = new_names[-1]
1941	old_middle = ' '.join(new_names[1:-1])
1942	else:
1943	old_last = new_names[0]
1944	old_first = ''
1945	old_middle = ''
1946	if old_first:
1947	new_first = old_first
1948	if old_middle:
1949	new_middle = old_middle
1950	if old_last:
1951	new_last = old_last
1952	if old_first.find('<') != -1 or\
1953	old_first.find('>') != -1 or\
1954	old_middle.find('<') != -1 or\
1955	old_middle.find('>') != -1 or\
1956	old_last.find('<') != -1 or\
1957	old_last.find('>') != -1:
1958	invalid = "invalid characters"
1959	else:
1960	new_first = old_first
1961	if new_first.strip() == '-':
1962	new_first = ''
1963	new_middle = old_middle
1964	if new_middle.strip() == '-':
1965	new_middle = ''
1966	new_last = old_last
1967	if new_last.strip() == '-':
1968	new_last = ''
1969	name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()
1970	if new_student:
1971	text = "New"
1972	else:
1973	text = "Returning"
1974	old_new = '"%s","%s","%s","%s"' % (invalid,text,
1975	student_entry.name,
1976	name)
1977	if not invalid:
1978	self.students_catalog.modifyRecord(id = student_id,
1979	name_fixed = fix,
1980	name = name)
1981	per_doc.edit(mapping = {'firstname' : new_first,
1982	'middlename' : new_middle,
1983	'lastname' : new_last,
1984	})
1985	return old_new
1986	###)
1987
1988	security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(
1989	def updateReturningStudents(self):
1990	"""load and overwrite Returning Student Data from CSV values"""
1991	import transaction
1992	import random
1993	#from pdb import set_trace
1994	wftool = self.portal_workflow
1995	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1996	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
1997	students_folder = self.portal_url.getPortalObject().campus.students
1998	tr_count = 1
1999	total = 0
2000	#name = 'pume_results'
2001	name = 'Returning_update'
2002	table = self.returning_import
2003	no_import = []
2004	imported = []
2005	logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')
2006	try:
2007	returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
2008	except:
2009	logger.error('Error reading %s.csv' % name)
2010	return
2011	l = self.portal_catalog({'meta_type': "Certificate"})
2012	certs = {}
2013	cert_docs = {}
2014	for f in l:
2015	certs[f.getId] = f.getObject().getContent()
2016	start = True
2017	res = table()
2018	regs = []
2019	if len(res) > 0:
2020	regs = [s.matric_no for s in res]
2021	for student in returning:
2022	if start:
2023	start = False
2024	logger.info('Start loading from %s.csv' % name)
2025	s = ','.join(['"%s"' % fn for fn in student.keys()])
2026	imported.append(s)
2027	no_import.append('%s,"Error"' % s)
2028	format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
2029	format_error = format + ',"%(Error)s"'
2030	no_certificate = "no certificate %s" % format
2031	matric_no = student.get('matric_no').upper()
2032	student['matric_no'] = matric_no
2033	if matric_no == '':
2034	student['Error'] = "Empty matric_no"
2035	no_import.append( format_error % student)
2036	continue
2037	# if matric_no in regs or self.returning_import(matric_no = matric_no):
2038	# student['Error'] = "Duplicate"
2039	# no_import.append( format_error % student)
2040	# continue
2041	# cert_id = makeCertificateCode(student.get('Coursemajorcode'))
2042	# if cert_id not in certs.keys():
2043	# student['Error'] = "No Certificate %s" % cert_id
2044	# no_import.append( format_error % student)
2045	# continue
2046	try:
2047	table.modifyRecord(**student)
2048	except KeyError:
2049	#import pdb;pdb.set_trace()
2050	student['Error'] = "no Student found to update"
2051	no_import.append( format_error % student)
2052	continue
2053	#s = self.students_catalog(matric_no=matric_no)
2054	#if s:
2055	# level = "%s" % (int(student.get('Level')) + 100)
2056	# self.students_catalog.modifyRecord(id = s[0].id,
2057	# level=level)
2058
2059	regs.append(student.get('matric_no'))
2060	imported.append(format % student)
2061	tr_count += 1
2062	if tr_count > 1000:
2063	if len(no_import) > 0:
2064	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
2065	'\n'.join(no_import) + '\n')
2066	no_import = []
2067	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
2068	'\n'.join(no_import) + "\n")
2069	imported = []
2070	em = '%d transactions commited total %s\n' % (tr_count,total)
2071	transaction.commit()
2072	regs = []
2073	logger.info(em)
2074	total += tr_count
2075	tr_count = 0
2076	open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
2077	'\n'.join(imported))
2078	open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
2079	'\n'.join(no_import))
2080	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
2081	###)
2082
2083	security.declareProtected(ModifyPortalContent,"exportStudents")###(
2084	def exportStudents(self):
2085	"""export Studentsdata to a file"""
2086	member = self.portal_membership.getAuthenticatedMember()
2087	logger = logging.getLogger('Students.exportStudents')
2088	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2089	students_folder = self.portal_url.getPortalObject().campus.students
2090	export_file = "%s/export/students_%s.csv" % (i_home,current)
2091
2092	from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp
2093	aq_students = self.students_catalog.evalAdvancedQuery
2094	toexport = {'students_catalog': ("id",
2095	"matric_no",
2096	"jamb_reg_no",
2097	"name",
2098	"review_state",
2099	"entry_session",
2100	"session",
2101	"entry_mode",
2102	"mode",
2103	"verdict",
2104	"sex",
2105	"email",
2106	"phone",
2107	"faculty",
2108	"department",
2109	"course",
2110	"level",
2111	),
2112	'personal': ('firstname',
2113	'middlename',
2114	'lastname',
2115	'perm_address',
2116	),
2117	'clearance': ('state', # is not an attribute of clearance but is needed for splitting lga
2118	'lga',
2119	)
2120	}
2121	res_list = []
2122	lines = []
2123	fields = []
2124	with_lga = False
2125	for k in toexport.keys():
2126	for f in toexport[k]:
2127	if f == 'lga' :
2128	with_lga = True
2129	fields.append(f)
2130	headline = ','.join(fields).replace('level','current_level')
2131	open(export_file,"a").write(headline +'\n')
2132	format = '"%(' + ')s","%('.join(fields) + ')s"'
2133	query = In('review_state',('cleared_and_validated',
2134	'clearance_requested',
2135	'school_fee_paid',
2136	'courses_registered',
2137	'courses_validated'))
2138	#query = In('review_state',('clearance_requested'))
2139	students = aq_students(query)
2140	nr2export = len(students)
2141	logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))
2142	chunk = 1000
2143	total = 0
2144	start = DateTime.DateTime().timeTime()
2145	start_chunk = DateTime.DateTime().timeTime()
2146	## alternative method slightly slower
2147	# student_recs = {}
2148	# for s in students:
2149	# student_recs[s.id] = s
2150	# catalog_recs = {}
2151	# brains = self.portal_catalog(portal_type = 'Student')
2152	# for cat_rec in brains:
2153	# sid = cat_rec.getId
2154	# catalog_recs[sid] = cat_rec
2155	# #import pdb;pdb.set_trace()
2156	# start = DateTime.DateTime().timeTime()
2157	# start_chunk = DateTime.DateTime().timeTime()
2158	# for student in students:
2159	# if student.id not in student_recs.keys():
2160	# continue
2161	# not_all = False
2162	# d = self.getFormattedStudentEntry(student_recs[student.id])
2163	# student_obj = catalog_recs[student.id].getObject()
2164	for student in students:
2165	not_all = False
2166	d = self.getFormattedStudentEntry(student)
2167	student_obj = getattr(students_folder,student.id)
2168	for k in toexport.keys()[1:]:
2169	try:
2170	object = getattr(student_obj,k)
2171	object_doc = object.getContent()
2172	except:
2173	logger.info('%s %s record not found' % (student.id,k))
2174	not_all = True
2175	continue
2176	for f in toexport[k]:
2177	d[f] = getattr(object_doc,f,'')
2178	if not_all:
2179	continue
2180	if with_lga:
2181	d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas)
2182	lines.append(format % d)
2183	total += 1
2184	if total and not total % chunk or total == len(students):
2185	open(export_file,"a").write('\n'.join(lines) +'\n')
2186	anz = len(lines)
2187	logger.info("wrote %(anz)d total written %(total)d" % vars())
2188	end_chunk = DateTime.DateTime().timeTime()
2189	duration = end_chunk-start_chunk
2190	per_record = duration/anz
2191	till_now = end_chunk - start
2192	avarage_per_record = till_now/total
2193	estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
2194	estimated_end = estimated_end.strftime("%H:%M:%S")
2195	logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
2196	start_chunk = DateTime.DateTime().timeTime()
2197	lines = []
2198	end = DateTime.DateTime().timeTime()
2199	logger.info('total time %6.2f m' % ((end-start)/60))
2200	filename, extension = os.path.splitext(export_file)
2201	from subprocess import call
2202	msg = "wrote %(total)d records to %(export_file)s" % vars()
2203	try:
2204	retcode = call('gzip %s' % (export_file),shell=True)
2205	if retcode == 0:
2206	msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
2207	except OSError, e:
2208	retcode = -99
2209	logger.info("zip failed with %s" % e)
2210	logger.info(msg)
2211	args = {'portal_status_message': msg}
2212	#url = self.REQUEST.get('URL1') + '?' + urlencode(args)
2213	url = self.REQUEST.get('URL2')
2214	return self.REQUEST.RESPONSE.redirect(url)
2215	###)
2216
2217	security.declareProtected(ModifyPortalContent,"updateStudyCourse")###(
2218	def updateStudyCourse(self):
2219	"""update StudyCourse from CSV values"""
2220	import transaction
2221	import random
2222	from pdb import set_trace
2223	wftool = self.portal_workflow
2224	#students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
2225	students_folder = self.portal_url.getPortalObject().campus.students
2226	csv_d = {'jamb_reg_no': "RegNumber",
2227	'jamb_lastname': "Name",
2228	'session': "Session",
2229	'pume_tot_score': "PUME SCORE",
2230	'jamb_score': "JambScore",
2231	'jamb_sex': "Sex",
2232	'jamb_state': "State",
2233	## 'jamb_first_cos': "AdminCourse",
2234	'faculty': "AdminFaculty",
2235	'course_code': "AdmitCoscode",
2236	'stud_status':"AdmitStatus",
2237	'department': "AdmitDept",
2238	'jamb_lga': "LGA",
2239	'app_email': "email",
2240	'app_mobile': "PhoneNumbers",
2241	}
2242	csv_fields = [f[1] for f in csv_d.items()]
2243	tr_count = 0
2244	total = 0
2245	#name = 'pume_results'
2246	name = 'StudyCourseChange'
2247	no_import = []
2248	s = ','.join(['"%s"' % fn for fn in csv_fields])
2249	no_import.append('"Error",%s' % s)
2250	format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
2251	no_certificate = "no certificate %s" % format
2252	open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
2253	'\n'.join(no_import))
2254	logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse')
2255	logger.info('Start loading from %s.csv' % name)
2256	l = self.portal_catalog({'meta_type': "Certificate"})
2257	try:
2258	result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
2259	except:
2260	logger.error('Error reading %s.csv' % name)
2261	return
2262	for jamb in result:
2263	jamb['Error'] = "Processing "
2264	logger.info(format % jamb)
2265	jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
2266	res = self.portal_catalog({'portal_type': "StudentApplication",
2267	'SearchableText': jamb_reg_no })
2268	if not res:
2269	em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no
2270	logger.info(em)
2271	jamb['Error'] = "Student does not exist"
2272	no_import.append(format % jamb)
2273	continue
2274	sid = res[0].getPath().split('/')[-2]
2275	cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
2276	res = self.portal_catalog(portal_type = "Certificate", id = cert_id)
2277	if not res:
2278	em = 'No Certificate with ID %s \n' % cert_id
2279	logger.info(em)
2280	jamb['Error'] = "No Certificate %s" % cert_id
2281	no_import.append( format % jamb)
2282	continue
2283	cert_brain = res[0]
2284	catalog_entry = {}
2285	student = getattr(self,sid)
2286	#
2287	# Study Course
2288	#
2289	study_course = student.study_course
2290	dsc = {}
2291	cert_pl = cert_brain.getPath().split('/')
2292	catalog_entry['id'] = sid
2293	catalog_entry['faculty'] = cert_pl[-4]
2294	catalog_entry['department'] = cert_pl[-3]
2295	catalog_entry['course'] = cert_id
2296	dsc['study_course'] = cert_id
2297	study_course.getContent().edit(mapping=dsc)
2298	self.students_catalog.modifyRecord(**catalog_entry)
2299	if tr_count > 10:
2300	if len(no_import) > 1:
2301	open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write(
2302	'\n'.join(no_import))
2303	no_import = []
2304	em = '%d transactions commited\n' % tr_count
2305	transaction.commit()
2306	logger.info(em)
2307	total += tr_count
2308	tr_count = 0
2309	tr_count += 1
2310	return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
2311	###)
2312
2313
2314
2315	From WAeUP_Tool.py:
2316
2317
2318	def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###(
2319	"""load data from CSV values"""
2320	import transaction
2321	import random
2322	member = self.portal_membership.getAuthenticatedMember()
2323	logger = logging.getLogger('WAeUPTool.importData')
2324	current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2325	import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S")
2326	students_folder = self.portal_url.getPortalObject().campus.students
2327	start = True
2328	tr_count = 0
2329	total_imported = 0
2330	total_not_imported = 0
2331	total = 0
2332	pending_only = False
2333	pend_str = '_pending'
2334	if name.endswith(pend_str):
2335	pending_only = True
2336	name = name[:-len(pend_str)]
2337	iname = "import_%s" % name
2338	if name in ('application','course_result',):
2339	commit_after = 2000
2340	else:
2341	commit_after = 100
2342	stool = getToolByName(self, 'portal_schemas')
2343	ltool = getToolByName(self, 'portal_layouts')
2344	schema = stool._getOb(iname)
2345	if schema is None:
2346	em = 'No such schema %s' % iname
2347	logger.error('No such schema %s' % iname)
2348	return em
2349	layout = ltool._getOb(iname)
2350	if layout is None:
2351	em = 'No such layout %s' % iname
2352	logger.error(em)
2353	return em
2354	validators = {}
2355	for widget in layout.keys():
2356	validators[widget] = layout[widget].validate
2357	mode = "create"
2358	if edit:
2359	if filename.endswith('_toDelete'):
2360	mode = "delete"
2361	else:
2362	mode = "edit"
2363	importer_name = "mass_%(mode)s_%(name)s" % vars()
2364	importer = getattr(self, '%s' % importer_name,None)
2365	if importer is None:
2366	em = 'No importer function %s' % importer_name
2367	logger.error(em)
2368	return em
2369	pending_fn = "%s/import/%ss_pending.csv" % (i_home,name)
2370	pending_imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
2371	if pending_only:
2372	import_source_fn = pending_fn
2373	imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
2374	not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current)
2375	if not os.path.exists(pending_fn):
2376	em = 'No %s' % os.path.split(pending_fn)
2377	return em
2378	else:
2379	import_source_fn = "%s/import/%s.csv" % (i_home,filename)
2380	imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
2381	not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
2382	if not os.path.exists(import_source_fn):
2383	em = 'No %s' % os.path.split(import_soure_fn)
2384	return em
2385	attrs = csv.reader(open(import_source_fn,"rb")).next()
2386	import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
2387	or k.strip() == 'Error')]
2388	diff2schema = set(import_keys).difference(set(schema.keys()))
2389	diff2layout = set(import_keys).difference(set(layout.keys()))
2390	if diff2schema and diff2schema != set(['id',]):
2391	msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2392	return msg
2393	if mode in ("create","edit"):
2394	required_keys = [layout.getIdUnprefixed(id)
2395	for id,widget in layout.objectItems()
2396	if widget.is_required]
2397	if not set(required_keys).issubset(set(import_keys)):
2398	diff2import = set(required_keys).difference(set(import_keys))
2399	msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2400	return msg
2401	#
2402	# not_imported
2403	#
2404	not_imported_keys = import_keys + ["Error",]
2405	not_imported_file = open(not_imported_fn,"w")
2406	not_imported_csv_writer = csv.DictWriter(not_imported_file,
2407	not_imported_keys,
2408	extrasaction='ignore')
2409	mapping = dict((k,k) for k in not_imported_keys)
2410	not_imported_csv_writer.writerow(mapping)
2411	not_imported = []
2412	#
2413	# imported
2414	#
2415	imported_keys = import_keys[:]
2416	if 'id' not in imported_keys:
2417	imported_keys.insert(0,"id")
2418	imported_file = open(imported_fn,"w")
2419	imported_csv_writer = csv.DictWriter(imported_file,
2420	imported_keys,
2421	extrasaction='ignore')
2422	mapping = dict((k,k) for k in imported_keys)
2423	imported_csv_writer.writerow(mapping)
2424	imported = []
2425	#
2426	# pending
2427	#
2428	pending_records = {}
2429	imports_pending = 0
2430	total_pending_imported = 0
2431	total_pending = 0
2432	format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2433	format_error = format + ',"%(Error)s"'
2434	#format = '"%(id)s",'+ format
2435	imported = []
2436	if name in ('verdict','course_result',):
2437	#pending_keys = imported_keys[:]
2438	sname = "%s_pending" % iname
2439	pending_schema = stool._getOb(sname)
2440	if schema is None:
2441	em = 'No such pending_schema %s' % sname
2442	logger.error('No such schema %s' % sname)
2443	return em
2444	pending_keys = pending_schema.keys()
2445	pending_keys += "Error",
2446	if 'id' not in pending_keys:
2447	pending_keys.insert(0,'id')
2448	pending_records = {}
2449	if os.path.exists(pending_fn):
2450	pending_imports_file = open(pending_fn)
2451	pending_imports = csv.DictReader(pending_imports_file,
2452	fieldnames=pending_keys)
2453	if pending_imports:
2454	pending_records = dict((row['id'],row) for row in pending_imports
2455	if row['id'] != 'id')
2456	pending_imports_file.close()
2457	new_imports_pending = 0
2458	pending_imported = []
2459	if pending_only:
2460	pending_imported_ids = []
2461	pending_imported = []
2462	for record in pending_records.values():
2463	item = {}
2464	for k,v in record.items():
2465	if v:
2466	item[k] = v
2467	results = importer(item)
2468	id = results[0]
2469	error = results[1]
2470	is_pending = len(results) == 3
2471	if is_pending:
2472	continue
2473	del item['Error']
2474	msg = ";".join(["%s : %s" % (k,v) for k,v in item.items()])
2475	logger.info("imported from %(pending_fn)s %(msg)s" % vars())
2476	pending_imported.append(item)
2477	pending_imported_ids += id,
2478	total_pending_imported = len(pending_imported_ids)
2479	logger.info("imported %d pending imports" % len(pending_imported_ids))
2480	for id in pending_imported_ids:
2481	if id:
2482	del pending_records[id]
2483	else:
2484	logger.info("tried to delete record with empty id")
2485	pending_imports_file = open(pending_fn,'w')
2486	pending_csv_writer = csv.DictWriter(pending_imports_file,
2487	pending_keys,
2488	extrasaction='ignore')
2489	mapping = dict((k,k) for k in pending_keys)
2490	pending_csv_writer.writerow(mapping)
2491	if len(pending_imported) > 0:
2492	pending_imported_file = open(pending_imported_fn,'w')
2493	pending_imported_csv_writer = csv.DictWriter(pending_imported_file,
2494	pending_keys,
2495	extrasaction='ignore')
2496	pending_imported_csv_writer.writerow(dict((k,k) for k in imported_keys))
2497	pending_imported_csv_writer.writerows(pending_imported)
2498	#
2499	if pending_only:
2500	items = []
2501	else:
2502	try:
2503	items = csv.DictReader(open(import_source_fn,"rb"))
2504	except:
2505	msg = 'Error reading %s.csv' % filename
2506	logger.error(msg)
2507	return msg
2508	not_imported = []
2509	pending_keys = pending_records.keys()[:]
2510	for item in items:
2511	item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2512	if start:
2513	start = False
2514	adapters = [MappingStorageAdapter(schema, item)]
2515	logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2516	dm = DataModel(item, adapters,context=self)
2517	ds = DataStructure(data=item,datamodel=dm)
2518	error_string = ""
2519	total += 1
2520	error_count = 0
2521	for k in import_keys:
2522	if not validators[k](ds,mode=mode):
2523	if error_count:
2524	error_string += ' ++ '
2525	error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
2526	ds.getErrorMapping(k)))
2527	error_count += 1
2528	if error_string:
2529	item['Error'] = error_string
2530	not_imported.append(item)
2531	total_not_imported += 1
2532	continue
2533	temp_item = item.copy()
2534	temp_item.update(dm)
2535	#id,error = importer(temp_item)
2536	results = importer(temp_item)
2537	id = results[0]
2538	error = results[1]
2539	is_pending = len(results) == 3
2540	if is_pending:
2541	temp_item = results[2]
2542	temp_item['Error'] = error
2543	msg = format_error % temp_item
2544	#if id not in pending_records.keys():
2545	if id not in pending_keys:
2546	temp_item['id'] = id
2547	pending_records[id] = temp_item
2548	logger.info("%(id)s added to pending %(msg)s" % vars())
2549	pending_keys += id,
2550	new_imports_pending += 1
2551	else:
2552	logger.info("%(id)s already in pending %(msg)s" % vars())
2553	continue
2554	elif error:
2555	item['Error'] = error
2556	not_imported.append(item)
2557	total_not_imported += 1
2558	continue
2559	item = temp_item
2560	item['id'] = id
2561	imported.append(item)
2562	tr_count += 1
2563	total_imported += 1
2564	msg = format % item
2565	logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
2566	if total and not total % commit_after:
2567	transaction.commit()
2568	if len(imported) > 0:
2569	imported_csv_writer.writerows(imported)
2570	imported = []
2571	if len(not_imported) > 0:
2572	not_imported_csv_writer.writerows(not_imported)
2573	not_imported = []
2574	if len(pending_records) > 0:
2575	pending_csv_writer.writerows(pending_records.values())
2576	total_pending += len(pending_records)
2577	pending_records = {}
2578	msg = '%d transactions committed\n' % (tr_count)
2579	regs = []
2580	logger.info(msg)
2581	tr_count = 0
2582	if len(imported) > 0:
2583	imported_csv_writer.writerows(imported)
2584	if len(not_imported) > 0:
2585	not_imported_csv_writer.writerows(not_imported)
2586	if len(pending_records) > 0:
2587	pending_csv_writer.writerows(pending_records.values())
2588	total_pending += len(pending_records)
2589	import_source_fn = os.path.split(import_source_fn)[1]
2590	pending_fn = os.path.split(pending_fn)[1]
2591	msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
2592	msg += "%(total_imported)d imported, %(total_not_imported)d not imported, " % vars()
2593	if total_pending:
2594	if pending_only:
2595	msg += "%(new_imports_pending)d pending added, %(total_pending_imported)d pending imported " % vars()
2596	else:
2597	msg += "%(new_imports_pending)d pending added " % vars()
2598	msg += "(of total %(total)d), " % vars()
2599	msg += "%(total_pending)d total pending in %(pending_fn)s " % vars()
2600	logger.info(msg)
2601	return msg
2602	###)

Note: See TracBrowser for help on using the repository browser.

Context navigation

source: WAeUP_SRP/trunk/unused_methods.txt @ 17718

Download in other formats: