#-*- mode: python; mode: fold -*- # $Id: Students.py 2654 2007-11-14 17:54:25Z henrik $ from string import Template from Globals import InitializeClass from AccessControl import ClassSecurityInfo from AccessControl.SecurityManagement import newSecurityManager from zExceptions import BadRequest from Products.ZCatalog.ZCatalog import ZCatalog from Products.CMFCore.utils import UniqueObject, getToolByName from Products.CMFCore.permissions import View from Products.CMFCore.permissions import ModifyPortalContent from Products.CPSCore.CPSBase import CPSBase_adder, CPSBaseFolder #from Products.CPSCore.CPSBase import CPSBaseDocument as BaseDocument from Products.CPSDocument.CPSDocument import CPSDocument from Products.CPSCore.CPSBase import CPSBaseBTreeFolder as BaseBTreeFolder from Products.CPSCore.CPSMembershipTool import CPSUnrestrictedUser from Products.WAeUP_SRP.Academics import makeCertificateCode from Products.AdvancedQuery import Eq, Between, Le,In import DateTime import logging import csv,re,os import Globals p_home = Globals.package_home(globals()) i_home = Globals.INSTANCE_HOME MAX_TRANS = 1000 from urllib import urlencode import DateTime #import PIL.Image from StringIO import StringIO def makeCertificateCode(code): ###( code = code.replace('.','') code = code.replace('(','') code = code.replace(')','') code = code.replace('/','') code = code.replace(' ','') code = code.replace('_','') return code ###) def response_write(response,s): ###( response.setHeader('Content-type','text/html; charset=ISO-8859-15') while s.find('<') > -1: s = s.replace('<','<') while s.find('>') > -1: #from pdb import set_trace;set_trace() s = s.replace('>','>') response.write("%s
\n" % s) ###) def getInt(s): ###( try: return int(s) except: return 0 def getFloat(s): try: return float(s) except: return 0.0 ###) def getStudentByRegNo(self,reg_no): ###( """search student by JAMB Reg No and return StudentFolder""" res = self.students_catalog(jamb_reg_no = reg_no.upper()) if len(res) == 1: return getattr(self.portal_url.getPortalObject().campus.students,res[0].id) else: return None # don't search in portal_catalog # search = ZCatalog.searchResults(self.portal_catalog_real,{'meta_type': 'StudentApplication', # 'SearchableText': reg_no, # }) # if len(search) < 1: # return None # return search[0].getObject().aq_parent ###) def checkJambNo(jnr): ###( try: if len(jnr) != 10: return False except: return False try: int(jnr[:8]) return True except: return False ###) def formatLGA(lga,voc=None): if voc is not None: if voc.has_key(lga): state,lga = voc[lga].split(' / ') elif lga.find(' / ') > -1: state,lga = lga.split(' / ') else: state,lga = "",lga return state.upper(),lga.upper() class StudentsFolder(CPSDocument): ###( """ WAeUP container for the various WAeUP containers data """ meta_type = 'StudentsFolder' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(ModifyPortalContent,"transferStudents")###( def transferStudents(self,filename): """ load Interfaculty transferStudents Studentdata from CSV values. """ import transaction import random current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") pm = self.portal_membership member = pm.getAuthenticatedMember() logger = logging.getLogger('Students.StudentsFolder.transferStudents') #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() students_folder = self.portal_url.getPortalObject().campus.students csv_fields = ('old_matric_no', 'matric_no', 'study_course', 'current_mode', 'current_level', ) tr_count = 0 total = 0 total_imported = 0 total_not_imported = 0 imported = [] not_imported = [] certs = {} try: results = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb")) except: logger.error('Error reading %s.csv' % filename) return start = True for result in results: total += 1 if start: start = False logger.info('%s starts import from %s.csv' % (member,filename)) import_keys = [k for k in result.keys() if not k.startswith('ignore')] diff2schema = set(import_keys).difference(set(csv_fields)) if diff2schema: em = "not ignorable key(s) %s found in heading" % diff2schema return em s = ','.join(['"%s"' % fn for fn in import_keys]) open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n') s = '"id",' + s open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n') format = ','.join(['"%%(%s)s"' % fn for fn in import_keys]) format_error = format + ',"%(Error)s"' format = '"%(id)s",'+ format old_matric_no = result.get('old_matric_no') res = self.students_catalog(matric_no = old_matric_no) result['id'] = "None" if not res: em = 'Student with matric_no %s not found' % old_matric_no logger.info(em) result['Error'] = "Student does not exist" not_imported.append(format_error % result) total_not_imported += 1 continue student_brain = res[0] student_object = getattr(students_folder,student_brain.id) result['id'] = student_brain.id cert_id = makeCertificateCode(result.get('study_course')) if cert_id not in certs.keys(): res = self.portal_catalog(meta_type = "Certificate",id = cert_id) if not res: em = 'No certificate with ID %s \n' % cert_id logger.info(em) result['Error'] = "No Certificate %s" % cert_id not_imported.append( format_error % result) total_not_imported += 1 continue cert = res[0] cert_path = cert.getPath().split('/') certificate = certs[cert_id] = {'faculty': cert_path[-4], 'department': cert_path[-3]} cert_doc = certs[cert_id] clearance = getattr(student_object,'clearance',None) if clearance is None: em = 'Student has no clearance object' logger.info('%s (%s) %s' % (student_brain.id, old_matric_no, em)) result['Error'] = em not_imported.append( format_error % result) total_not_imported += 1 continue clearance_doc = clearance.getContent() study_course = student_object.study_course study_course_doc = study_course.getContent() old_study_course = study_course_doc.study_course old_current_level = study_course_doc.current_level current_level = result.get('current_level',None) new_study_course = result.get('study_course',None) try: icl = int(current_level) except: em = 'Invalid new level %s' % current_level logger.info(em) result['Error'] = em not_imported.append( format_error % result) total_not_imported += 1 continue if icl == int(old_current_level) and old_study_course == new_study_course: em = 'Already transferred' logger.info('%s (%s) %s' % (student_brain.id, old_matric_no, em)) result['Error'] = em not_imported.append( format_error % result) total_not_imported += 1 continue if study_course.objectIds(): em = 'Already registered level %s for %s, but is supposed to study %s at level %s' % (old_current_level,old_study_course,new_study_course,current_level) logger.info('%s (%s) %s' % (student_brain.id, old_matric_no, em)) result['Error'] = em not_imported.append( format_error % result) total_not_imported += 1 continue #from pdb import set_trace; set_trace() cd = {} matric_no_history = getattr(clearance_doc,'matric_no_history',[]) if not matric_no_history: matric_no_history = [] matric_no_history.append(old_matric_no) cd['matric_no_history'] = matric_no_history cd['matric_no'] = result.get('matric_no') clearance_doc.edit(mapping = cd) dsc = {} study_course_history = getattr(study_course_doc,'study_course_history',[]) if not study_course_history: study_course_history = [] study_course_history.append(old_study_course) dsc['study_course_history'] = study_course_history dsc['study_course'] = new_study_course dsc['current_level'] = current_level dsc['current_mode'] = result.get('current_mode') study_course_doc.edit(mapping=dsc) imported.append( format % result) tr_count += 1 total_imported += 1 if tr_count > 1000: if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported) + '\n') not_imported = [] if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported) + '\n') imported = [] em = '%d transactions committed\n' % (tr_count) transaction.commit() regs = [] logger.info(em) tr_count = 0 if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported)) if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported)) em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total) logger.info(em) return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) ###) security.declareProtected(ModifyPortalContent,"dumpStudentsCatalog")###( def dumpStudentsCatalog(self): """dump all data in students_catalog to a csv""" member = self.portal_membership.getAuthenticatedMember() logger = logging.getLogger('Students.dumpStudentsCatalog') current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") export_file = "%s/export/students_catalog_%s.csv" % (i_home,current) res_list = [] lines = [] fields = [] for f in self.students_catalog.schema(): fields.append(f) fields.append('state') headline = ','.join(fields) #open(export_file,"a").write(headline +'\n') out = open(export_file,"wb") out.write(headline +'\n') out.close() out = open(export_file,"a") csv_writer = csv.DictWriter(out,fields,) format = '"%(' + ')s","%('.join(fields) + ')s"' students = self.students_catalog() nr2export = len(students) logger.info('%s starts dumpStudentsCatalog, %s student records to export' % (member,nr2export)) chunk = 2000 total = 0 start = DateTime.DateTime().timeTime() start_chunk = DateTime.DateTime().timeTime() for student in students: not_all = False d = self.getFormattedStudentEntry(student) d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas) #lines.append(format % d) lines.append(d) total += 1 if total and not total % chunk or total == len(students): #open(export_file,"a").write('\n'.join(lines) +'\n') csv_writer.writerows(lines) anz = len(lines) logger.info("wrote %(anz)d total written %(total)d" % vars()) end_chunk = DateTime.DateTime().timeTime() duration = end_chunk-start_chunk per_record = duration/anz till_now = end_chunk - start avarage_per_record = till_now/total estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export) estimated_end = estimated_end.strftime("%H:%M:%S") logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars()) start_chunk = DateTime.DateTime().timeTime() lines = [] end = DateTime.DateTime().timeTime() logger.info('total time %6.2f m' % ((end-start)/60)) filename, extension = os.path.splitext(export_file) from subprocess import call msg = "wrote %(total)d records to %(export_file)s" % vars() try: retcode = call('gzip %s' % (export_file),shell=True) if retcode == 0: msg = "wrote %(total)d records to %(export_file)s.gz" % vars() except OSError, e: retcode = -99 logger.info("zip failed with %s" % e) logger.info(msg) args = {'portal_status_message': msg} #url = self.REQUEST.get('URL1') + '?' + urlencode(args) url = self.REQUEST.get('URL2') return self.REQUEST.RESPONSE.redirect(url) ###) security.declareProtected(ModifyPortalContent,"importResults")###( def importResults(self): """load Returning Students Results from CSV""" import transaction import random #from pdb import set_trace wftool = self.portal_workflow current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() students_folder = self.portal_url.getPortalObject().campus.students tr_count = 1 total = 0 #name = 'pume_results' name = 'Results' table = self.results_import no_import = [] imported = [] logger = logging.getLogger('Students.StudentsFolder.importResults') try: results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) except: logger.error('Error reading %s.csv' % name) return l = self.portal_catalog({'meta_type': "Course"}) courses = [f.getId for f in l] start = True res = table() regs = [] if len(res) > 0: regs = [s.key for s in res] no_course = [] no_course_list = [] course_count = 0 for result in results: if start: start = False logger.info('Start loading from %s.csv' % name) s = ','.join(['"%s"' % fn for fn in result.keys()]) imported.append(s) no_import.append('%s,"Error"' % s) format = ','.join(['"%%(%s)s"' % fn for fn in result.keys()]) format_error = format + ',"%(Error)s"' no_certificate = "no certificate %s" % format course_id = result.get('CosCode') if not course_id: course_id = 'N/A' result['CosCode'] = course_id matric_no = result.get('matric_no').upper() result['matric_no'] = matric_no key = matric_no+course_id if matric_no == '': result['Error'] = "Empty matric_no" no_import.append( format_error % result) continue if key in regs or self.results_import(key = key): result['Error'] = "Duplicate" no_import.append( format_error % result) continue if course_id not in courses: if course_id not in no_course: course_count +=1 no_course.append(course_id) no_course_list.append('"%s"' % course_id) #result['Error'] = "No Course" #logger.info(format_error % result) result['key'] = key try: table.addRecord(**result) except ValueError: #import pdb;pdb.set_trace() result['Error'] = "Duplicate" no_import.append( format_error % result) continue regs.append(key) imported.append(format % result) tr_count += 1 if tr_count > 1000: if len(no_import) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import)+'\n') no_import = [] open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( '\n'.join(imported) + '\n') imported = [] if no_course_list: open("%s/import/%sno_courses%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_course_list) + '\n') no_course_list = [] transaction.commit() regs = [] total += tr_count em = '%d transactions totally comitted, %s courses not found ' % (total,course_count) logger.info(em) tr_count = 0 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( '\n'.join(imported)) open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import)) if no_course_list: open("%s/import/%sno_courses%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_course_list)) em = '%d transactions totally committed, %s courses not found ' % (total,course_count) logger.info(em) return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) ###) security.declareProtected(View,"fixOwnership") ###( def fixOwnership(self): """fix Ownership""" for s in self.portal_catalog(meta_type = 'Student'): student = s.getObject() sid = s.getId import pdb;pdb.set_trace() student.application.manage_setLocalRoles(sid, ['Owner',]) student.personal.manage_setLocalRoles(sid, ['Owner',]) ###) security.declareProtected(View,"Title") ###( def Title(self): """compose title""" return "Student Section" ###) def generateStudentId(self,letter,students = None): ###( import random r = random if students is None: students = self.portal_url.getPortalObject().campus.students if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') sid = "%c%d" % (letter,r.randint(99999,1000000)) while hasattr(students, sid): sid = "%c%d" % (letter,r.randint(99999,1000000)) return sid #return "%c%d" % (r.choice('ABCDEFGHKLMNPQRSTUVWXY'),r.randint(99999,1000000)) ###) InitializeClass(StudentsFolder) def addStudentsFolder(container, id, REQUEST=None, **kw): ###( """Add a Student.""" ob = StudentsFolder(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) ###) class Student(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'Student' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" reg_nr = self.getId()[1:] data = getattr(self,'personal',None) if data: content = data.getContent() return "%s %s %s" % (content.firstname,content.middlename,content.lastname) data = getattr(self,'application',None) if data: content = data.getContent() return "%s" % (content.jamb_lastname) return self.title security.declarePrivate('makeStudentMember') ###( def makeStudentMember(self,sid,password='uNsEt'): """make the student a member""" membership = self.portal_membership membership.addMember(sid, password , roles=('Member', 'Student', ), domains='', properties = {'memberareaCreationFlag': False, 'homeless': True},) member = membership.getMemberById(sid) self.portal_registration.afterAdd(member, sid, password, None) self.manage_setLocalRoles(sid, ['Owner',]) ###) security.declareProtected(View,'createSubObjects') ###( def createSubObjects(self): """make the student a member""" dp = {'Title': 'Personal Data'} app_doc = self.application.getContent() names = app_doc.jamb_lastname.split() if len(names) == 3: dp['firstname'] = names[0].capitalize() dp['middlename'] = names[1].capitalize() dp['lastname'] = names[2].capitalize() elif len(names) == 2: dp['firstname'] = names[0].capitalize() dp['lastname'] = names[1].capitalize() else: dp['lastname'] = app_doc.jamb_lastname dp['sex'] = app_doc.jamb_sex == 'F' dp['lga'] = "%s/%s" % (app_doc.jamb_state,app_doc.jamb_lga ) proxy = self.aq_parent proxy.invokeFactory('StudentPersonal','personal') per = proxy.personal per_doc = per.getContent() per_doc.edit(mapping = dp) per.manage_setLocalRoles(proxy.getId(), ['Owner',]) #self.portal_workflow.doActionFor(per,'open',dest_container=per) ###) InitializeClass(Student) def addStudent(container, id, REQUEST=None, **kw): """Add a Student.""" ob = Student(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentAccommodation(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentAccommodation' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" content = self.getContent() #return "Accommodation Data for %s %s" % (content.firstname,content.lastname) return "Accommodation Data for Session %s" % content.session InitializeClass(StudentAccommodation) def addStudentAccommodation(container, id, REQUEST=None, **kw): """Add a Students personal data.""" ob = StudentAccommodation(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentPersonal(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentPersonal' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" content = self.getContent() #return "Personal Data for %s %s" % (content.firstname,content.lastname) return "Personal Data" InitializeClass(StudentPersonal) def addStudentPersonal(container, id, REQUEST=None, **kw): """Add a Students personal data.""" ob = StudentPersonal(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentClearance(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentClearance' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" content = self.getContent() #return "Clearance/Eligibility Record for %s %s" % (content.firstname,content.lastname) return "Clearance/Eligibility Record" InitializeClass(StudentClearance) def addStudentClearance(container, id, REQUEST=None, **kw): """Add a Students personal data.""" ob = StudentClearance(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentStudyLevel(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentStudyLevel' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" return self.portal_vocabularies.student_levels.get(self.aq_parent.getId()) #return "Level %s" % self.aq_parent.getId() def create_course_results(self,cert_id,current_level): ###( "create all courses in a level" aq_portal = self.portal_catalog.evalAdvancedQuery res = self.portal_catalog(portal_type="Certificate", id = cert_id) l = [] import transaction if res: cert = res[0] path = cert.getPath() query = Eq("path","%s/%s" % (path,current_level)) &\ Eq('portal_type','CertificateCourse') courses = aq_portal(query) #from pdb import set_trace;set_trace() self_proxy = self.aq_parent for c in courses: d = self.getCourseInfo(c.getId) cr_id = self_proxy.invokeFactory('StudentCourseResult',c.getId) course_result = getattr(self_proxy,cr_id) self.portal_workflow.doActionFor(course_result,'open') d['core_or_elective'] = getattr(c.getObject().getContent(),'core_or_elective') course_result.getContent().edit(mapping=d) #transaction.commit() ###) InitializeClass(StudentStudyLevel) def addStudentStudyLevel(container, id, REQUEST=None, **kw): """Add a Students personal data.""" ob = StudentStudyLevel(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentStudyCourse(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentStudyCourse' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" content = self.getContent() return "Study Course" InitializeClass(StudentStudyCourse) def addStudentStudyCourse(container, id, REQUEST=None, **kw): """Add a Students personal data.""" ob = StudentStudyCourse(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentApplication(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentApplication' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" return "Application Data" InitializeClass(StudentApplication) def addStudentApplication(container, id, REQUEST=None, **kw): """Add a Students eligibility data.""" ob = StudentApplication(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) class StudentPume(CPSDocument): ###( """ WAeUP Student container for the various student data """ meta_type = 'StudentPume' portal_type = meta_type security = ClassSecurityInfo() security.declareProtected(View,"Title") def Title(self): """compose title""" return "PUME Results" InitializeClass(StudentPume) def addStudentPume(container, id, REQUEST=None, **kw): """Add a Students PUME data.""" ob = StudentPume(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) ##class StudentSemester(CPSDocument): ###( ## """ ## WAeUP StudentSemester containing the courses and students ## """ ## meta_type = 'StudentSemester' ## portal_type = meta_type ## security = ClassSecurityInfo() ## ##InitializeClass(StudentSemester) ## ##def addStudentSemester(container, id, REQUEST=None, **kw): ## """Add a StudentSemester.""" ## ob = StudentSemester(id, **kw) ## return CPSBase_adder(container, ob, REQUEST=REQUEST) ## #####) ##class Semester(CPSDocument): ###( ## """ ## WAeUP Semester containing the courses and students ## """ ## meta_type = 'Semester' ## portal_type = meta_type ## security = ClassSecurityInfo() ## ##InitializeClass(Semester) ## ##def addSemester(container, id, REQUEST=None, **kw): ## """Add a Semester.""" ## ob = Semester(id, **kw) ## return CPSBase_adder(container, ob, REQUEST=REQUEST) ## #####) class StudentCourseResult(CPSDocument): ###( """ WAeUP StudentCourseResult """ meta_type = 'StudentCourseResult' portal_type = meta_type security = ClassSecurityInfo() def getCourseEntry(self,cid): res = self.portal_catalog({'meta_type': "Course", 'id': cid}) if res: return res[-1] else: return None security.declareProtected(View,"Title") def Title(self): """compose title""" cid = self.aq_parent.getId() ce = self.getCourseEntry(cid) if ce: return "%s" % ce.Title return "No course with id %s" % cid InitializeClass(StudentCourseResult) def addStudentCourseResult(container, id, REQUEST=None, **kw): """Add a StudentCourseResult.""" ob = StudentCourseResult(id, **kw) return CPSBase_adder(container, ob, REQUEST=REQUEST) ###) # Backward Compatibility StudyLevel from Products.WAeUP_SRP.Academics import StudyLevel from Products.WAeUP_SRP.Academics import addStudyLevel