- Timestamp:
- 14 Nov 2007, 17:54:25 (17 years ago)
- Location:
- WAeUP_SRP/base
- Files:
-
- 2 edited
-
Students.py (modified) (3 diffs)
-
Students.unused_methods.txt (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
WAeUP_SRP/base/Students.py
r2640 r2654 116 116 security = ClassSecurityInfo() 117 117 118 security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(119 def loadPumeResultsFromCSV(self):120 """load Fulltime Studentdata from CSV values into pumeresults catalog"""121 import transaction122 import random123 ## csv_d = {'jamb_reg_no': "RegNumber", ###(124 ## 'status': "Admission Status",125 ## 'name': "Name",126 ## 'score': "Score",127 ## 'sex': "Sex",128 ## 'faculty': "Faculty",129 ## 'department': "Dept",130 ## 'course': "Course",131 ## 'course_code_org': "Course Code",132 ## }133 ###)134 ## csv_d = {'jamb_reg_no': "JAMBRegno",135 ## 'name': "Name",136 ## 'score': "Score",137 ## 'sex': "Sex",138 ## 'course': "Course",139 ## 'faculty': "Faculty",140 ## 'department': "Dept",141 ## 'course_code_org': "Course Code",142 ## 'status': "Admission Status",143 ## 'result_type': None,144 ## }145 146 csv_d = {'jamb_reg_no': "reg_no",147 'name': "fullname",148 'score': "pume_score",149 'sex': "sex",150 'course': "study_course",151 'course_code_org': "study_course",152 'status': "admission_status",153 'result_type': "entry_mode",154 }155 156 csv_fields = [f[1] for f in csv_d.items() if f[1]]157 tr_count = 0158 total = 0159 #name = 'pup_new'160 #name = 'pup_update'161 name = 'Admitted_update'162 update = name.endswith('update')163 no_import = []164 ok_import = []165 ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))166 no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))167 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")168 ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)169 #open(ok_import_name,"w").write('\n'.join(no_import))170 no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)171 #open(no_import_name,"w").write('\n'.join(no_import))172 logger = logging.getLogger('Students.loadPumeResultsFromCSV')173 starttime = DateTime.now()174 logger.info('Start loading from %s.csv' % name)175 try:176 result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))177 except:178 logger.error('Error reading %s.csv' % name)179 return180 pume = self.portal_pumeresults181 format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])182 import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])183 eduplicate = '%s,"duplicate"' % format184 eoriginal = '%s,"original"' % format185 einvalidjamb = '%s,"invalid JambRegNo"' % format186 added = 'added ,%s' % format187 #from pdb import set_trace;set_trace()188 for jamb in result:189 if not jamb.get(csv_d['score']):190 logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no']))191 continue192 dict = {}193 for f,fn in csv_d.items():194 dict[f] = jamb.get(csv_d[f])195 dict['result_type'] = 'CEST'196 jnr = jamb.get(csv_d['jamb_reg_no'])197 #if not checkJambNo(jnr):198 # logger.info(einvalidjamb % jamb)199 # dd = {}200 # for f,fn in csv_d.items():201 # dd[fn] = getattr(data,f)202 # no_import.append(eduplicate % dd)203 # no_import.append(eduplicate % jamb)204 # continue205 res = pume(jamb_reg_no=jnr)206 if len(res) > 0:207 if update:208 try:209 pume.modifyRecord(**dict)210 # Can not happen, but anyway...211 except ValueError:212 logger.info(eduplicate % jamb)213 continue214 # Can not happen, but anyway...215 except KeyError:216 pume.addRecord(**dict)217 logger.info(added % jamb)218 continue219 else:220 data = res[0]221 if data.name != jamb.get(csv_d['name']):222 #set_trace()223 logger.info(eduplicate % jamb)224 #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict225 #logger.info(em)226 dd = {}227 for f,fn in csv_d.items():228 dd[fn] = getattr(data,f)229 no_import.append(eoriginal % dd)230 no_import.append(eduplicate % jamb)231 continue232 else:233 try:234 pume.addRecord(**dict)235 ok_import.append(import_format % dict)236 except ValueError:237 logger.info(eduplicate % jamb)238 #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict239 #logger.info(em)240 no_import.append(eduplicate % jamb)241 logger.info('End loading from %s.csv' % name)242 if len(no_import) > 1:243 open(no_import_name,"w+").write('\n'.join(no_import))244 open(ok_import_name,"w+").write('\n'.join(ok_import))245 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))246 ###)247 248 security.declareProtected(ModifyPortalContent,"createStudents")###(249 def createStudents(self):250 """251 load addmitted Studentdata from CSV values and create Studentobjects.252 This is the current method to create new addmitted Students.253 Before running the eventservice for the students_catalog must be disabled.254 """255 import transaction256 import random257 #from pdb import set_trace258 wftool = self.portal_workflow259 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()260 students_folder = self.portal_url.getPortalObject().campus.students261 levels = {'ume_ft':'100',262 'de_ft': '200',263 'ug_pt': '100',264 'de_pt': '200',265 'pg_ft': '700',266 'pg_pt': '700',267 'dp_pt': '100',268 'dp_ft': '100',269 }270 csv_d = {'jamb_reg_no': "reg_no",271 'entry_mode': 'entry_mode',272 'jamb_firstname': "firstname",273 'jamb_middlename': "middlename",274 'jamb_lastname': "lastname",275 'jamb_sex': "sex",276 'jamb_state': "state",277 'birthday': "date_of_birth",278 'app_email': "email",279 'study_course': "study_course",280 'perm_address': "address",281 'admission_status': "admission_status",282 }283 csv_fields = [f[1] for f in csv_d.items()]284 tr_count = 0285 total = 0286 #name = 'pume_results'287 name = 'Admitted_update'288 no_import = []289 s = ','.join(['"%s"' % fn for fn in csv_fields])290 no_import.append('"Error",%s' % s)291 format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])292 no_certificate = "no certificate %s" % format293 open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))294 logger = logging.getLogger('Students.StudentsFolder.createStudents')295 logger.info('Start loading from %s.csv' % name)296 certs = {}297 try:298 results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))299 except:300 logger.error('Error reading %s.csv' % name)301 return302 for result in results:303 if not result.get(csv_d['admission_status']).startswith('Admitted'):304 continue305 #result['Error'] = "Processing "306 #logger.info(format % result)307 jamb_reg_no = result.get(csv_d['jamb_reg_no'])308 res = self.students_catalog(jamb_reg_no = jamb_reg_no)309 if res:310 em = 'Student with RegNo %s already exists\n' % jamb_reg_no311 logger.info(em)312 result['Error'] = "Student exists"313 no_import.append(format % result)314 continue315 cert_id = makeCertificateCode(result.get(csv_d['study_course']))316 if cert_id not in certs.keys():317 res = self.portal_catalog(meta_type = "Certificate",id = cert_id)318 if not res:319 em = 'No Certificate with ID %s \n' % cert_id320 logger.info(em)321 result['Error'] = "No Certificate %s" % cert_id322 no_import.append( format % result)323 continue324 cert = res[0]325 cert_path = cert.getPath().split('/')326 certificate = certs[cert_id] = {'faculty': cert_path[-4],327 'department': cert_path[-3]}328 cert_doc = certs[cert_id]329 catalog_entry = {}330 catalog_entry['jamb_reg_no'] = jamb_reg_no331 firstname = result.get(csv_d['jamb_firstname'])332 middlename = result.get(csv_d['jamb_middlename'])333 lastname = result.get(csv_d['jamb_lastname'])334 if len(firstname) < 3\335 and len(middlename) < 3\336 and len(lastname) < 3:337 em = 'Student Names to short \n'338 logger.info(em)339 result['Error'] = "Names to short"340 no_import.append( format % result)341 continue342 perm_address = result.get(csv_d['perm_address'])343 sid = self.generateStudentId('x')344 students_folder.invokeFactory('Student', sid)345 catalog_entry['id'] = sid346 tr_count += 1347 logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())348 student = getattr(self,sid)349 student.manage_setLocalRoles(sid, ['Owner',])350 student.invokeFactory('StudentApplication','application')351 da = {'Title': 'Application Data'}352 da["jamb_firstname"] = firstname353 da["jamb_middlename"] = middlename354 da["jamb_lastname"] = lastname355 catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0]356 catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')357 da_fields = ('jamb_reg_no',358 'jamb_sex',359 'jamb_state',360 'entry_mode',361 'app_email',362 )363 for f in da_fields:364 da[f] = result.get(csv_d[f])365 catalog_entry['email'] = da['app_email']366 catalog_entry['entry_mode'] = da['entry_mode']367 app = student.application368 app_doc = app.getContent()369 app.getContent().edit(mapping=da)370 picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)371 app.manage_setLocalRoles(sid, ['Owner',])372 373 picture_id = da['jamb_reg_no'].replace('/','_')374 file = None375 for ext in ('jpg','JPG'):376 picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)377 if os.path.exists(picture):378 file = open(picture)379 break380 if file is not None:381 382 ## file conversion does not work383 #img = PIL.Image.open(file)384 #img.thumbnail((150,200),385 # resample=PIL.Image.ANTIALIAS)386 #outfile = StringIO()387 #img.save(outfile, format=img.format)388 389 outfile = file.read()390 app_doc.manage_addFile('passport',391 file=outfile,392 title="%s.jpg" % jamb_reg_no)393 #wftool.doActionFor(app,'close')394 dp = {}395 dp['firstname'] = firstname396 dp['middlename'] = middlename397 dp['lastname'] = lastname398 dp['email'] = da['app_email']399 dp['sex'] = sex400 dp['perm_address'] = perm_address401 catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp402 student.invokeFactory('StudentPersonal','personal')403 per = student.personal404 per_doc = per.getContent()405 per_doc.edit(mapping = dp)406 per.manage_setLocalRoles(sid, ['Owner',])407 wftool.doActionFor(student,'pume_pass')408 wftool.doActionFor(student,'admit')409 #410 # Clearance411 #412 student.invokeFactory('StudentClearance','clearance')413 #wftool.doActionFor(student.clearance,'open')414 clearance = student.clearance415 dc = {'Title': 'Clearance/Eligibility Record'}416 clearance = student.clearance417 date_str = result.get(csv_d['birthday'])418 try:419 date = DateTime.DateTime(date_str)420 except:421 #import pdb;pdb.set_trace()422 date = None423 dc['birthday'] = date424 clearance.getContent().edit(mapping=dc)425 clearance.manage_setLocalRoles(sid, ['Owner',])426 #427 # Study Course428 #429 student.invokeFactory('StudentStudyCourse','study_course')430 study_course = student.study_course431 dsc = {}432 catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100')433 #catalog_entry['level'] = dsc['current_level'] = '100' # Attention: not for DE students434 catalog_entry['session'] = dsc['current_session'] = da['entry_session']435 catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']436 catalog_entry['course'] = dsc['study_course'] = cert_id437 catalog_entry['faculty'] = certificate['faculty']438 catalog_entry['department'] = certificate['department']439 catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'440 catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)441 study_course.getContent().edit(mapping=dsc)442 #import pdb;pdb.set_trace()443 self.students_catalog.addRecord(**catalog_entry)444 if tr_count > 100:445 if len(no_import) > 0:446 open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(447 '\n'.join(no_import) + "\n")448 no_import = []449 em = '%d transactions commited\n' % tr_count450 transaction.commit()451 logger.info(em)452 total += tr_count453 tr_count = 0454 open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(455 '\n'.join(no_import))456 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))457 ###)458 118 459 119 security.declareProtected(ModifyPortalContent,"transferStudents")###( … … 619 279 ###) 620 280 621 security.declareProtected(ModifyPortalContent,"importReturningStudents")###(622 def importReturningStudents(self):623 """load Returning Studentdata from CSV values"""624 import transaction625 import random626 #from pdb import set_trace627 wftool = self.portal_workflow628 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")629 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()630 students_folder = self.portal_url.getPortalObject().campus.students631 tr_count = 1632 total = 0633 #name = 'pume_results'634 name = 'Returning'635 table = self.returning_import636 no_import = []637 imported = []638 logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')639 try:640 returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))641 except:642 logger.error('Error reading %s.csv' % name)643 return644 l = self.portal_catalog({'meta_type': "Certificate"})645 certs = {}646 cert_docs = {}647 for f in l:648 certs[f.getId] = f.getObject().getContent()649 start = True650 res = table()651 regs = []652 if len(res) > 0:653 regs = [s.matric_no for s in res]654 #import pdb;pdb.set_trace()655 for student in returning:656 if start:657 start = False658 logger.info('Start loading from %s.csv' % name)659 s = ','.join(['"%s"' % fn for fn in student.keys()])660 imported.append(s)661 no_import.append('%s,"Error"' % s)662 format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])663 format_error = format + ',"%(Error)s"'664 no_certificate = "no certificate %s" % format665 student['matric_no'] = matric_no = student.get('matric_no').upper()666 student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()667 student['Permanent_Address'] = perm_address = student.get('Permanent Address')668 if matric_no == '':669 student['Error'] = "Empty matric_no"670 no_import.append( format_error % student)671 continue672 if matric_no in regs or self.returning_import(matric_no = matric_no):673 student['Error'] = "Duplicate"674 no_import.append( format_error % student)675 continue676 cert_id = makeCertificateCode(student.get('Coursemajorcode'))677 if cert_id not in certs.keys():678 student['Error'] = "No Certificate %s" % cert_id679 no_import.append( format_error % student)680 continue681 try:682 table.addRecord(**student)683 except ValueError:684 student['Error'] = "Duplicate"685 no_import.append( format_error % student)686 continue687 regs.append(student.get('matric_no'))688 imported.append(format % student)689 tr_count += 1690 if tr_count > 1000:691 if len(no_import) > 0:692 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(693 '\n'.join(no_import) + '\n')694 no_import = []695 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(696 '\n'.join(no_import) + "\n")697 imported = []698 em = '%d transactions commited total %s\n' % (tr_count,total)699 transaction.commit()700 regs = []701 logger.info(em)702 total += tr_count703 tr_count = 0704 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(705 '\n'.join(imported))706 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(707 '\n'.join(no_import))708 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))709 ###)710 711 security.declareProtected(ModifyPortalContent,"fixAllNames")###(712 def fixAllNames(self):713 "fix all students names"714 import transaction715 response = self.REQUEST.RESPONSE716 logger = logging.getLogger('fixAllNames')717 logger.info('Start')718 students = self.portal_catalog(portal_type='Student')719 count = 0720 total = 0721 for student in students:722 scat_res = self.students_catalog(id = student.getId)723 if not scat_res:724 self.students_catalog.addRecord(id = student.getId)725 scat_res = self.students_catalog(id = student.getId)726 student_entry = scat_res[0]727 old_new = self.fixName(student,student_entry)728 count += 1729 response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))730 if count > 2000:731 transaction.commit()732 logger.info("%d transactions commited" % count)733 total += count734 count = 0735 ###)736 737 security.declareProtected(ModifyPortalContent,"fixName")###(738 def fixName(self,student_brain, student_entry):739 "fix the name of a student"740 fix = "first"741 if student_entry.get('name_fixed',None) == fix:742 return "Name already fixed"743 student_id = student_entry.id744 new_student = student_entry.jamb_reg_no.startswith('6')745 student_obj = student_brain.getObject()746 personal = getattr(student_obj,'personal',None)747 invalid = ''748 if personal is None:749 return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")750 per_doc = personal.getContent()751 old_first = per_doc.firstname752 old_middle = per_doc.middlename753 old_last = per_doc.lastname754 new_first = ''755 new_middle = ''756 new_last = ''757 if new_student:758 if not old_first and not old_middle and old_last:759 new_names = [n.capitalize() for n in old_last.split()]760 if len(new_names) > 1:761 old_first = new_names[0]762 old_last = new_names[-1]763 old_middle = ' '.join(new_names[1:-1])764 else:765 old_last = new_names[0]766 old_first = ''767 old_middle = ''768 if old_first:769 new_first = old_first770 if old_middle:771 new_middle = old_middle772 if old_last:773 new_last = old_last774 if old_first.find('<') != -1 or\775 old_first.find('>') != -1 or\776 old_middle.find('<') != -1 or\777 old_middle.find('>') != -1 or\778 old_last.find('<') != -1 or\779 old_last.find('>') != -1:780 invalid = "invalid characters"781 else:782 new_first = old_first783 if new_first.strip() == '-':784 new_first = ''785 new_middle = old_middle786 if new_middle.strip() == '-':787 new_middle = ''788 new_last = old_last789 if new_last.strip() == '-':790 new_last = ''791 name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()792 if new_student:793 text = "New"794 else:795 text = "Returning"796 old_new = '"%s","%s","%s","%s"' % (invalid,text,797 student_entry.name,798 name)799 if not invalid:800 self.students_catalog.modifyRecord(id = student_id,801 name_fixed = fix,802 name = name)803 per_doc.edit(mapping = {'firstname' : new_first,804 'middlename' : new_middle,805 'lastname' : new_last,806 })807 return old_new808 ###)809 810 security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(811 def updateReturningStudents(self):812 """load and overwrite Returning Student Data from CSV values"""813 import transaction814 import random815 #from pdb import set_trace816 wftool = self.portal_workflow817 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")818 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()819 students_folder = self.portal_url.getPortalObject().campus.students820 tr_count = 1821 total = 0822 #name = 'pume_results'823 name = 'Returning_update'824 table = self.returning_import825 no_import = []826 imported = []827 logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')828 try:829 returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))830 except:831 logger.error('Error reading %s.csv' % name)832 return833 l = self.portal_catalog({'meta_type': "Certificate"})834 certs = {}835 cert_docs = {}836 for f in l:837 certs[f.getId] = f.getObject().getContent()838 start = True839 res = table()840 regs = []841 if len(res) > 0:842 regs = [s.matric_no for s in res]843 for student in returning:844 if start:845 start = False846 logger.info('Start loading from %s.csv' % name)847 s = ','.join(['"%s"' % fn for fn in student.keys()])848 imported.append(s)849 no_import.append('%s,"Error"' % s)850 format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])851 format_error = format + ',"%(Error)s"'852 no_certificate = "no certificate %s" % format853 matric_no = student.get('matric_no').upper()854 student['matric_no'] = matric_no855 if matric_no == '':856 student['Error'] = "Empty matric_no"857 no_import.append( format_error % student)858 continue859 # if matric_no in regs or self.returning_import(matric_no = matric_no):860 # student['Error'] = "Duplicate"861 # no_import.append( format_error % student)862 # continue863 # cert_id = makeCertificateCode(student.get('Coursemajorcode'))864 # if cert_id not in certs.keys():865 # student['Error'] = "No Certificate %s" % cert_id866 # no_import.append( format_error % student)867 # continue868 try:869 table.modifyRecord(**student)870 except KeyError:871 #import pdb;pdb.set_trace()872 student['Error'] = "no Student found to update"873 no_import.append( format_error % student)874 continue875 #s = self.students_catalog(matric_no=matric_no)876 #if s:877 # level = "%s" % (int(student.get('Level')) + 100)878 # self.students_catalog.modifyRecord(id = s[0].id,879 # level=level)880 881 regs.append(student.get('matric_no'))882 imported.append(format % student)883 tr_count += 1884 if tr_count > 1000:885 if len(no_import) > 0:886 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(887 '\n'.join(no_import) + '\n')888 no_import = []889 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(890 '\n'.join(no_import) + "\n")891 imported = []892 em = '%d transactions commited total %s\n' % (tr_count,total)893 transaction.commit()894 regs = []895 logger.info(em)896 total += tr_count897 tr_count = 0898 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(899 '\n'.join(imported))900 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(901 '\n'.join(no_import))902 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))903 ###)904 905 security.declareProtected(ModifyPortalContent,"exportStudents")###(906 def exportStudents(self):907 """export Studentsdata to a file"""908 member = self.portal_membership.getAuthenticatedMember()909 logger = logging.getLogger('Students.exportStudents')910 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")911 students_folder = self.portal_url.getPortalObject().campus.students912 export_file = "%s/export/students_%s.csv" % (i_home,current)913 914 from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp915 aq_students = self.students_catalog.evalAdvancedQuery916 toexport = {'students_catalog': ("id",917 "matric_no",918 "jamb_reg_no",919 "name",920 "review_state",921 "entry_session",922 "session",923 "entry_mode",924 "mode",925 "verdict",926 "sex",927 "email",928 "phone",929 "faculty",930 "department",931 "course",932 "level",933 ),934 'personal': ('firstname',935 'middlename',936 'lastname',937 'perm_address',938 ),939 'clearance': ('state', # is not an attribute of clearance but is needed for splitting lga940 'lga',941 )942 }943 res_list = []944 lines = []945 fields = []946 with_lga = False947 for k in toexport.keys():948 for f in toexport[k]:949 if f == 'lga' :950 with_lga = True951 fields.append(f)952 headline = ','.join(fields).replace('level','current_level')953 open(export_file,"a").write(headline +'\n')954 format = '"%(' + ')s","%('.join(fields) + ')s"'955 query = In('review_state',('cleared_and_validated',956 'clearance_requested',957 'school_fee_paid',958 'courses_registered',959 'courses_validated'))960 #query = In('review_state',('clearance_requested'))961 students = aq_students(query)962 nr2export = len(students)963 logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))964 chunk = 1000965 total = 0966 start = DateTime.DateTime().timeTime()967 start_chunk = DateTime.DateTime().timeTime()968 ## alternative method slightly slower969 # student_recs = {}970 # for s in students:971 # student_recs[s.id] = s972 # catalog_recs = {}973 # brains = self.portal_catalog(portal_type = 'Student')974 # for cat_rec in brains:975 # sid = cat_rec.getId976 # catalog_recs[sid] = cat_rec977 # #import pdb;pdb.set_trace()978 # start = DateTime.DateTime().timeTime()979 # start_chunk = DateTime.DateTime().timeTime()980 # for student in students:981 # if student.id not in student_recs.keys():982 # continue983 # not_all = False984 # d = self.getFormattedStudentEntry(student_recs[student.id])985 # student_obj = catalog_recs[student.id].getObject()986 for student in students:987 not_all = False988 d = self.getFormattedStudentEntry(student)989 student_obj = getattr(students_folder,student.id)990 for k in toexport.keys()[1:]:991 try:992 object = getattr(student_obj,k)993 object_doc = object.getContent()994 except:995 logger.info('%s %s record not found' % (student.id,k))996 not_all = True997 continue998 for f in toexport[k]:999 d[f] = getattr(object_doc,f,'')1000 if not_all:1001 continue1002 if with_lga:1003 d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas)1004 lines.append(format % d)1005 total += 11006 if total and not total % chunk or total == len(students):1007 open(export_file,"a").write('\n'.join(lines) +'\n')1008 anz = len(lines)1009 logger.info("wrote %(anz)d total written %(total)d" % vars())1010 end_chunk = DateTime.DateTime().timeTime()1011 duration = end_chunk-start_chunk1012 per_record = duration/anz1013 till_now = end_chunk - start1014 avarage_per_record = till_now/total1015 estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)1016 estimated_end = estimated_end.strftime("%H:%M:%S")1017 logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())1018 start_chunk = DateTime.DateTime().timeTime()1019 lines = []1020 end = DateTime.DateTime().timeTime()1021 logger.info('total time %6.2f m' % ((end-start)/60))1022 filename, extension = os.path.splitext(export_file)1023 from subprocess import call1024 msg = "wrote %(total)d records to %(export_file)s" % vars()1025 try:1026 retcode = call('gzip %s' % (export_file),shell=True)1027 if retcode == 0:1028 msg = "wrote %(total)d records to %(export_file)s.gz" % vars()1029 except OSError, e:1030 retcode = -991031 logger.info("zip failed with %s" % e)1032 logger.info(msg)1033 args = {'portal_status_message': msg}1034 #url = self.REQUEST.get('URL1') + '?' + urlencode(args)1035 url = self.REQUEST.get('URL2')1036 return self.REQUEST.RESPONSE.redirect(url)1037 ###)1038 281 1039 282 security.declareProtected(ModifyPortalContent,"dumpStudentsCatalog")###( … … 1216 459 ###) 1217 460 1218 security.declareProtected(ModifyPortalContent,"updateStudyCourse")###( 1219 def updateStudyCourse(self): 1220 """update StudyCourse from CSV values""" 1221 import transaction 1222 import random 1223 from pdb import set_trace 1224 wftool = self.portal_workflow 1225 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() 1226 students_folder = self.portal_url.getPortalObject().campus.students 1227 csv_d = {'jamb_reg_no': "RegNumber", 1228 'jamb_lastname': "Name", 1229 'session': "Session", 1230 'pume_tot_score': "PUME SCORE", 1231 'jamb_score': "JambScore", 1232 'jamb_sex': "Sex", 1233 'jamb_state': "State", 1234 ## 'jamb_first_cos': "AdminCourse", 1235 'faculty': "AdminFaculty", 1236 'course_code': "AdmitCoscode", 1237 'stud_status':"AdmitStatus", 1238 'department': "AdmitDept", 1239 'jamb_lga': "LGA", 1240 'app_email': "email", 1241 'app_mobile': "PhoneNumbers", 1242 } 1243 csv_fields = [f[1] for f in csv_d.items()] 1244 tr_count = 0 1245 total = 0 1246 #name = 'pume_results' 1247 name = 'StudyCourseChange' 1248 no_import = [] 1249 s = ','.join(['"%s"' % fn for fn in csv_fields]) 1250 no_import.append('"Error",%s' % s) 1251 format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields]) 1252 no_certificate = "no certificate %s" % format 1253 open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write( 1254 '\n'.join(no_import)) 1255 logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse') 1256 logger.info('Start loading from %s.csv' % name) 1257 l = self.portal_catalog({'meta_type': "Certificate"}) 1258 try: 1259 result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 1260 except: 1261 logger.error('Error reading %s.csv' % name) 1262 return 1263 for jamb in result: 1264 jamb['Error'] = "Processing " 1265 logger.info(format % jamb) 1266 jamb_reg_no = jamb.get(csv_d['jamb_reg_no']) 1267 res = self.portal_catalog({'portal_type': "StudentApplication", 1268 'SearchableText': jamb_reg_no }) 1269 if not res: 1270 em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no 1271 logger.info(em) 1272 jamb['Error'] = "Student does not exist" 1273 no_import.append(format % jamb) 1274 continue 1275 sid = res[0].getPath().split('/')[-2] 1276 cert_id = makeCertificateCode(jamb.get(csv_d['course_code'])) 1277 res = self.portal_catalog(portal_type = "Certificate", id = cert_id) 1278 if not res: 1279 em = 'No Certificate with ID %s \n' % cert_id 1280 logger.info(em) 1281 jamb['Error'] = "No Certificate %s" % cert_id 1282 no_import.append( format % jamb) 1283 continue 1284 cert_brain = res[0] 1285 catalog_entry = {} 1286 student = getattr(self,sid) 1287 # 1288 # Study Course 1289 # 1290 study_course = student.study_course 1291 dsc = {} 1292 cert_pl = cert_brain.getPath().split('/') 1293 catalog_entry['id'] = sid 1294 catalog_entry['faculty'] = cert_pl[-4] 1295 catalog_entry['department'] = cert_pl[-3] 1296 catalog_entry['course'] = cert_id 1297 dsc['study_course'] = cert_id 1298 study_course.getContent().edit(mapping=dsc) 1299 self.students_catalog.modifyRecord(**catalog_entry) 1300 if tr_count > 10: 1301 if len(no_import) > 1: 1302 open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write( 1303 '\n'.join(no_import)) 1304 no_import = [] 1305 em = '%d transactions commited\n' % tr_count 1306 transaction.commit() 1307 logger.info(em) 1308 total += tr_count 1309 tr_count = 0 1310 tr_count += 1 1311 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 1312 ###) 461 1313 462 1314 463 security.declareProtected(View,"fixOwnership") ###( -
WAeUP_SRP/base/Students.unused_methods.txt
r1783 r2654 1452 1452 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 1453 1453 ###) 1454 1455 security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###( 1456 def loadPumeResultsFromCSV(self): 1457 """load Fulltime Studentdata from CSV values into pumeresults catalog""" 1458 import transaction 1459 import random 1460 ## csv_d = {'jamb_reg_no': "RegNumber", ###( 1461 ## 'status': "Admission Status", 1462 ## 'name': "Name", 1463 ## 'score': "Score", 1464 ## 'sex': "Sex", 1465 ## 'faculty': "Faculty", 1466 ## 'department': "Dept", 1467 ## 'course': "Course", 1468 ## 'course_code_org': "Course Code", 1469 ## } 1470 ###) 1471 ## csv_d = {'jamb_reg_no': "JAMBRegno", 1472 ## 'name': "Name", 1473 ## 'score': "Score", 1474 ## 'sex': "Sex", 1475 ## 'course': "Course", 1476 ## 'faculty': "Faculty", 1477 ## 'department': "Dept", 1478 ## 'course_code_org': "Course Code", 1479 ## 'status': "Admission Status", 1480 ## 'result_type': None, 1481 ## } 1482 1483 csv_d = {'jamb_reg_no': "reg_no", 1484 'name': "fullname", 1485 'score': "pume_score", 1486 'sex': "sex", 1487 'course': "study_course", 1488 'course_code_org': "study_course", 1489 'status': "admission_status", 1490 'result_type': "entry_mode", 1491 } 1492 1493 csv_fields = [f[1] for f in csv_d.items() if f[1]] 1494 tr_count = 0 1495 total = 0 1496 #name = 'pup_new' 1497 #name = 'pup_update' 1498 name = 'Admitted_update' 1499 update = name.endswith('update') 1500 no_import = [] 1501 ok_import = [] 1502 ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()])) 1503 no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields])) 1504 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") 1505 ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current) 1506 #open(ok_import_name,"w").write('\n'.join(no_import)) 1507 no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current) 1508 #open(no_import_name,"w").write('\n'.join(no_import)) 1509 logger = logging.getLogger('Students.loadPumeResultsFromCSV') 1510 starttime = DateTime.now() 1511 logger.info('Start loading from %s.csv' % name) 1512 try: 1513 result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 1514 except: 1515 logger.error('Error reading %s.csv' % name) 1516 return 1517 pume = self.portal_pumeresults 1518 format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields]) 1519 import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()]) 1520 eduplicate = '%s,"duplicate"' % format 1521 eoriginal = '%s,"original"' % format 1522 einvalidjamb = '%s,"invalid JambRegNo"' % format 1523 added = 'added ,%s' % format 1524 #from pdb import set_trace;set_trace() 1525 for jamb in result: 1526 if not jamb.get(csv_d['score']): 1527 logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no'])) 1528 continue 1529 dict = {} 1530 for f,fn in csv_d.items(): 1531 dict[f] = jamb.get(csv_d[f]) 1532 dict['result_type'] = 'CEST' 1533 jnr = jamb.get(csv_d['jamb_reg_no']) 1534 #if not checkJambNo(jnr): 1535 # logger.info(einvalidjamb % jamb) 1536 # dd = {} 1537 # for f,fn in csv_d.items(): 1538 # dd[fn] = getattr(data,f) 1539 # no_import.append(eduplicate % dd) 1540 # no_import.append(eduplicate % jamb) 1541 # continue 1542 res = pume(jamb_reg_no=jnr) 1543 if len(res) > 0: 1544 if update: 1545 try: 1546 pume.modifyRecord(**dict) 1547 # Can not happen, but anyway... 1548 except ValueError: 1549 logger.info(eduplicate % jamb) 1550 continue 1551 # Can not happen, but anyway... 1552 except KeyError: 1553 pume.addRecord(**dict) 1554 logger.info(added % jamb) 1555 continue 1556 else: 1557 data = res[0] 1558 if data.name != jamb.get(csv_d['name']): 1559 #set_trace() 1560 logger.info(eduplicate % jamb) 1561 #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict 1562 #logger.info(em) 1563 dd = {} 1564 for f,fn in csv_d.items(): 1565 dd[fn] = getattr(data,f) 1566 no_import.append(eoriginal % dd) 1567 no_import.append(eduplicate % jamb) 1568 continue 1569 else: 1570 try: 1571 pume.addRecord(**dict) 1572 ok_import.append(import_format % dict) 1573 except ValueError: 1574 logger.info(eduplicate % jamb) 1575 #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict 1576 #logger.info(em) 1577 no_import.append(eduplicate % jamb) 1578 logger.info('End loading from %s.csv' % name) 1579 if len(no_import) > 1: 1580 open(no_import_name,"w+").write('\n'.join(no_import)) 1581 open(ok_import_name,"w+").write('\n'.join(ok_import)) 1582 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 1583 ###) 1584 1585 security.declareProtected(ModifyPortalContent,"createStudents")###( 1586 def createStudents(self): 1587 """ 1588 load addmitted Studentdata from CSV values and create Studentobjects. 1589 This is the current method to create new addmitted Students. 1590 Before running the eventservice for the students_catalog must be disabled. 1591 """ 1592 import transaction 1593 import random 1594 #from pdb import set_trace 1595 wftool = self.portal_workflow 1596 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() 1597 students_folder = self.portal_url.getPortalObject().campus.students 1598 levels = {'ume_ft':'100', 1599 'de_ft': '200', 1600 'ug_pt': '100', 1601 'de_pt': '200', 1602 'pg_ft': '700', 1603 'pg_pt': '700', 1604 'dp_pt': '100', 1605 'dp_ft': '100', 1606 } 1607 csv_d = {'jamb_reg_no': "reg_no", 1608 'entry_mode': 'entry_mode', 1609 'jamb_firstname': "firstname", 1610 'jamb_middlename': "middlename", 1611 'jamb_lastname': "lastname", 1612 'jamb_sex': "sex", 1613 'jamb_state': "state", 1614 'birthday': "date_of_birth", 1615 'app_email': "email", 1616 'study_course': "study_course", 1617 'perm_address': "address", 1618 'admission_status': "admission_status", 1619 } 1620 csv_fields = [f[1] for f in csv_d.items()] 1621 tr_count = 0 1622 total = 0 1623 #name = 'pume_results' 1624 name = 'Admitted_update' 1625 no_import = [] 1626 s = ','.join(['"%s"' % fn for fn in csv_fields]) 1627 no_import.append('"Error",%s' % s) 1628 format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields]) 1629 no_certificate = "no certificate %s" % format 1630 open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import)) 1631 logger = logging.getLogger('Students.StudentsFolder.createStudents') 1632 logger.info('Start loading from %s.csv' % name) 1633 certs = {} 1634 try: 1635 results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 1636 except: 1637 logger.error('Error reading %s.csv' % name) 1638 return 1639 for result in results: 1640 if not result.get(csv_d['admission_status']).startswith('Admitted'): 1641 continue 1642 #result['Error'] = "Processing " 1643 #logger.info(format % result) 1644 jamb_reg_no = result.get(csv_d['jamb_reg_no']) 1645 res = self.students_catalog(jamb_reg_no = jamb_reg_no) 1646 if res: 1647 em = 'Student with RegNo %s already exists\n' % jamb_reg_no 1648 logger.info(em) 1649 result['Error'] = "Student exists" 1650 no_import.append(format % result) 1651 continue 1652 cert_id = makeCertificateCode(result.get(csv_d['study_course'])) 1653 if cert_id not in certs.keys(): 1654 res = self.portal_catalog(meta_type = "Certificate",id = cert_id) 1655 if not res: 1656 em = 'No Certificate with ID %s \n' % cert_id 1657 logger.info(em) 1658 result['Error'] = "No Certificate %s" % cert_id 1659 no_import.append( format % result) 1660 continue 1661 cert = res[0] 1662 cert_path = cert.getPath().split('/') 1663 certificate = certs[cert_id] = {'faculty': cert_path[-4], 1664 'department': cert_path[-3]} 1665 cert_doc = certs[cert_id] 1666 catalog_entry = {} 1667 catalog_entry['jamb_reg_no'] = jamb_reg_no 1668 firstname = result.get(csv_d['jamb_firstname']) 1669 middlename = result.get(csv_d['jamb_middlename']) 1670 lastname = result.get(csv_d['jamb_lastname']) 1671 if len(firstname) < 3\ 1672 and len(middlename) < 3\ 1673 and len(lastname) < 3: 1674 em = 'Student Names to short \n' 1675 logger.info(em) 1676 result['Error'] = "Names to short" 1677 no_import.append( format % result) 1678 continue 1679 perm_address = result.get(csv_d['perm_address']) 1680 sid = self.generateStudentId('x') 1681 students_folder.invokeFactory('Student', sid) 1682 catalog_entry['id'] = sid 1683 tr_count += 1 1684 logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars()) 1685 student = getattr(self,sid) 1686 student.manage_setLocalRoles(sid, ['Owner',]) 1687 student.invokeFactory('StudentApplication','application') 1688 da = {'Title': 'Application Data'} 1689 da["jamb_firstname"] = firstname 1690 da["jamb_middlename"] = middlename 1691 da["jamb_lastname"] = lastname 1692 catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0] 1693 catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F') 1694 da_fields = ('jamb_reg_no', 1695 'jamb_sex', 1696 'jamb_state', 1697 'entry_mode', 1698 'app_email', 1699 ) 1700 for f in da_fields: 1701 da[f] = result.get(csv_d[f]) 1702 catalog_entry['email'] = da['app_email'] 1703 catalog_entry['entry_mode'] = da['entry_mode'] 1704 app = student.application 1705 app_doc = app.getContent() 1706 app.getContent().edit(mapping=da) 1707 picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no) 1708 app.manage_setLocalRoles(sid, ['Owner',]) 1709 1710 picture_id = da['jamb_reg_no'].replace('/','_') 1711 file = None 1712 for ext in ('jpg','JPG'): 1713 picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext) 1714 if os.path.exists(picture): 1715 file = open(picture) 1716 break 1717 if file is not None: 1718 1719 ## file conversion does not work 1720 #img = PIL.Image.open(file) 1721 #img.thumbnail((150,200), 1722 # resample=PIL.Image.ANTIALIAS) 1723 #outfile = StringIO() 1724 #img.save(outfile, format=img.format) 1725 1726 outfile = file.read() 1727 app_doc.manage_addFile('passport', 1728 file=outfile, 1729 title="%s.jpg" % jamb_reg_no) 1730 #wftool.doActionFor(app,'close') 1731 dp = {} 1732 dp['firstname'] = firstname 1733 dp['middlename'] = middlename 1734 dp['lastname'] = lastname 1735 dp['email'] = da['app_email'] 1736 dp['sex'] = sex 1737 dp['perm_address'] = perm_address 1738 catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp 1739 student.invokeFactory('StudentPersonal','personal') 1740 per = student.personal 1741 per_doc = per.getContent() 1742 per_doc.edit(mapping = dp) 1743 per.manage_setLocalRoles(sid, ['Owner',]) 1744 wftool.doActionFor(student,'pume_pass') 1745 wftool.doActionFor(student,'admit') 1746 # 1747 # Clearance 1748 # 1749 student.invokeFactory('StudentClearance','clearance') 1750 #wftool.doActionFor(student.clearance,'open') 1751 clearance = student.clearance 1752 dc = {'Title': 'Clearance/Eligibility Record'} 1753 clearance = student.clearance 1754 date_str = result.get(csv_d['birthday']) 1755 try: 1756 date = DateTime.DateTime(date_str) 1757 except: 1758 #import pdb;pdb.set_trace() 1759 date = None 1760 dc['birthday'] = date 1761 clearance.getContent().edit(mapping=dc) 1762 clearance.manage_setLocalRoles(sid, ['Owner',]) 1763 # 1764 # Study Course 1765 # 1766 student.invokeFactory('StudentStudyCourse','study_course') 1767 study_course = student.study_course 1768 dsc = {} 1769 catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100') 1770 #catalog_entry['level'] = dsc['current_level'] = '100' # Attention: not for DE students 1771 catalog_entry['session'] = dsc['current_session'] = da['entry_session'] 1772 catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode'] 1773 catalog_entry['course'] = dsc['study_course'] = cert_id 1774 catalog_entry['faculty'] = certificate['faculty'] 1775 catalog_entry['department'] = certificate['department'] 1776 catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A' 1777 catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None) 1778 study_course.getContent().edit(mapping=dsc) 1779 #import pdb;pdb.set_trace() 1780 self.students_catalog.addRecord(**catalog_entry) 1781 if tr_count > 100: 1782 if len(no_import) > 0: 1783 open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write( 1784 '\n'.join(no_import) + "\n") 1785 no_import = [] 1786 em = '%d transactions commited\n' % tr_count 1787 transaction.commit() 1788 logger.info(em) 1789 total += tr_count 1790 tr_count = 0 1791 open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write( 1792 '\n'.join(no_import)) 1793 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 1794 ###) 1795 1796 1797 security.declareProtected(ModifyPortalContent,"importReturningStudents")###( 1798 def importReturningStudents(self): 1799 """load Returning Studentdata from CSV values""" 1800 import transaction 1801 import random 1802 #from pdb import set_trace 1803 wftool = self.portal_workflow 1804 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") 1805 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() 1806 students_folder = self.portal_url.getPortalObject().campus.students 1807 tr_count = 1 1808 total = 0 1809 #name = 'pume_results' 1810 name = 'Returning' 1811 table = self.returning_import 1812 no_import = [] 1813 imported = [] 1814 logger = logging.getLogger('Students.StudentsFolder.importReturningStudents') 1815 try: 1816 returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 1817 except: 1818 logger.error('Error reading %s.csv' % name) 1819 return 1820 l = self.portal_catalog({'meta_type': "Certificate"}) 1821 certs = {} 1822 cert_docs = {} 1823 for f in l: 1824 certs[f.getId] = f.getObject().getContent() 1825 start = True 1826 res = table() 1827 regs = [] 1828 if len(res) > 0: 1829 regs = [s.matric_no for s in res] 1830 #import pdb;pdb.set_trace() 1831 for student in returning: 1832 if start: 1833 start = False 1834 logger.info('Start loading from %s.csv' % name) 1835 s = ','.join(['"%s"' % fn for fn in student.keys()]) 1836 imported.append(s) 1837 no_import.append('%s,"Error"' % s) 1838 format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()]) 1839 format_error = format + ',"%(Error)s"' 1840 no_certificate = "no certificate %s" % format 1841 student['matric_no'] = matric_no = student.get('matric_no').upper() 1842 student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper() 1843 student['Permanent_Address'] = perm_address = student.get('Permanent Address') 1844 if matric_no == '': 1845 student['Error'] = "Empty matric_no" 1846 no_import.append( format_error % student) 1847 continue 1848 if matric_no in regs or self.returning_import(matric_no = matric_no): 1849 student['Error'] = "Duplicate" 1850 no_import.append( format_error % student) 1851 continue 1852 cert_id = makeCertificateCode(student.get('Coursemajorcode')) 1853 if cert_id not in certs.keys(): 1854 student['Error'] = "No Certificate %s" % cert_id 1855 no_import.append( format_error % student) 1856 continue 1857 try: 1858 table.addRecord(**student) 1859 except ValueError: 1860 student['Error'] = "Duplicate" 1861 no_import.append( format_error % student) 1862 continue 1863 regs.append(student.get('matric_no')) 1864 imported.append(format % student) 1865 tr_count += 1 1866 if tr_count > 1000: 1867 if len(no_import) > 0: 1868 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( 1869 '\n'.join(no_import) + '\n') 1870 no_import = [] 1871 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( 1872 '\n'.join(no_import) + "\n") 1873 imported = [] 1874 em = '%d transactions commited total %s\n' % (tr_count,total) 1875 transaction.commit() 1876 regs = [] 1877 logger.info(em) 1878 total += tr_count 1879 tr_count = 0 1880 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( 1881 '\n'.join(imported)) 1882 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( 1883 '\n'.join(no_import)) 1884 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 1885 ###) 1886 1887 security.declareProtected(ModifyPortalContent,"fixAllNames")###( 1888 def fixAllNames(self): 1889 "fix all students names" 1890 import transaction 1891 response = self.REQUEST.RESPONSE 1892 logger = logging.getLogger('fixAllNames') 1893 logger.info('Start') 1894 students = self.portal_catalog(portal_type='Student') 1895 count = 0 1896 total = 0 1897 for student in students: 1898 scat_res = self.students_catalog(id = student.getId) 1899 if not scat_res: 1900 self.students_catalog.addRecord(id = student.getId) 1901 scat_res = self.students_catalog(id = student.getId) 1902 student_entry = scat_res[0] 1903 old_new = self.fixName(student,student_entry) 1904 count += 1 1905 response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new)) 1906 if count > 2000: 1907 transaction.commit() 1908 logger.info("%d transactions commited" % count) 1909 total += count 1910 count = 0 1911 ###) 1912 1913 security.declareProtected(ModifyPortalContent,"fixName")###( 1914 def fixName(self,student_brain, student_entry): 1915 "fix the name of a student" 1916 fix = "first" 1917 if student_entry.get('name_fixed',None) == fix: 1918 return "Name already fixed" 1919 student_id = student_entry.id 1920 new_student = student_entry.jamb_reg_no.startswith('6') 1921 student_obj = student_brain.getObject() 1922 personal = getattr(student_obj,'personal',None) 1923 invalid = '' 1924 if personal is None: 1925 return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in") 1926 per_doc = personal.getContent() 1927 old_first = per_doc.firstname 1928 old_middle = per_doc.middlename 1929 old_last = per_doc.lastname 1930 new_first = '' 1931 new_middle = '' 1932 new_last = '' 1933 if new_student: 1934 if not old_first and not old_middle and old_last: 1935 new_names = [n.capitalize() for n in old_last.split()] 1936 if len(new_names) > 1: 1937 old_first = new_names[0] 1938 old_last = new_names[-1] 1939 old_middle = ' '.join(new_names[1:-1]) 1940 else: 1941 old_last = new_names[0] 1942 old_first = '' 1943 old_middle = '' 1944 if old_first: 1945 new_first = old_first 1946 if old_middle: 1947 new_middle = old_middle 1948 if old_last: 1949 new_last = old_last 1950 if old_first.find('<') != -1 or\ 1951 old_first.find('>') != -1 or\ 1952 old_middle.find('<') != -1 or\ 1953 old_middle.find('>') != -1 or\ 1954 old_last.find('<') != -1 or\ 1955 old_last.find('>') != -1: 1956 invalid = "invalid characters" 1957 else: 1958 new_first = old_first 1959 if new_first.strip() == '-': 1960 new_first = '' 1961 new_middle = old_middle 1962 if new_middle.strip() == '-': 1963 new_middle = '' 1964 new_last = old_last 1965 if new_last.strip() == '-': 1966 new_last = '' 1967 name = "%(new_first)s %(new_middle)s %(new_last)s" % vars() 1968 if new_student: 1969 text = "New" 1970 else: 1971 text = "Returning" 1972 old_new = '"%s","%s","%s","%s"' % (invalid,text, 1973 student_entry.name, 1974 name) 1975 if not invalid: 1976 self.students_catalog.modifyRecord(id = student_id, 1977 name_fixed = fix, 1978 name = name) 1979 per_doc.edit(mapping = {'firstname' : new_first, 1980 'middlename' : new_middle, 1981 'lastname' : new_last, 1982 }) 1983 return old_new 1984 ###) 1985 1986 security.declareProtected(ModifyPortalContent,"updateReturningStudents")###( 1987 def updateReturningStudents(self): 1988 """load and overwrite Returning Student Data from CSV values""" 1989 import transaction 1990 import random 1991 #from pdb import set_trace 1992 wftool = self.portal_workflow 1993 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") 1994 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() 1995 students_folder = self.portal_url.getPortalObject().campus.students 1996 tr_count = 1 1997 total = 0 1998 #name = 'pume_results' 1999 name = 'Returning_update' 2000 table = self.returning_import 2001 no_import = [] 2002 imported = [] 2003 logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents') 2004 try: 2005 returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 2006 except: 2007 logger.error('Error reading %s.csv' % name) 2008 return 2009 l = self.portal_catalog({'meta_type': "Certificate"}) 2010 certs = {} 2011 cert_docs = {} 2012 for f in l: 2013 certs[f.getId] = f.getObject().getContent() 2014 start = True 2015 res = table() 2016 regs = [] 2017 if len(res) > 0: 2018 regs = [s.matric_no for s in res] 2019 for student in returning: 2020 if start: 2021 start = False 2022 logger.info('Start loading from %s.csv' % name) 2023 s = ','.join(['"%s"' % fn for fn in student.keys()]) 2024 imported.append(s) 2025 no_import.append('%s,"Error"' % s) 2026 format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()]) 2027 format_error = format + ',"%(Error)s"' 2028 no_certificate = "no certificate %s" % format 2029 matric_no = student.get('matric_no').upper() 2030 student['matric_no'] = matric_no 2031 if matric_no == '': 2032 student['Error'] = "Empty matric_no" 2033 no_import.append( format_error % student) 2034 continue 2035 # if matric_no in regs or self.returning_import(matric_no = matric_no): 2036 # student['Error'] = "Duplicate" 2037 # no_import.append( format_error % student) 2038 # continue 2039 # cert_id = makeCertificateCode(student.get('Coursemajorcode')) 2040 # if cert_id not in certs.keys(): 2041 # student['Error'] = "No Certificate %s" % cert_id 2042 # no_import.append( format_error % student) 2043 # continue 2044 try: 2045 table.modifyRecord(**student) 2046 except KeyError: 2047 #import pdb;pdb.set_trace() 2048 student['Error'] = "no Student found to update" 2049 no_import.append( format_error % student) 2050 continue 2051 #s = self.students_catalog(matric_no=matric_no) 2052 #if s: 2053 # level = "%s" % (int(student.get('Level')) + 100) 2054 # self.students_catalog.modifyRecord(id = s[0].id, 2055 # level=level) 2056 2057 regs.append(student.get('matric_no')) 2058 imported.append(format % student) 2059 tr_count += 1 2060 if tr_count > 1000: 2061 if len(no_import) > 0: 2062 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( 2063 '\n'.join(no_import) + '\n') 2064 no_import = [] 2065 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( 2066 '\n'.join(no_import) + "\n") 2067 imported = [] 2068 em = '%d transactions commited total %s\n' % (tr_count,total) 2069 transaction.commit() 2070 regs = [] 2071 logger.info(em) 2072 total += tr_count 2073 tr_count = 0 2074 open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write( 2075 '\n'.join(imported)) 2076 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( 2077 '\n'.join(no_import)) 2078 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 2079 ###) 2080 2081 security.declareProtected(ModifyPortalContent,"exportStudents")###( 2082 def exportStudents(self): 2083 """export Studentsdata to a file""" 2084 member = self.portal_membership.getAuthenticatedMember() 2085 logger = logging.getLogger('Students.exportStudents') 2086 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") 2087 students_folder = self.portal_url.getPortalObject().campus.students 2088 export_file = "%s/export/students_%s.csv" % (i_home,current) 2089 2090 from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp 2091 aq_students = self.students_catalog.evalAdvancedQuery 2092 toexport = {'students_catalog': ("id", 2093 "matric_no", 2094 "jamb_reg_no", 2095 "name", 2096 "review_state", 2097 "entry_session", 2098 "session", 2099 "entry_mode", 2100 "mode", 2101 "verdict", 2102 "sex", 2103 "email", 2104 "phone", 2105 "faculty", 2106 "department", 2107 "course", 2108 "level", 2109 ), 2110 'personal': ('firstname', 2111 'middlename', 2112 'lastname', 2113 'perm_address', 2114 ), 2115 'clearance': ('state', # is not an attribute of clearance but is needed for splitting lga 2116 'lga', 2117 ) 2118 } 2119 res_list = [] 2120 lines = [] 2121 fields = [] 2122 with_lga = False 2123 for k in toexport.keys(): 2124 for f in toexport[k]: 2125 if f == 'lga' : 2126 with_lga = True 2127 fields.append(f) 2128 headline = ','.join(fields).replace('level','current_level') 2129 open(export_file,"a").write(headline +'\n') 2130 format = '"%(' + ')s","%('.join(fields) + ')s"' 2131 query = In('review_state',('cleared_and_validated', 2132 'clearance_requested', 2133 'school_fee_paid', 2134 'courses_registered', 2135 'courses_validated')) 2136 #query = In('review_state',('clearance_requested')) 2137 students = aq_students(query) 2138 nr2export = len(students) 2139 logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export)) 2140 chunk = 1000 2141 total = 0 2142 start = DateTime.DateTime().timeTime() 2143 start_chunk = DateTime.DateTime().timeTime() 2144 ## alternative method slightly slower 2145 # student_recs = {} 2146 # for s in students: 2147 # student_recs[s.id] = s 2148 # catalog_recs = {} 2149 # brains = self.portal_catalog(portal_type = 'Student') 2150 # for cat_rec in brains: 2151 # sid = cat_rec.getId 2152 # catalog_recs[sid] = cat_rec 2153 # #import pdb;pdb.set_trace() 2154 # start = DateTime.DateTime().timeTime() 2155 # start_chunk = DateTime.DateTime().timeTime() 2156 # for student in students: 2157 # if student.id not in student_recs.keys(): 2158 # continue 2159 # not_all = False 2160 # d = self.getFormattedStudentEntry(student_recs[student.id]) 2161 # student_obj = catalog_recs[student.id].getObject() 2162 for student in students: 2163 not_all = False 2164 d = self.getFormattedStudentEntry(student) 2165 student_obj = getattr(students_folder,student.id) 2166 for k in toexport.keys()[1:]: 2167 try: 2168 object = getattr(student_obj,k) 2169 object_doc = object.getContent() 2170 except: 2171 logger.info('%s %s record not found' % (student.id,k)) 2172 not_all = True 2173 continue 2174 for f in toexport[k]: 2175 d[f] = getattr(object_doc,f,'') 2176 if not_all: 2177 continue 2178 if with_lga: 2179 d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas) 2180 lines.append(format % d) 2181 total += 1 2182 if total and not total % chunk or total == len(students): 2183 open(export_file,"a").write('\n'.join(lines) +'\n') 2184 anz = len(lines) 2185 logger.info("wrote %(anz)d total written %(total)d" % vars()) 2186 end_chunk = DateTime.DateTime().timeTime() 2187 duration = end_chunk-start_chunk 2188 per_record = duration/anz 2189 till_now = end_chunk - start 2190 avarage_per_record = till_now/total 2191 estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export) 2192 estimated_end = estimated_end.strftime("%H:%M:%S") 2193 logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars()) 2194 start_chunk = DateTime.DateTime().timeTime() 2195 lines = [] 2196 end = DateTime.DateTime().timeTime() 2197 logger.info('total time %6.2f m' % ((end-start)/60)) 2198 filename, extension = os.path.splitext(export_file) 2199 from subprocess import call 2200 msg = "wrote %(total)d records to %(export_file)s" % vars() 2201 try: 2202 retcode = call('gzip %s' % (export_file),shell=True) 2203 if retcode == 0: 2204 msg = "wrote %(total)d records to %(export_file)s.gz" % vars() 2205 except OSError, e: 2206 retcode = -99 2207 logger.info("zip failed with %s" % e) 2208 logger.info(msg) 2209 args = {'portal_status_message': msg} 2210 #url = self.REQUEST.get('URL1') + '?' + urlencode(args) 2211 url = self.REQUEST.get('URL2') 2212 return self.REQUEST.RESPONSE.redirect(url) 2213 ###) 2214 2215 security.declareProtected(ModifyPortalContent,"updateStudyCourse")###( 2216 def updateStudyCourse(self): 2217 """update StudyCourse from CSV values""" 2218 import transaction 2219 import random 2220 from pdb import set_trace 2221 wftool = self.portal_workflow 2222 #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject() 2223 students_folder = self.portal_url.getPortalObject().campus.students 2224 csv_d = {'jamb_reg_no': "RegNumber", 2225 'jamb_lastname': "Name", 2226 'session': "Session", 2227 'pume_tot_score': "PUME SCORE", 2228 'jamb_score': "JambScore", 2229 'jamb_sex': "Sex", 2230 'jamb_state': "State", 2231 ## 'jamb_first_cos': "AdminCourse", 2232 'faculty': "AdminFaculty", 2233 'course_code': "AdmitCoscode", 2234 'stud_status':"AdmitStatus", 2235 'department': "AdmitDept", 2236 'jamb_lga': "LGA", 2237 'app_email': "email", 2238 'app_mobile': "PhoneNumbers", 2239 } 2240 csv_fields = [f[1] for f in csv_d.items()] 2241 tr_count = 0 2242 total = 0 2243 #name = 'pume_results' 2244 name = 'StudyCourseChange' 2245 no_import = [] 2246 s = ','.join(['"%s"' % fn for fn in csv_fields]) 2247 no_import.append('"Error",%s' % s) 2248 format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields]) 2249 no_certificate = "no certificate %s" % format 2250 open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write( 2251 '\n'.join(no_import)) 2252 logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse') 2253 logger.info('Start loading from %s.csv' % name) 2254 l = self.portal_catalog({'meta_type': "Certificate"}) 2255 try: 2256 result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) 2257 except: 2258 logger.error('Error reading %s.csv' % name) 2259 return 2260 for jamb in result: 2261 jamb['Error'] = "Processing " 2262 logger.info(format % jamb) 2263 jamb_reg_no = jamb.get(csv_d['jamb_reg_no']) 2264 res = self.portal_catalog({'portal_type': "StudentApplication", 2265 'SearchableText': jamb_reg_no }) 2266 if not res: 2267 em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no 2268 logger.info(em) 2269 jamb['Error'] = "Student does not exist" 2270 no_import.append(format % jamb) 2271 continue 2272 sid = res[0].getPath().split('/')[-2] 2273 cert_id = makeCertificateCode(jamb.get(csv_d['course_code'])) 2274 res = self.portal_catalog(portal_type = "Certificate", id = cert_id) 2275 if not res: 2276 em = 'No Certificate with ID %s \n' % cert_id 2277 logger.info(em) 2278 jamb['Error'] = "No Certificate %s" % cert_id 2279 no_import.append( format % jamb) 2280 continue 2281 cert_brain = res[0] 2282 catalog_entry = {} 2283 student = getattr(self,sid) 2284 # 2285 # Study Course 2286 # 2287 study_course = student.study_course 2288 dsc = {} 2289 cert_pl = cert_brain.getPath().split('/') 2290 catalog_entry['id'] = sid 2291 catalog_entry['faculty'] = cert_pl[-4] 2292 catalog_entry['department'] = cert_pl[-3] 2293 catalog_entry['course'] = cert_id 2294 dsc['study_course'] = cert_id 2295 study_course.getContent().edit(mapping=dsc) 2296 self.students_catalog.modifyRecord(**catalog_entry) 2297 if tr_count > 10: 2298 if len(no_import) > 1: 2299 open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write( 2300 '\n'.join(no_import)) 2301 no_import = [] 2302 em = '%d transactions commited\n' % tr_count 2303 transaction.commit() 2304 logger.info(em) 2305 total += tr_count 2306 tr_count = 0 2307 tr_count += 1 2308 return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) 2309 ###)
Note: See TracChangeset for help on using the changeset viewer.
