Index: /WAeUP_SRP/base/Students.py
===================================================================
--- /WAeUP_SRP/base/Students.py	(revision 2653)
+++ /WAeUP_SRP/base/Students.py	(revision 2654)
@@ -116,344 +116,4 @@
     security = ClassSecurityInfo()
 
-    security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
-    def loadPumeResultsFromCSV(self):
-        """load Fulltime Studentdata from CSV values into pumeresults catalog"""
-        import transaction
-        import random
-##        csv_d = {'jamb_reg_no': "RegNumber", ###(
-##                 'status': "Admission Status",
-##                 'name': "Name",
-##                 'score': "Score",
-##                 'sex': "Sex",
-##                 'faculty': "Faculty",
-##                 'department': "Dept",
-##                 'course': "Course",
-##                 'course_code_org': "Course Code",
-##                 }
-###)
-##        csv_d = {'jamb_reg_no': "JAMBRegno",
-##                'name': "Name",
-##                'score': "Score",
-##                 'sex': "Sex",
-##                 'course': "Course",
-##                 'faculty': "Faculty",
-##                 'department': "Dept",
-##                 'course_code_org': "Course Code",
-##                 'status': "Admission Status",
-##                 'result_type': None,
-##                 }
-
-        csv_d = {'jamb_reg_no': "reg_no",
-                 'name': "fullname",
-                 'score': "pume_score",
-                 'sex': "sex",
-                 'course': "study_course",
-                 'course_code_org': "study_course",
-                 'status': "admission_status",
-                 'result_type': "entry_mode",
-                 }
-
-        csv_fields = [f[1] for f in csv_d.items() if f[1]]
-        tr_count = 0
-        total = 0
-        #name = 'pup_new'
-        #name = 'pup_update'
-        name = 'Admitted_update'
-        update = name.endswith('update')
-        no_import = []
-        ok_import = []
-        ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
-        no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
-        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
-        ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
-        #open(ok_import_name,"w").write('\n'.join(no_import))
-        no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
-        #open(no_import_name,"w").write('\n'.join(no_import))
-        logger = logging.getLogger('Students.loadPumeResultsFromCSV')
-        starttime = DateTime.now()
-        logger.info('Start loading from %s.csv' % name)
-        try:
-            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
-        except:
-            logger.error('Error reading %s.csv' % name)
-            return
-        pume = self.portal_pumeresults
-        format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
-        import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
-        eduplicate = '%s,"duplicate"' % format
-        eoriginal = '%s,"original"' % format
-        einvalidjamb = '%s,"invalid JambRegNo"' % format
-        added = 'added ,%s' % format
-        #from pdb import set_trace;set_trace()
-        for jamb in result:
-            if not jamb.get(csv_d['score']):
-                logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no']))
-                continue
-            dict = {}
-            for f,fn in csv_d.items():
-                dict[f] = jamb.get(csv_d[f])
-            dict['result_type'] = 'CEST'
-            jnr = jamb.get(csv_d['jamb_reg_no'])
-            #if not checkJambNo(jnr):
-            #    logger.info(einvalidjamb % jamb)
-            #    dd = {}
-            #    for f,fn in csv_d.items():
-            #        dd[fn] = getattr(data,f)
-            #        no_import.append(eduplicate % dd)
-            #        no_import.append(eduplicate % jamb)
-            #    continue
-            res = pume(jamb_reg_no=jnr)
-            if len(res) > 0:
-                if update:
-                    try:
-                        pume.modifyRecord(**dict)
-                    # Can not happen, but anyway...
-                    except ValueError:
-                        logger.info(eduplicate % jamb)
-                        continue
-                    # Can not happen, but anyway...
-                    except KeyError:
-                        pume.addRecord(**dict)
-                        logger.info(added % jamb)
-                        continue
-                else:
-                    data = res[0]
-                    if data.name != jamb.get(csv_d['name']):
-                        #set_trace()
-                        logger.info(eduplicate % jamb)
-                        #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
-                        #logger.info(em)
-                        dd = {}
-                        for f,fn in csv_d.items():
-                            dd[fn] = getattr(data,f)
-                        no_import.append(eoriginal % dd)
-                        no_import.append(eduplicate % jamb)
-                    continue
-            else:
-                try:
-                    pume.addRecord(**dict)
-                    ok_import.append(import_format % dict)
-                except ValueError:
-                    logger.info(eduplicate % jamb)
-                    #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
-                    #logger.info(em)
-                    no_import.append(eduplicate % jamb)
-        logger.info('End loading from %s.csv' % name)
-        if len(no_import) > 1:
-            open(no_import_name,"w+").write('\n'.join(no_import))
-        open(ok_import_name,"w+").write('\n'.join(ok_import))
-        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
-    ###)
-
-    security.declareProtected(ModifyPortalContent,"createStudents")###(
-    def createStudents(self):
-        """
-        load addmitted Studentdata from CSV values and create Studentobjects.
-        This is the current method to create new addmitted Students.
-        Before running the eventservice for the students_catalog must be disabled.
-        """
-        import transaction
-        import random
-        #from pdb import set_trace
-        wftool = self.portal_workflow
-        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
-        students_folder = self.portal_url.getPortalObject().campus.students
-        levels =       {'ume_ft':'100',
-                        'de_ft': '200',
-                        'ug_pt': '100',
-                        'de_pt': '200',
-                        'pg_ft': '700',
-                        'pg_pt': '700',
-                        'dp_pt': '100',
-                        'dp_ft': '100',
-                        }
-        csv_d = {'jamb_reg_no': "reg_no",
-                 'entry_mode': 'entry_mode',
-                 'jamb_firstname': "firstname",
-                 'jamb_middlename': "middlename",
-                 'jamb_lastname': "lastname",
-                 'jamb_sex': "sex",
-                 'jamb_state': "state",
-                 'birthday': "date_of_birth",
-                 'app_email': "email",
-                 'study_course': "study_course",
-                 'perm_address': "address",
-                 'admission_status': "admission_status",
-                 }
-        csv_fields = [f[1] for f in csv_d.items()]
-        tr_count = 0
-        total = 0
-        #name = 'pume_results'
-        name = 'Admitted_update'
-        no_import = []
-        s = ','.join(['"%s"' % fn for fn in csv_fields])
-        no_import.append('"Error",%s' % s)
-        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
-        no_certificate = "no certificate %s" % format
-        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
-        logger = logging.getLogger('Students.StudentsFolder.createStudents')
-        logger.info('Start loading from %s.csv' % name)
-        certs = {}
-        try:
-            results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
-        except:
-            logger.error('Error reading %s.csv' % name)
-            return
-        for result in results:
-            if not result.get(csv_d['admission_status']).startswith('Admitted'):
-                continue
-            #result['Error'] = "Processing "
-            #logger.info(format % result)
-            jamb_reg_no = result.get(csv_d['jamb_reg_no'])
-            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
-            if res:
-                em = 'Student with RegNo %s already exists\n' % jamb_reg_no
-                logger.info(em)
-                result['Error'] = "Student exists"
-                no_import.append(format % result)
-                continue
-            cert_id = makeCertificateCode(result.get(csv_d['study_course']))
-            if cert_id not in certs.keys():
-                res = self.portal_catalog(meta_type = "Certificate",id = cert_id)
-                if not res:
-                    em = 'No Certificate with ID %s \n' % cert_id
-                    logger.info(em)
-                    result['Error'] = "No Certificate %s" % cert_id
-                    no_import.append( format % result)
-                    continue
-                cert = res[0]
-                cert_path = cert.getPath().split('/')
-                certificate = certs[cert_id] = {'faculty': cert_path[-4],
-                                     'department': cert_path[-3]}
-            cert_doc = certs[cert_id]
-            catalog_entry = {}
-            catalog_entry['jamb_reg_no'] = jamb_reg_no
-            firstname = result.get(csv_d['jamb_firstname'])
-            middlename = result.get(csv_d['jamb_middlename'])
-            lastname = result.get(csv_d['jamb_lastname'])
-            if len(firstname) < 3\
-               and len(middlename) < 3\
-               and len(lastname) < 3:
-                em = 'Student Names to short \n'
-                logger.info(em)
-                result['Error'] = "Names to short"
-                no_import.append( format % result)
-                continue
-            perm_address = result.get(csv_d['perm_address'])
-            sid = self.generateStudentId('x')
-            students_folder.invokeFactory('Student', sid)
-            catalog_entry['id'] = sid
-            tr_count += 1
-            logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
-            student = getattr(self,sid)
-            student.manage_setLocalRoles(sid, ['Owner',])
-            student.invokeFactory('StudentApplication','application')
-            da = {'Title': 'Application Data'}
-            da["jamb_firstname"] = firstname
-            da["jamb_middlename"] = middlename
-            da["jamb_lastname"] = lastname
-            catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0]
-            catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')
-            da_fields = ('jamb_reg_no',
-                         'jamb_sex',
-                         'jamb_state',
-                         'entry_mode',
-                         'app_email',
-                         )
-            for f in da_fields:
-                da[f] = result.get(csv_d[f])
-            catalog_entry['email'] = da['app_email']
-            catalog_entry['entry_mode'] = da['entry_mode']
-            app = student.application
-            app_doc = app.getContent()
-            app.getContent().edit(mapping=da)
-            picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
-            app.manage_setLocalRoles(sid, ['Owner',])
-
-            picture_id = da['jamb_reg_no'].replace('/','_')
-            file = None
-            for ext in ('jpg','JPG'):
-                picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)
-                if os.path.exists(picture):
-                    file = open(picture)
-                    break
-            if file is not None:
-
-                ## file conversion does not work
-                #img = PIL.Image.open(file)
-                #img.thumbnail((150,200),
-                #              resample=PIL.Image.ANTIALIAS)
-                #outfile = StringIO()
-                #img.save(outfile, format=img.format)
-
-                outfile = file.read()
-                app_doc.manage_addFile('passport',
-                                       file=outfile,
-                                       title="%s.jpg" % jamb_reg_no)
-            #wftool.doActionFor(app,'close')
-            dp = {}
-            dp['firstname'] = firstname
-            dp['middlename'] = middlename
-            dp['lastname'] = lastname
-            dp['email'] = da['app_email']
-            dp['sex'] = sex
-            dp['perm_address'] = perm_address
-            catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
-            student.invokeFactory('StudentPersonal','personal')
-            per = student.personal
-            per_doc = per.getContent()
-            per_doc.edit(mapping = dp)
-            per.manage_setLocalRoles(sid, ['Owner',])
-            wftool.doActionFor(student,'pume_pass')
-            wftool.doActionFor(student,'admit')
-            #
-            # Clearance
-            #
-            student.invokeFactory('StudentClearance','clearance')
-            #wftool.doActionFor(student.clearance,'open')
-            clearance = student.clearance
-            dc = {'Title': 'Clearance/Eligibility Record'}
-            clearance = student.clearance
-            date_str = result.get(csv_d['birthday'])
-            try:
-                date = DateTime.DateTime(date_str)
-            except:
-                #import pdb;pdb.set_trace()
-                date = None
-            dc['birthday'] = date
-            clearance.getContent().edit(mapping=dc)
-            clearance.manage_setLocalRoles(sid, ['Owner',])
-            #
-            # Study Course
-            #
-            student.invokeFactory('StudentStudyCourse','study_course')
-            study_course = student.study_course
-            dsc = {}
-            catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100')
-            #catalog_entry['level'] = dsc['current_level'] = '100'  # Attention: not for DE students
-            catalog_entry['session'] = dsc['current_session'] = da['entry_session']
-            catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']
-            catalog_entry['course'] = dsc['study_course'] = cert_id
-            catalog_entry['faculty'] = certificate['faculty']
-            catalog_entry['department'] = certificate['department']
-            catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'
-            catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)
-            study_course.getContent().edit(mapping=dsc)
-            #import pdb;pdb.set_trace()
-            self.students_catalog.addRecord(**catalog_entry)
-            if tr_count > 100:
-                if len(no_import) > 0:
-                    open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
-                             '\n'.join(no_import) + "\n")
-                    no_import = []
-                em = '%d transactions commited\n' % tr_count
-                transaction.commit()
-                logger.info(em)
-                total += tr_count
-                tr_count = 0
-        open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
-                                                '\n'.join(no_import))
-        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
-    ###)
 
     security.declareProtected(ModifyPortalContent,"transferStudents")###(
@@ -619,421 +279,4 @@
     ###)
 
-    security.declareProtected(ModifyPortalContent,"importReturningStudents")###(
-    def importReturningStudents(self):
-        """load Returning Studentdata from CSV values"""
-        import transaction
-        import random
-        #from pdb import set_trace
-        wftool = self.portal_workflow
-        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
-        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
-        students_folder = self.portal_url.getPortalObject().campus.students
-        tr_count = 1
-        total = 0
-        #name = 'pume_results'
-        name = 'Returning'
-        table = self.returning_import
-        no_import = []
-        imported = []
-        logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')
-        try:
-            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
-        except:
-            logger.error('Error reading %s.csv' % name)
-            return
-        l = self.portal_catalog({'meta_type': "Certificate"})
-        certs = {}
-        cert_docs = {}
-        for f in l:
-            certs[f.getId] = f.getObject().getContent()
-        start = True
-        res = table()
-        regs = []
-        if len(res) > 0:
-            regs = [s.matric_no for s in res]
-        #import pdb;pdb.set_trace()
-        for student in returning:
-            if start:
-                start = False
-                logger.info('Start loading from %s.csv' % name)
-                s = ','.join(['"%s"' % fn for fn in student.keys()])
-                imported.append(s)
-                no_import.append('%s,"Error"' % s)
-                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
-                format_error = format + ',"%(Error)s"'
-                no_certificate = "no certificate %s" % format
-            student['matric_no'] = matric_no = student.get('matric_no').upper()
-            student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
-            student['Permanent_Address'] = perm_address = student.get('Permanent Address')
-            if matric_no == '':
-                student['Error'] = "Empty matric_no"
-                no_import.append( format_error % student)
-                continue
-            if matric_no in regs or self.returning_import(matric_no = matric_no):
-                student['Error'] = "Duplicate"
-                no_import.append( format_error % student)
-                continue
-            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
-            if cert_id not in certs.keys():
-                student['Error'] = "No Certificate %s" % cert_id
-                no_import.append( format_error % student)
-                continue
-            try:
-                table.addRecord(**student)
-            except ValueError:
-                student['Error'] = "Duplicate"
-                no_import.append( format_error % student)
-                continue
-            regs.append(student.get('matric_no'))
-            imported.append(format % student)
-            tr_count += 1
-            if tr_count > 1000:
-                if len(no_import) > 0:
-                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
-                             '\n'.join(no_import) + '\n')
-                    no_import = []
-                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
-                                            '\n'.join(no_import) + "\n")
-                imported = []
-                em = '%d transactions commited total %s\n' % (tr_count,total)
-                transaction.commit()
-                regs = []
-                logger.info(em)
-                total += tr_count
-                tr_count = 0
-        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
-                                            '\n'.join(imported))
-        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
-                                                '\n'.join(no_import))
-        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
-    ###)
-
-    security.declareProtected(ModifyPortalContent,"fixAllNames")###(
-    def fixAllNames(self):
-        "fix all students names"
-        import transaction
-        response = self.REQUEST.RESPONSE
-        logger = logging.getLogger('fixAllNames')
-        logger.info('Start')
-        students = self.portal_catalog(portal_type='Student')
-        count = 0
-        total = 0
-        for student in students:
-            scat_res = self.students_catalog(id = student.getId)
-            if not scat_res:
-                self.students_catalog.addRecord(id = student.getId)
-                scat_res = self.students_catalog(id = student.getId)
-            student_entry = scat_res[0]
-            old_new = self.fixName(student,student_entry)
-            count += 1
-            response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))
-            if count > 2000:
-                transaction.commit()
-                logger.info("%d transactions commited" % count)
-                total += count
-                count = 0
-    ###)
-
-    security.declareProtected(ModifyPortalContent,"fixName")###(
-    def fixName(self,student_brain, student_entry):
-        "fix the name of a student"
-        fix = "first"
-        if student_entry.get('name_fixed',None) == fix:
-            return "Name already fixed"
-        student_id = student_entry.id
-        new_student = student_entry.jamb_reg_no.startswith('6')
-        student_obj = student_brain.getObject()
-        personal = getattr(student_obj,'personal',None)
-        invalid = ''
-        if personal is None:
-            return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")
-        per_doc = personal.getContent()
-        old_first = per_doc.firstname
-        old_middle = per_doc.middlename
-        old_last = per_doc.lastname
-        new_first = ''
-        new_middle = ''
-        new_last = ''
-        if new_student:
-            if not old_first and not old_middle and old_last:
-                new_names = [n.capitalize() for n in old_last.split()]
-                if len(new_names) > 1:
-                    old_first = new_names[0]
-                    old_last = new_names[-1]
-                    old_middle = ' '.join(new_names[1:-1])
-                else:
-                    old_last = new_names[0]
-                    old_first = ''
-                    old_middle = ''
-            if old_first:
-                new_first = old_first
-            if old_middle:
-                new_middle = old_middle
-            if old_last:
-                new_last = old_last
-            if old_first.find('<') != -1 or\
-               old_first.find('>') != -1 or\
-               old_middle.find('<') != -1 or\
-               old_middle.find('>') != -1 or\
-               old_last.find('<') != -1 or\
-               old_last.find('>') != -1:
-                   invalid = "invalid characters"
-        else:
-            new_first = old_first
-            if new_first.strip() == '-':
-                new_first = ''
-            new_middle = old_middle
-            if new_middle.strip() == '-':
-                new_middle = ''
-            new_last = old_last
-            if new_last.strip() == '-':
-                new_last = ''
-        name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()
-        if new_student:
-            text = "New"
-        else:
-            text = "Returning"
-        old_new = '"%s","%s","%s","%s"' % (invalid,text,
-                                           student_entry.name,
-                                           name)
-        if not invalid:
-            self.students_catalog.modifyRecord(id = student_id,
-                                      name_fixed = fix,
-                                      name = name)
-            per_doc.edit(mapping = {'firstname' : new_first,
-                                'middlename' : new_middle,
-                                'lastname' : new_last,
-                                })
-        return old_new
-    ###)
-
-    security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(
-    def updateReturningStudents(self):
-        """load and overwrite Returning Student Data from CSV values"""
-        import transaction
-        import random
-        #from pdb import set_trace
-        wftool = self.portal_workflow
-        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
-        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
-        students_folder = self.portal_url.getPortalObject().campus.students
-        tr_count = 1
-        total = 0
-        #name = 'pume_results'
-        name = 'Returning_update'
-        table = self.returning_import
-        no_import = []
-        imported = []
-        logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')
-        try:
-            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
-        except:
-            logger.error('Error reading %s.csv' % name)
-            return
-        l = self.portal_catalog({'meta_type': "Certificate"})
-        certs = {}
-        cert_docs = {}
-        for f in l:
-            certs[f.getId] = f.getObject().getContent()
-        start = True
-        res = table()
-        regs = []
-        if len(res) > 0:
-            regs = [s.matric_no for s in res]
-        for student in returning:
-            if start:
-                start = False
-                logger.info('Start loading from %s.csv' % name)
-                s = ','.join(['"%s"' % fn for fn in student.keys()])
-                imported.append(s)
-                no_import.append('%s,"Error"' % s)
-                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
-                format_error = format + ',"%(Error)s"'
-                no_certificate = "no certificate %s" % format
-            matric_no = student.get('matric_no').upper()
-            student['matric_no'] = matric_no
-            if matric_no == '':
-                student['Error'] = "Empty matric_no"
-                no_import.append( format_error % student)
-                continue
-#            if matric_no in regs or self.returning_import(matric_no = matric_no):
-#                student['Error'] = "Duplicate"
-#                no_import.append( format_error % student)
-#                continue
-#            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
-#            if cert_id not in certs.keys():
-#                student['Error'] = "No Certificate %s" % cert_id
-#                no_import.append( format_error % student)
-#                continue
-            try:
-                table.modifyRecord(**student)
-            except KeyError:
-                #import pdb;pdb.set_trace()
-                student['Error'] = "no Student found to update"
-                no_import.append( format_error % student)
-                continue
-            #s = self.students_catalog(matric_no=matric_no)
-            #if s:
-            #    level = "%s" % (int(student.get('Level')) + 100)
-            #    self.students_catalog.modifyRecord(id = s[0].id,
-            #                                           level=level)
-
-            regs.append(student.get('matric_no'))
-            imported.append(format % student)
-            tr_count += 1
-            if tr_count > 1000:
-                if len(no_import) > 0:
-                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
-                             '\n'.join(no_import) + '\n')
-                    no_import = []
-                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
-                                            '\n'.join(no_import) + "\n")
-                imported = []
-                em = '%d transactions commited total %s\n' % (tr_count,total)
-                transaction.commit()
-                regs = []
-                logger.info(em)
-                total += tr_count
-                tr_count = 0
-        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
-                                            '\n'.join(imported))
-        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
-                                                '\n'.join(no_import))
-        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
-    ###)
-
-    security.declareProtected(ModifyPortalContent,"exportStudents")###(
-    def exportStudents(self):
-        """export Studentsdata to a file"""
-        member = self.portal_membership.getAuthenticatedMember()
-        logger = logging.getLogger('Students.exportStudents')
-        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
-        students_folder = self.portal_url.getPortalObject().campus.students
-        export_file = "%s/export/students_%s.csv" % (i_home,current)
-
-        from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp
-        aq_students = self.students_catalog.evalAdvancedQuery
-        toexport = {'students_catalog': ("id",
-                                         "matric_no",
-                                         "jamb_reg_no",
-                                         "name",
-                                         "review_state",
-                                         "entry_session",
-                                         "session",
-                                         "entry_mode",
-                                         "mode",
-                                         "verdict",
-                                         "sex",
-                                         "email",
-                                         "phone",
-                                         "faculty",
-                                         "department",
-                                         "course",
-                                         "level",
-                                        ),
-                    'personal': ('firstname',
-                                 'middlename',
-                                 'lastname',
-                                 'perm_address',
-                                 ),
-                    'clearance': ('state',  # is not an attribute of clearance but is needed for splitting lga
-                                  'lga',
-                                 )
-                  }
-        res_list = []
-        lines = []
-        fields = []
-        with_lga = False
-        for k in toexport.keys():
-            for f in toexport[k]:
-                if f == 'lga' :
-                    with_lga = True
-                fields.append(f)
-        headline = ','.join(fields).replace('level','current_level')
-        open(export_file,"a").write(headline +'\n')
-        format = '"%(' + ')s","%('.join(fields) + ')s"'
-        query = In('review_state',('cleared_and_validated',
-                                'clearance_requested',
-                                'school_fee_paid',
-                                'courses_registered',
-                                'courses_validated'))
-        #query = In('review_state',('clearance_requested'))
-        students = aq_students(query)
-        nr2export = len(students)
-        logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))
-        chunk = 1000
-        total = 0
-        start = DateTime.DateTime().timeTime()
-        start_chunk = DateTime.DateTime().timeTime()
-        ## alternative method slightly slower
-        # student_recs = {}
-        # for s in students:
-        #      student_recs[s.id] = s
-        # catalog_recs = {}
-        # brains = self.portal_catalog(portal_type = 'Student')
-        # for cat_rec in brains:
-        #     sid = cat_rec.getId
-        #     catalog_recs[sid] = cat_rec
-        # #import pdb;pdb.set_trace()
-        # start = DateTime.DateTime().timeTime()
-        # start_chunk = DateTime.DateTime().timeTime()
-        # for student in students:
-        #     if student.id not in student_recs.keys():
-        #         continue
-        #     not_all = False
-        #     d = self.getFormattedStudentEntry(student_recs[student.id])
-        #     student_obj = catalog_recs[student.id].getObject()
-        for student in students:
-            not_all = False
-            d = self.getFormattedStudentEntry(student)
-            student_obj = getattr(students_folder,student.id)
-            for k in toexport.keys()[1:]:
-                try:
-                    object = getattr(student_obj,k)
-                    object_doc = object.getContent()
-                except:
-                    logger.info('%s %s record not found' % (student.id,k))
-                    not_all = True
-                    continue
-                for f in toexport[k]:
-                    d[f] = getattr(object_doc,f,'')
-            if not_all:
-                continue
-            if with_lga:
-                d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas)
-            lines.append(format % d)
-            total += 1
-            if total and not total % chunk or total == len(students):
-                open(export_file,"a").write('\n'.join(lines) +'\n')
-                anz = len(lines)
-                logger.info("wrote %(anz)d  total written %(total)d" % vars())
-                end_chunk = DateTime.DateTime().timeTime()
-                duration = end_chunk-start_chunk
-                per_record = duration/anz
-                till_now = end_chunk - start
-                avarage_per_record = till_now/total
-                estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
-                estimated_end = estimated_end.strftime("%H:%M:%S")
-                logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
-                start_chunk = DateTime.DateTime().timeTime()
-                lines = []
-        end = DateTime.DateTime().timeTime()
-        logger.info('total time %6.2f m' % ((end-start)/60))
-        filename, extension = os.path.splitext(export_file)
-        from subprocess import call
-        msg = "wrote %(total)d records to %(export_file)s" % vars()
-        try:
-            retcode = call('gzip %s' % (export_file),shell=True)
-            if retcode == 0:
-                msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
-        except OSError, e:
-            retcode = -99
-            logger.info("zip failed with %s" % e)
-        logger.info(msg)
-        args = {'portal_status_message': msg}
-        #url = self.REQUEST.get('URL1') + '?' + urlencode(args)
-        url = self.REQUEST.get('URL2')
-        return self.REQUEST.RESPONSE.redirect(url)
-    ###)
 
     security.declareProtected(ModifyPortalContent,"dumpStudentsCatalog")###(
@@ -1216,99 +459,5 @@
     ###)
 
-    security.declareProtected(ModifyPortalContent,"updateStudyCourse")###(
-    def updateStudyCourse(self):
-        """update StudyCourse from CSV values"""
-        import transaction
-        import random
-        from pdb import set_trace
-        wftool = self.portal_workflow
-        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
-        students_folder = self.portal_url.getPortalObject().campus.students
-        csv_d = {'jamb_reg_no': "RegNumber",
-                 'jamb_lastname': "Name",
-                 'session': "Session",
-                 'pume_tot_score': "PUME SCORE",
-                 'jamb_score': "JambScore",
-                 'jamb_sex': "Sex",
-                 'jamb_state': "State",
-##                 'jamb_first_cos': "AdminCourse",
-                 'faculty': "AdminFaculty",
-                 'course_code': "AdmitCoscode",
-                 'stud_status':"AdmitStatus",
-                 'department': "AdmitDept",
-                 'jamb_lga': "LGA",
-                 'app_email': "email",
-                 'app_mobile': "PhoneNumbers",
-                 }
-        csv_fields = [f[1] for f in csv_d.items()]
-        tr_count = 0
-        total = 0
-        #name = 'pume_results'
-        name = 'StudyCourseChange'
-        no_import = []
-        s = ','.join(['"%s"' % fn for fn in csv_fields])
-        no_import.append('"Error",%s' % s)
-        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
-        no_certificate = "no certificate %s" % format
-        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
-                    '\n'.join(no_import))
-        logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse')
-        logger.info('Start loading from %s.csv' % name)
-        l = self.portal_catalog({'meta_type': "Certificate"})
-        try:
-            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
-        except:
-            logger.error('Error reading %s.csv' % name)
-            return
-        for jamb in result:
-            jamb['Error'] = "Processing "
-            logger.info(format % jamb)
-            jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
-            res = self.portal_catalog({'portal_type': "StudentApplication",
-                                     'SearchableText': jamb_reg_no })
-            if not res:
-                em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no
-                logger.info(em)
-                jamb['Error'] = "Student does not exist"
-                no_import.append(format % jamb)
-                continue
-            sid = res[0].getPath().split('/')[-2]
-            cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
-            res = self.portal_catalog(portal_type = "Certificate", id = cert_id)
-            if not res:
-                em = 'No Certificate with ID %s \n' % cert_id
-                logger.info(em)
-                jamb['Error'] = "No Certificate %s" % cert_id
-                no_import.append( format % jamb)
-                continue
-            cert_brain = res[0]
-            catalog_entry = {}
-            student = getattr(self,sid)
-            #
-            # Study Course
-            #
-            study_course = student.study_course
-            dsc = {}
-            cert_pl = cert_brain.getPath().split('/')
-            catalog_entry['id'] = sid
-            catalog_entry['faculty'] = cert_pl[-4]
-            catalog_entry['department'] = cert_pl[-3]
-            catalog_entry['course'] = cert_id
-            dsc['study_course'] = cert_id
-            study_course.getContent().edit(mapping=dsc)
-            self.students_catalog.modifyRecord(**catalog_entry)
-            if tr_count > 10:
-                if len(no_import) > 1:
-                    open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write(
-                             '\n'.join(no_import))
-                    no_import = []
-                em = '%d transactions commited\n' % tr_count
-                transaction.commit()
-                logger.info(em)
-                total += tr_count
-                tr_count = 0
-            tr_count += 1
-        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
-    ###)
+
 
     security.declareProtected(View,"fixOwnership") ###(
Index: /WAeUP_SRP/base/Students.unused_methods.txt
===================================================================
--- /WAeUP_SRP/base/Students.unused_methods.txt	(revision 2653)
+++ /WAeUP_SRP/base/Students.unused_methods.txt	(revision 2654)
@@ -1452,2 +1452,858 @@
         return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
     ###)
+
+    security.declareProtected(ModifyPortalContent,"loadPumeResultsFromCSV")###(
+    def loadPumeResultsFromCSV(self):
+        """load Fulltime Studentdata from CSV values into pumeresults catalog"""
+        import transaction
+        import random
+##        csv_d = {'jamb_reg_no': "RegNumber", ###(
+##                 'status': "Admission Status",
+##                 'name': "Name",
+##                 'score': "Score",
+##                 'sex': "Sex",
+##                 'faculty': "Faculty",
+##                 'department': "Dept",
+##                 'course': "Course",
+##                 'course_code_org': "Course Code",
+##                 }
+###)
+##        csv_d = {'jamb_reg_no': "JAMBRegno",
+##                'name': "Name",
+##                'score': "Score",
+##                 'sex': "Sex",
+##                 'course': "Course",
+##                 'faculty': "Faculty",
+##                 'department': "Dept",
+##                 'course_code_org': "Course Code",
+##                 'status': "Admission Status",
+##                 'result_type': None,
+##                 }
+
+        csv_d = {'jamb_reg_no': "reg_no",
+                 'name': "fullname",
+                 'score': "pume_score",
+                 'sex': "sex",
+                 'course': "study_course",
+                 'course_code_org': "study_course",
+                 'status': "admission_status",
+                 'result_type': "entry_mode",
+                 }
+
+        csv_fields = [f[1] for f in csv_d.items() if f[1]]
+        tr_count = 0
+        total = 0
+        #name = 'pup_new'
+        #name = 'pup_update'
+        name = 'Admitted_update'
+        update = name.endswith('update')
+        no_import = []
+        ok_import = []
+        ok_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_d.keys()]))
+        no_import.append('%s' % ','.join(['"%s"' % fn for fn in csv_fields]))
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        ok_import_name = "%s/import/%s_imported_%s.csv" % (i_home,name,current)
+        #open(ok_import_name,"w").write('\n'.join(no_import))
+        no_import_name = "%s/import/%s_not_imported_%s.csv" % (i_home,name,current)
+        #open(no_import_name,"w").write('\n'.join(no_import))
+        logger = logging.getLogger('Students.loadPumeResultsFromCSV')
+        starttime = DateTime.now()
+        logger.info('Start loading from %s.csv' % name)
+        try:
+            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        pume = self.portal_pumeresults
+        format = ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
+        import_format = ','.join(['"%%(%s)s"' % fn for fn in csv_d.keys()])
+        eduplicate = '%s,"duplicate"' % format
+        eoriginal = '%s,"original"' % format
+        einvalidjamb = '%s,"invalid JambRegNo"' % format
+        added = 'added ,%s' % format
+        #from pdb import set_trace;set_trace()
+        for jamb in result:
+            if not jamb.get(csv_d['score']):
+                logger.info('Student %s has no pume_score' % jamb.get(csv_d['jamb_reg_no']))
+                continue
+            dict = {}
+            for f,fn in csv_d.items():
+                dict[f] = jamb.get(csv_d[f])
+            dict['result_type'] = 'CEST'
+            jnr = jamb.get(csv_d['jamb_reg_no'])
+            #if not checkJambNo(jnr):
+            #    logger.info(einvalidjamb % jamb)
+            #    dd = {}
+            #    for f,fn in csv_d.items():
+            #        dd[fn] = getattr(data,f)
+            #        no_import.append(eduplicate % dd)
+            #        no_import.append(eduplicate % jamb)
+            #    continue
+            res = pume(jamb_reg_no=jnr)
+            if len(res) > 0:
+                if update:
+                    try:
+                        pume.modifyRecord(**dict)
+                    # Can not happen, but anyway...
+                    except ValueError:
+                        logger.info(eduplicate % jamb)
+                        continue
+                    # Can not happen, but anyway...
+                    except KeyError:
+                        pume.addRecord(**dict)
+                        logger.info(added % jamb)
+                        continue
+                else:
+                    data = res[0]
+                    if data.name != jamb.get(csv_d['name']):
+                        #set_trace()
+                        logger.info(eduplicate % jamb)
+                        #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
+                        #logger.info(em)
+                        dd = {}
+                        for f,fn in csv_d.items():
+                            dd[fn] = getattr(data,f)
+                        no_import.append(eoriginal % dd)
+                        no_import.append(eduplicate % jamb)
+                    continue
+            else:
+                try:
+                    pume.addRecord(**dict)
+                    ok_import.append(import_format % dict)
+                except ValueError:
+                    logger.info(eduplicate % jamb)
+                    #em = 'Student with REG-NO %(jamb_reg_no)s already exists\n' % dict
+                    #logger.info(em)
+                    no_import.append(eduplicate % jamb)
+        logger.info('End loading from %s.csv' % name)
+        if len(no_import) > 1:
+            open(no_import_name,"w+").write('\n'.join(no_import))
+        open(ok_import_name,"w+").write('\n'.join(ok_import))
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"createStudents")###(
+    def createStudents(self):
+        """
+        load addmitted Studentdata from CSV values and create Studentobjects.
+        This is the current method to create new addmitted Students.
+        Before running the eventservice for the students_catalog must be disabled.
+        """
+        import transaction
+        import random
+        #from pdb import set_trace
+        wftool = self.portal_workflow
+        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
+        students_folder = self.portal_url.getPortalObject().campus.students
+        levels =       {'ume_ft':'100',
+                        'de_ft': '200',
+                        'ug_pt': '100',
+                        'de_pt': '200',
+                        'pg_ft': '700',
+                        'pg_pt': '700',
+                        'dp_pt': '100',
+                        'dp_ft': '100',
+                        }
+        csv_d = {'jamb_reg_no': "reg_no",
+                 'entry_mode': 'entry_mode',
+                 'jamb_firstname': "firstname",
+                 'jamb_middlename': "middlename",
+                 'jamb_lastname': "lastname",
+                 'jamb_sex': "sex",
+                 'jamb_state': "state",
+                 'birthday': "date_of_birth",
+                 'app_email': "email",
+                 'study_course': "study_course",
+                 'perm_address': "address",
+                 'admission_status': "admission_status",
+                 }
+        csv_fields = [f[1] for f in csv_d.items()]
+        tr_count = 0
+        total = 0
+        #name = 'pume_results'
+        name = 'Admitted_update'
+        no_import = []
+        s = ','.join(['"%s"' % fn for fn in csv_fields])
+        no_import.append('"Error",%s' % s)
+        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
+        no_certificate = "no certificate %s" % format
+        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
+        logger = logging.getLogger('Students.StudentsFolder.createStudents')
+        logger.info('Start loading from %s.csv' % name)
+        certs = {}
+        try:
+            results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        for result in results:
+            if not result.get(csv_d['admission_status']).startswith('Admitted'):
+                continue
+            #result['Error'] = "Processing "
+            #logger.info(format % result)
+            jamb_reg_no = result.get(csv_d['jamb_reg_no'])
+            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
+            if res:
+                em = 'Student with RegNo %s already exists\n' % jamb_reg_no
+                logger.info(em)
+                result['Error'] = "Student exists"
+                no_import.append(format % result)
+                continue
+            cert_id = makeCertificateCode(result.get(csv_d['study_course']))
+            if cert_id not in certs.keys():
+                res = self.portal_catalog(meta_type = "Certificate",id = cert_id)
+                if not res:
+                    em = 'No Certificate with ID %s \n' % cert_id
+                    logger.info(em)
+                    result['Error'] = "No Certificate %s" % cert_id
+                    no_import.append( format % result)
+                    continue
+                cert = res[0]
+                cert_path = cert.getPath().split('/')
+                certificate = certs[cert_id] = {'faculty': cert_path[-4],
+                                     'department': cert_path[-3]}
+            cert_doc = certs[cert_id]
+            catalog_entry = {}
+            catalog_entry['jamb_reg_no'] = jamb_reg_no
+            firstname = result.get(csv_d['jamb_firstname'])
+            middlename = result.get(csv_d['jamb_middlename'])
+            lastname = result.get(csv_d['jamb_lastname'])
+            if len(firstname) < 3\
+               and len(middlename) < 3\
+               and len(lastname) < 3:
+                em = 'Student Names to short \n'
+                logger.info(em)
+                result['Error'] = "Names to short"
+                no_import.append( format % result)
+                continue
+            perm_address = result.get(csv_d['perm_address'])
+            sid = self.generateStudentId('x')
+            students_folder.invokeFactory('Student', sid)
+            catalog_entry['id'] = sid
+            tr_count += 1
+            logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
+            student = getattr(self,sid)
+            student.manage_setLocalRoles(sid, ['Owner',])
+            student.invokeFactory('StudentApplication','application')
+            da = {'Title': 'Application Data'}
+            da["jamb_firstname"] = firstname
+            da["jamb_middlename"] = middlename
+            da["jamb_lastname"] = lastname
+            catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[0]
+            catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')
+            da_fields = ('jamb_reg_no',
+                         'jamb_sex',
+                         'jamb_state',
+                         'entry_mode',
+                         'app_email',
+                         )
+            for f in da_fields:
+                da[f] = result.get(csv_d[f])
+            catalog_entry['email'] = da['app_email']
+            catalog_entry['entry_mode'] = da['entry_mode']
+            app = student.application
+            app_doc = app.getContent()
+            app.getContent().edit(mapping=da)
+            picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
+            app.manage_setLocalRoles(sid, ['Owner',])
+
+            picture_id = da['jamb_reg_no'].replace('/','_')
+            file = None
+            for ext in ('jpg','JPG'):
+                picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)
+                if os.path.exists(picture):
+                    file = open(picture)
+                    break
+            if file is not None:
+
+                ## file conversion does not work
+                #img = PIL.Image.open(file)
+                #img.thumbnail((150,200),
+                #              resample=PIL.Image.ANTIALIAS)
+                #outfile = StringIO()
+                #img.save(outfile, format=img.format)
+
+                outfile = file.read()
+                app_doc.manage_addFile('passport',
+                                       file=outfile,
+                                       title="%s.jpg" % jamb_reg_no)
+            #wftool.doActionFor(app,'close')
+            dp = {}
+            dp['firstname'] = firstname
+            dp['middlename'] = middlename
+            dp['lastname'] = lastname
+            dp['email'] = da['app_email']
+            dp['sex'] = sex
+            dp['perm_address'] = perm_address
+            catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
+            student.invokeFactory('StudentPersonal','personal')
+            per = student.personal
+            per_doc = per.getContent()
+            per_doc.edit(mapping = dp)
+            per.manage_setLocalRoles(sid, ['Owner',])
+            wftool.doActionFor(student,'pume_pass')
+            wftool.doActionFor(student,'admit')
+            #
+            # Clearance
+            #
+            student.invokeFactory('StudentClearance','clearance')
+            #wftool.doActionFor(student.clearance,'open')
+            clearance = student.clearance
+            dc = {'Title': 'Clearance/Eligibility Record'}
+            clearance = student.clearance
+            date_str = result.get(csv_d['birthday'])
+            try:
+                date = DateTime.DateTime(date_str)
+            except:
+                #import pdb;pdb.set_trace()
+                date = None
+            dc['birthday'] = date
+            clearance.getContent().edit(mapping=dc)
+            clearance.manage_setLocalRoles(sid, ['Owner',])
+            #
+            # Study Course
+            #
+            student.invokeFactory('StudentStudyCourse','study_course')
+            study_course = student.study_course
+            dsc = {}
+            catalog_entry['level'] = dsc['current_level'] = levels.get(da['entry_mode'],'100')
+            #catalog_entry['level'] = dsc['current_level'] = '100'  # Attention: not for DE students
+            catalog_entry['session'] = dsc['current_session'] = da['entry_session']
+            catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']
+            catalog_entry['course'] = dsc['study_course'] = cert_id
+            catalog_entry['faculty'] = certificate['faculty']
+            catalog_entry['department'] = certificate['department']
+            catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'
+            catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)
+            study_course.getContent().edit(mapping=dsc)
+            #import pdb;pdb.set_trace()
+            self.students_catalog.addRecord(**catalog_entry)
+            if tr_count > 100:
+                if len(no_import) > 0:
+                    open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
+                             '\n'.join(no_import) + "\n")
+                    no_import = []
+                em = '%d transactions commited\n' % tr_count
+                transaction.commit()
+                logger.info(em)
+                total += tr_count
+                tr_count = 0
+        open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
+                                                '\n'.join(no_import))
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
+
+
+    security.declareProtected(ModifyPortalContent,"importReturningStudents")###(
+    def importReturningStudents(self):
+        """load Returning Studentdata from CSV values"""
+        import transaction
+        import random
+        #from pdb import set_trace
+        wftool = self.portal_workflow
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
+        students_folder = self.portal_url.getPortalObject().campus.students
+        tr_count = 1
+        total = 0
+        #name = 'pume_results'
+        name = 'Returning'
+        table = self.returning_import
+        no_import = []
+        imported = []
+        logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')
+        try:
+            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        l = self.portal_catalog({'meta_type': "Certificate"})
+        certs = {}
+        cert_docs = {}
+        for f in l:
+            certs[f.getId] = f.getObject().getContent()
+        start = True
+        res = table()
+        regs = []
+        if len(res) > 0:
+            regs = [s.matric_no for s in res]
+        #import pdb;pdb.set_trace()
+        for student in returning:
+            if start:
+                start = False
+                logger.info('Start loading from %s.csv' % name)
+                s = ','.join(['"%s"' % fn for fn in student.keys()])
+                imported.append(s)
+                no_import.append('%s,"Error"' % s)
+                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
+                format_error = format + ',"%(Error)s"'
+                no_certificate = "no certificate %s" % format
+            student['matric_no'] = matric_no = student.get('matric_no').upper()
+            student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
+            student['Permanent_Address'] = perm_address = student.get('Permanent Address')
+            if matric_no == '':
+                student['Error'] = "Empty matric_no"
+                no_import.append( format_error % student)
+                continue
+            if matric_no in regs or self.returning_import(matric_no = matric_no):
+                student['Error'] = "Duplicate"
+                no_import.append( format_error % student)
+                continue
+            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
+            if cert_id not in certs.keys():
+                student['Error'] = "No Certificate %s" % cert_id
+                no_import.append( format_error % student)
+                continue
+            try:
+                table.addRecord(**student)
+            except ValueError:
+                student['Error'] = "Duplicate"
+                no_import.append( format_error % student)
+                continue
+            regs.append(student.get('matric_no'))
+            imported.append(format % student)
+            tr_count += 1
+            if tr_count > 1000:
+                if len(no_import) > 0:
+                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                             '\n'.join(no_import) + '\n')
+                    no_import = []
+                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
+                                            '\n'.join(no_import) + "\n")
+                imported = []
+                em = '%d transactions commited total %s\n' % (tr_count,total)
+                transaction.commit()
+                regs = []
+                logger.info(em)
+                total += tr_count
+                tr_count = 0
+        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
+                                            '\n'.join(imported))
+        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                                                '\n'.join(no_import))
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"fixAllNames")###(
+    def fixAllNames(self):
+        "fix all students names"
+        import transaction
+        response = self.REQUEST.RESPONSE
+        logger = logging.getLogger('fixAllNames')
+        logger.info('Start')
+        students = self.portal_catalog(portal_type='Student')
+        count = 0
+        total = 0
+        for student in students:
+            scat_res = self.students_catalog(id = student.getId)
+            if not scat_res:
+                self.students_catalog.addRecord(id = student.getId)
+                scat_res = self.students_catalog(id = student.getId)
+            student_entry = scat_res[0]
+            old_new = self.fixName(student,student_entry)
+            count += 1
+            response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))
+            if count > 2000:
+                transaction.commit()
+                logger.info("%d transactions commited" % count)
+                total += count
+                count = 0
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"fixName")###(
+    def fixName(self,student_brain, student_entry):
+        "fix the name of a student"
+        fix = "first"
+        if student_entry.get('name_fixed',None) == fix:
+            return "Name already fixed"
+        student_id = student_entry.id
+        new_student = student_entry.jamb_reg_no.startswith('6')
+        student_obj = student_brain.getObject()
+        personal = getattr(student_obj,'personal',None)
+        invalid = ''
+        if personal is None:
+            return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")
+        per_doc = personal.getContent()
+        old_first = per_doc.firstname
+        old_middle = per_doc.middlename
+        old_last = per_doc.lastname
+        new_first = ''
+        new_middle = ''
+        new_last = ''
+        if new_student:
+            if not old_first and not old_middle and old_last:
+                new_names = [n.capitalize() for n in old_last.split()]
+                if len(new_names) > 1:
+                    old_first = new_names[0]
+                    old_last = new_names[-1]
+                    old_middle = ' '.join(new_names[1:-1])
+                else:
+                    old_last = new_names[0]
+                    old_first = ''
+                    old_middle = ''
+            if old_first:
+                new_first = old_first
+            if old_middle:
+                new_middle = old_middle
+            if old_last:
+                new_last = old_last
+            if old_first.find('<') != -1 or\
+               old_first.find('>') != -1 or\
+               old_middle.find('<') != -1 or\
+               old_middle.find('>') != -1 or\
+               old_last.find('<') != -1 or\
+               old_last.find('>') != -1:
+                   invalid = "invalid characters"
+        else:
+            new_first = old_first
+            if new_first.strip() == '-':
+                new_first = ''
+            new_middle = old_middle
+            if new_middle.strip() == '-':
+                new_middle = ''
+            new_last = old_last
+            if new_last.strip() == '-':
+                new_last = ''
+        name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()
+        if new_student:
+            text = "New"
+        else:
+            text = "Returning"
+        old_new = '"%s","%s","%s","%s"' % (invalid,text,
+                                           student_entry.name,
+                                           name)
+        if not invalid:
+            self.students_catalog.modifyRecord(id = student_id,
+                                      name_fixed = fix,
+                                      name = name)
+            per_doc.edit(mapping = {'firstname' : new_first,
+                                'middlename' : new_middle,
+                                'lastname' : new_last,
+                                })
+        return old_new
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(
+    def updateReturningStudents(self):
+        """load and overwrite Returning Student Data from CSV values"""
+        import transaction
+        import random
+        #from pdb import set_trace
+        wftool = self.portal_workflow
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
+        students_folder = self.portal_url.getPortalObject().campus.students
+        tr_count = 1
+        total = 0
+        #name = 'pume_results'
+        name = 'Returning_update'
+        table = self.returning_import
+        no_import = []
+        imported = []
+        logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')
+        try:
+            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        l = self.portal_catalog({'meta_type': "Certificate"})
+        certs = {}
+        cert_docs = {}
+        for f in l:
+            certs[f.getId] = f.getObject().getContent()
+        start = True
+        res = table()
+        regs = []
+        if len(res) > 0:
+            regs = [s.matric_no for s in res]
+        for student in returning:
+            if start:
+                start = False
+                logger.info('Start loading from %s.csv' % name)
+                s = ','.join(['"%s"' % fn for fn in student.keys()])
+                imported.append(s)
+                no_import.append('%s,"Error"' % s)
+                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
+                format_error = format + ',"%(Error)s"'
+                no_certificate = "no certificate %s" % format
+            matric_no = student.get('matric_no').upper()
+            student['matric_no'] = matric_no
+            if matric_no == '':
+                student['Error'] = "Empty matric_no"
+                no_import.append( format_error % student)
+                continue
+#            if matric_no in regs or self.returning_import(matric_no = matric_no):
+#                student['Error'] = "Duplicate"
+#                no_import.append( format_error % student)
+#                continue
+#            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
+#            if cert_id not in certs.keys():
+#                student['Error'] = "No Certificate %s" % cert_id
+#                no_import.append( format_error % student)
+#                continue
+            try:
+                table.modifyRecord(**student)
+            except KeyError:
+                #import pdb;pdb.set_trace()
+                student['Error'] = "no Student found to update"
+                no_import.append( format_error % student)
+                continue
+            #s = self.students_catalog(matric_no=matric_no)
+            #if s:
+            #    level = "%s" % (int(student.get('Level')) + 100)
+            #    self.students_catalog.modifyRecord(id = s[0].id,
+            #                                           level=level)
+
+            regs.append(student.get('matric_no'))
+            imported.append(format % student)
+            tr_count += 1
+            if tr_count > 1000:
+                if len(no_import) > 0:
+                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                             '\n'.join(no_import) + '\n')
+                    no_import = []
+                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
+                                            '\n'.join(no_import) + "\n")
+                imported = []
+                em = '%d transactions commited total %s\n' % (tr_count,total)
+                transaction.commit()
+                regs = []
+                logger.info(em)
+                total += tr_count
+                tr_count = 0
+        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
+                                            '\n'.join(imported))
+        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                                                '\n'.join(no_import))
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"exportStudents")###(
+    def exportStudents(self):
+        """export Studentsdata to a file"""
+        member = self.portal_membership.getAuthenticatedMember()
+        logger = logging.getLogger('Students.exportStudents')
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        students_folder = self.portal_url.getPortalObject().campus.students
+        export_file = "%s/export/students_%s.csv" % (i_home,current)
+
+        from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp
+        aq_students = self.students_catalog.evalAdvancedQuery
+        toexport = {'students_catalog': ("id",
+                                         "matric_no",
+                                         "jamb_reg_no",
+                                         "name",
+                                         "review_state",
+                                         "entry_session",
+                                         "session",
+                                         "entry_mode",
+                                         "mode",
+                                         "verdict",
+                                         "sex",
+                                         "email",
+                                         "phone",
+                                         "faculty",
+                                         "department",
+                                         "course",
+                                         "level",
+                                        ),
+                    'personal': ('firstname',
+                                 'middlename',
+                                 'lastname',
+                                 'perm_address',
+                                 ),
+                    'clearance': ('state',  # is not an attribute of clearance but is needed for splitting lga
+                                  'lga',
+                                 )
+                  }
+        res_list = []
+        lines = []
+        fields = []
+        with_lga = False
+        for k in toexport.keys():
+            for f in toexport[k]:
+                if f == 'lga' :
+                    with_lga = True
+                fields.append(f)
+        headline = ','.join(fields).replace('level','current_level')
+        open(export_file,"a").write(headline +'\n')
+        format = '"%(' + ')s","%('.join(fields) + ')s"'
+        query = In('review_state',('cleared_and_validated',
+                                'clearance_requested',
+                                'school_fee_paid',
+                                'courses_registered',
+                                'courses_validated'))
+        #query = In('review_state',('clearance_requested'))
+        students = aq_students(query)
+        nr2export = len(students)
+        logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))
+        chunk = 1000
+        total = 0
+        start = DateTime.DateTime().timeTime()
+        start_chunk = DateTime.DateTime().timeTime()
+        ## alternative method slightly slower
+        # student_recs = {}
+        # for s in students:
+        #      student_recs[s.id] = s
+        # catalog_recs = {}
+        # brains = self.portal_catalog(portal_type = 'Student')
+        # for cat_rec in brains:
+        #     sid = cat_rec.getId
+        #     catalog_recs[sid] = cat_rec
+        # #import pdb;pdb.set_trace()
+        # start = DateTime.DateTime().timeTime()
+        # start_chunk = DateTime.DateTime().timeTime()
+        # for student in students:
+        #     if student.id not in student_recs.keys():
+        #         continue
+        #     not_all = False
+        #     d = self.getFormattedStudentEntry(student_recs[student.id])
+        #     student_obj = catalog_recs[student.id].getObject()
+        for student in students:
+            not_all = False
+            d = self.getFormattedStudentEntry(student)
+            student_obj = getattr(students_folder,student.id)
+            for k in toexport.keys()[1:]:
+                try:
+                    object = getattr(student_obj,k)
+                    object_doc = object.getContent()
+                except:
+                    logger.info('%s %s record not found' % (student.id,k))
+                    not_all = True
+                    continue
+                for f in toexport[k]:
+                    d[f] = getattr(object_doc,f,'')
+            if not_all:
+                continue
+            if with_lga:
+                d['state'],d['lga'] = formatLGA(d['lga'],voc = self.portal_vocabularies.local_gov_areas)
+            lines.append(format % d)
+            total += 1
+            if total and not total % chunk or total == len(students):
+                open(export_file,"a").write('\n'.join(lines) +'\n')
+                anz = len(lines)
+                logger.info("wrote %(anz)d  total written %(total)d" % vars())
+                end_chunk = DateTime.DateTime().timeTime()
+                duration = end_chunk-start_chunk
+                per_record = duration/anz
+                till_now = end_chunk - start
+                avarage_per_record = till_now/total
+                estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
+                estimated_end = estimated_end.strftime("%H:%M:%S")
+                logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
+                start_chunk = DateTime.DateTime().timeTime()
+                lines = []
+        end = DateTime.DateTime().timeTime()
+        logger.info('total time %6.2f m' % ((end-start)/60))
+        filename, extension = os.path.splitext(export_file)
+        from subprocess import call
+        msg = "wrote %(total)d records to %(export_file)s" % vars()
+        try:
+            retcode = call('gzip %s' % (export_file),shell=True)
+            if retcode == 0:
+                msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
+        except OSError, e:
+            retcode = -99
+            logger.info("zip failed with %s" % e)
+        logger.info(msg)
+        args = {'portal_status_message': msg}
+        #url = self.REQUEST.get('URL1') + '?' + urlencode(args)
+        url = self.REQUEST.get('URL2')
+        return self.REQUEST.RESPONSE.redirect(url)
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"updateStudyCourse")###(
+    def updateStudyCourse(self):
+        """update StudyCourse from CSV values"""
+        import transaction
+        import random
+        from pdb import set_trace
+        wftool = self.portal_workflow
+        #students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
+        students_folder = self.portal_url.getPortalObject().campus.students
+        csv_d = {'jamb_reg_no': "RegNumber",
+                 'jamb_lastname': "Name",
+                 'session': "Session",
+                 'pume_tot_score': "PUME SCORE",
+                 'jamb_score': "JambScore",
+                 'jamb_sex': "Sex",
+                 'jamb_state': "State",
+##                 'jamb_first_cos': "AdminCourse",
+                 'faculty': "AdminFaculty",
+                 'course_code': "AdmitCoscode",
+                 'stud_status':"AdmitStatus",
+                 'department': "AdmitDept",
+                 'jamb_lga': "LGA",
+                 'app_email': "email",
+                 'app_mobile': "PhoneNumbers",
+                 }
+        csv_fields = [f[1] for f in csv_d.items()]
+        tr_count = 0
+        total = 0
+        #name = 'pume_results'
+        name = 'StudyCourseChange'
+        no_import = []
+        s = ','.join(['"%s"' % fn for fn in csv_fields])
+        no_import.append('"Error",%s' % s)
+        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
+        no_certificate = "no certificate %s" % format
+        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
+                    '\n'.join(no_import))
+        logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse')
+        logger.info('Start loading from %s.csv' % name)
+        l = self.portal_catalog({'meta_type': "Certificate"})
+        try:
+            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        for jamb in result:
+            jamb['Error'] = "Processing "
+            logger.info(format % jamb)
+            jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
+            res = self.portal_catalog({'portal_type': "StudentApplication",
+                                     'SearchableText': jamb_reg_no })
+            if not res:
+                em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no
+                logger.info(em)
+                jamb['Error'] = "Student does not exist"
+                no_import.append(format % jamb)
+                continue
+            sid = res[0].getPath().split('/')[-2]
+            cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
+            res = self.portal_catalog(portal_type = "Certificate", id = cert_id)
+            if not res:
+                em = 'No Certificate with ID %s \n' % cert_id
+                logger.info(em)
+                jamb['Error'] = "No Certificate %s" % cert_id
+                no_import.append( format % jamb)
+                continue
+            cert_brain = res[0]
+            catalog_entry = {}
+            student = getattr(self,sid)
+            #
+            # Study Course
+            #
+            study_course = student.study_course
+            dsc = {}
+            cert_pl = cert_brain.getPath().split('/')
+            catalog_entry['id'] = sid
+            catalog_entry['faculty'] = cert_pl[-4]
+            catalog_entry['department'] = cert_pl[-3]
+            catalog_entry['course'] = cert_id
+            dsc['study_course'] = cert_id
+            study_course.getContent().edit(mapping=dsc)
+            self.students_catalog.modifyRecord(**catalog_entry)
+            if tr_count > 10:
+                if len(no_import) > 1:
+                    open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write(
+                             '\n'.join(no_import))
+                    no_import = []
+                em = '%d transactions commited\n' % tr_count
+                transaction.commit()
+                logger.info(em)
+                total += tr_count
+                tr_count = 0
+            tr_count += 1
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
