#-*- mode: python; mode: fold -*-
# $Id: Students.py 1728 2007-05-03 07:54:44Z joachim $
from string import Template
from Globals import InitializeClass
from AccessControl import ClassSecurityInfo
from AccessControl.SecurityManagement import newSecurityManager
from zExceptions import BadRequest
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.CMFCore.utils import UniqueObject, getToolByName
from Products.CMFCore.permissions import View
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CPSCore.CPSBase import CPSBase_adder, CPSBaseFolder
#from Products.CPSCore.CPSBase import CPSBaseDocument as BaseDocument
from Products.CPSDocument.CPSDocument import CPSDocument
from Products.CPSCore.CPSBase import CPSBaseBTreeFolder as BaseBTreeFolder
from Products.CPSCore.CPSMembershipTool import CPSUnrestrictedUser
from Products.WAeUP_SRP.Academics import makeCertificateCode
from Products.AdvancedQuery import Eq, Between, Le,In
import logging
import csv,re,os
import Globals
p_home = Globals.package_home(globals())
i_home = Globals.INSTANCE_HOME
MAX_TRANS = 1000
from urllib import urlencode

import DateTime
import PIL.Image
from StringIO import StringIO

def makeCertificateCode(code): ###(
    code = code.replace('.','')
    code = code.replace('(','')
    code = code.replace(')','')
    code = code.replace('/','')
    code = code.replace(' ','')
    code = code.replace('_','')
    return code

###)

def response_write(response,s):
    response.setHeader('Content-type','text/html; charset=ISO-8859-15')
    while s.find('<') > -1:
        s = s.replace('<','&lt;')
    while s.find('>') > -1:
        #from pdb import set_trace;set_trace()
        s = s.replace('>','&gt;')
    response.write("%s<br>\n" % s)

def getInt(s): ###(
    try:
        return int(s)
    except:
        return 0

def getFloat(s):
    try:
        return float(s)
    except:
        return 0.0

###)

def getStudentByRegNo(self,reg_no): ###(
    """search student by JAMB Reg No and return StudentFolder"""
    search = ZCatalog.searchResults(self.portal_catalog,{'meta_type': 'StudentApplication',
                                  'SearchableText': reg_no,
                                  })
    if len(search) < 1:
        return None
    return search[0].getObject().aq_parent

###)

def checkJambNo(jnr):
    try:
        if len(jnr) != 10:
            return False
    except:
        return False
    try:
        int(jnr[:8])
        return True
    except:
        return False

class StudentsFolder(CPSDocument): ###(
    """
    WAeUP container for the various WAeUP containers data
    """
    meta_type = 'StudentsFolder'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(ModifyPortalContent,"createDEStudents")###(
    def createDEStudents(self):
        """load Fulltime Studentdata from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        csv_d = {'jamb_reg_no': "RegNumber",
                 'jamb_lastname': "Name",
                 'session': "Session",
                 'pume_tot_score': "PUDE SCORE",
                 ##'jamb_score': "JambScore",
                 'entry_mode': "EntryMode",
                 'jamb_sex': "Sex",
                 'jamb_state': "State",
                 'jamb_first_cos': "AdminCourse",
                 'faculty': "AdminFaculty",
                 'course_code': "AdmitCoscode",
                 'stud_status':"AdmitStatus",
                 'department': "AdmitDept",
                 'jamb_lga': "LGA",
                 'app_email': "email",
                 'app_mobile': "PhoneNumbers",
                 }
        csv_fields = [f[1] for f in csv_d.items()]
        tr_count = 0
        total = 0
        #name = 'pume_results'
        name = 'DE_Admitted'
        no_import = []
        s = ','.join(['"%s"' % fn for fn in csv_fields])
        no_import.append('"Error",%s' % s)
        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
        no_certificate = "no certificate %s" % format
        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
        logger = logging.getLogger('Students.StudentsFolder.createDEStudents')
        logger.info('Start loading from %s.csv' % name)
        l = self.portal_catalog({'meta_type': "Certificate"})
        certs = {}
        cert_docs = {}
        for f in l:
            certs[f.getId] = f.getObject().getContent()
        try:
            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for jamb in result:
            jamb['Error'] = "Processing"
            logger.info(format % jamb)
            jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
            res = self.portal_catalog({'portal_type': "StudentApplication",
                                     'SearchableText': jamb_reg_no })
            if res:
                em = 'Student with RegNo %s already exists\n' % jamb_reg_no
                logger.info(em)
                jamb['Error'] = "Student exists"
                no_import.append(format % jamb)
                continue
            cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
            if cert_id not in certs.keys():
                em = 'No Certificate with ID %s \n' % cert_id
                logger.info(em)
                jamb['Error'] = "No Certificate %s" % cert_id
                no_import.append( format % jamb)
                continue
            jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
            cert_doc = certs[cert_id]
            catalog_entry = {}
            catalog_entry['jamb_reg_no'] = jamb_reg_no
            jamb_name = jamb.get(csv_d['jamb_lastname'])
            jamb_name.replace('>','')
            jamb_name.replace('<','')
            names = jamb_name.split()
            letter = names[-1][0].upper()
            sid = self.generateStudentId(letter)
            not_created = True
            while not_created:
                try:
                    students_folder.invokeFactory('Student', sid)
                    not_created = False
                except BadRequest:
                    sid = self.generateStudentId(letter)
            catalog_entry['id'] = sid
            tr_count += 1
            logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s REG-NO %(jamb_reg_no)s ' % vars())
            student = getattr(self,sid)
            student.manage_setLocalRoles(sid, ['Owner',])
            student.invokeFactory('StudentPume','pume')
            dp = {'Title': 'Pume Data'}
            student.invokeFactory('StudentApplication','application')
            da = {'Title': 'Application Data'}
            da["jamb_lastname"] = jamb_name
            da_fields = ('jamb_reg_no',
                         'jamb_sex',
                         'entry_mode',
                         #'jamb_score',
                         'jamb_first_cos',
                         'jamb_sex',
                         'jamb_state',
                         'jamb_lga',
                         'app_email',
                         'app_mobile',
                         )
            for f in da_fields:
                da[f] = jamb.get(csv_d[f])
            catalog_entry['email'] = jamb.get(csv_d['app_email'])
            app = student.application
            app_doc = app.getContent()
            picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
            #import pdb;pdb.set_trace()
            if os.path.exists(picture):
                file = open(picture)
                if False:
                    img = PIL.Image.open(file)
                    img.thumbnail((150,200),
                                  resample=PIL.Image.ANTIALIAS)
                    # We now need a buffer to write to. It can't be the same
                    # as the inbuffer as the PNG writer will write over itself.
                    outfile = StringIO()
                    img.save(outfile, format=img.format)
                else:
                    outfile = file.read()
                app_doc.manage_addFile('passport',
                                       file=outfile,
                                       title="%s.jpg" % jamb_reg_no)
            app.getContent().edit(mapping=da)
            app.manage_setLocalRoles(sid, ['Owner',])
            #wftool.doActionFor(app,'close')
            dp_fields = (
                         #'pume_eng_score',
                         #'pume_gen_score',
                         'pume_tot_score',
                         )
            dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
            pume = student.pume
            pume.getContent().edit(mapping=dp)
            #wftool.doActionFor(pume,'close')
            pume.manage_setLocalRoles(sid, ['Owner',])
            #student.getContent().createSubObjects()
            dp = {}
            if len(names) == 3:
                dp['firstname'] = names[0].capitalize()
                dp['middlename'] = names[1].capitalize()
                dp['lastname'] = names[2].capitalize()
            elif len(names) == 2:
                dp['firstname'] = names[0].capitalize()
                dp['middlename'] = ''
                dp['lastname'] = names[1].capitalize()
            else:
                dp['firstname'] = ''
                dp['middlename'] = ''
                dp['lastname'] = jamb_name
            dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
            catalog_entry['sex'] = dp['sex']
            catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
            student.invokeFactory('StudentPersonal','personal')
            per = student.personal
            per_doc = per.getContent()
            per_doc.edit(mapping = dp)
            per.manage_setLocalRoles(sid, ['Owner',])
            if jamb.get(csv_d['stud_status']) == "Admitted":
                wftool.doActionFor(student,'pume_pass')
                wftool.doActionFor(student,'admit')
            else:
                wftool.doActionFor(student,'pume_fail')
                wftool.doActionFor(student,'reject_admission')
                continue
            #
            # Clearance
            #
            student.invokeFactory('StudentClearance','clearance')
            #wftool.doActionFor(student.clearance,'open')
            dp = {'Title': 'Clearance/Eligibility Record'}
            student.clearance.manage_setLocalRoles(sid, ['Owner',])
            #
            # Study Course
            #
            student.invokeFactory('StudentStudyCourse','study_course')
            study_course = student.study_course
            dsc = {}
            #from_certificate = ['title',
            #                   'max_elect',
            #                   'max_pass',
            #                   'n_core',
            #                   'nr_years',
            #                   'probation_credits',
            #                   'promotion_credits',
            #                   'start_level',
            #                  ]
            #for f in from_certificate:
            #    dsc[f] = getattr(cert_doc,f)
            #dsc['faculty'] = jamb.get(csv_d['faculty'])
            #dsc['department'] = jamb.get(csv_d['department'])
            catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
            catalog_entry['department'] = jamb.get(csv_d['department'])
            catalog_entry['course'] = cert_id
            #catalog_entry['level'] = getattr(cert_doc,'start_level')
            catalog_entry['level'] = '200'
            dsc['study_course'] = cert_id
            dsc['current_level'] = '200'
            #dsc['entry_session'] = jamb.get(csv_d['session'])
            study_course.getContent().edit(mapping=dsc)
            self.students_catalog.addRecord(**catalog_entry)
            if tr_count > 10:
                if len(no_import) > 1:
                    open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                             '\n'.join(no_import)+'\n')
                    no_import = []
                em = '%d transactions commited\n' % tr_count
                transaction.commit()
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                                               '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"createNewStudents")###(
    def createNewStudents(self):
        """load Fulltime Studentdata from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        csv_d = {'jamb_reg_no': "RegNumber",
                 'jamb_lastname': "Name",
                 'session': "Session",
                 'pume_tot_score': "PUME SCORE",
                 'jamb_score': "JambScore",
                 'jamb_sex': "Sex",
                 'jamb_state': "State",
##                 'jamb_first_cos': "AdminCourse",
                 'faculty': "AdminFaculty",
                 'course_code': "AdmitCoscode",
                 'stud_status':"AdmitStatus",
                 'department': "AdmitDept",
                 'jamb_lga': "LGA",
                 'app_email': "email",
                 'app_mobile': "PhoneNumbers",
                 }
        csv_fields = [f[1] for f in csv_d.items()]
        tr_count = 0
        total = 0
        #name = 'pume_results'
        name = 'Admitted'
        no_import = []
        s = ','.join(['"%s"' % fn for fn in csv_fields])
        no_import.append('"Error",%s' % s)
        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
        no_certificate = "no certificate %s" % format
        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
        logger = logging.getLogger('Students.StudentsFolder.createNewStudents')
        logger.info('Start loading from %s.csv' % name)
        l = self.portal_catalog({'meta_type': "Certificate"})
        certs = {}
        cert_docs = {}
        for f in l:
            certs[f.getId] = f.getObject().getContent()
        try:
            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for jamb in result:
            jamb['Error'] = "Processing "
            logger.info(format % jamb)
            jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
            res = self.portal_catalog({'portal_type': "StudentApplication",
                                     'SearchableText': jamb_reg_no })
            if res:
                em = 'Student with RegNo %s already exists\n' % jamb_reg_no
                logger.info(em)
                jamb['Error'] = "Student exists"
                no_import.append(format % jamb)
                continue
            cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
            if cert_id not in certs.keys():
                em = 'No Certificate with ID %s \n' % cert_id
                logger.info(em)
                jamb['Error'] = "No Certificate %s" % cert_id
                no_import.append( format % jamb)
                continue
            res = self.portal_pumeresults(jamb_reg_no = jamb_reg_no)
            if len(res) == 1:
                self.portal_pumeresults.modifyRecord(jamb_reg_no = jamb_reg_no,
                                                     status = jamb.get(csv_d['stud_status']),
                                                     )
            jamb_reg_no =jamb.get(csv_d['jamb_reg_no'])
            cert_doc = certs[cert_id]
            catalog_entry = {}
            catalog_entry['jamb_reg_no'] = jamb_reg_no
            jamb_name = jamb.get(csv_d['jamb_lastname'])
            jamb_name.replace('>','')
            jamb_name.replace('<','')
            names = jamb_name.split()
            letter = names[-1][0].upper()
            sid = self.generateStudentId(letter)
            not_created = True
            while not_created:
                try:
                    students_folder.invokeFactory('Student', sid)
                    not_created = False
                except BadRequest:
                    sid = self.generateStudentId(letter)
            catalog_entry['id'] = sid
            tr_count += 1
            logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
            student = getattr(self,sid)
            student.manage_setLocalRoles(sid, ['Owner',])
            student.invokeFactory('StudentPume','pume')
            dp = {'Title': 'Pume Data'}
            student.invokeFactory('StudentApplication','application')
            da = {'Title': 'Application Data'}
            da["jamb_lastname"] = jamb_name
            da_fields = ('jamb_reg_no',
                         'jamb_sex',
                         #'jamb_state',
                         'jamb_score',
##                         'jamb_first_cos',
                         'jamb_sex',
                         'jamb_state',
                         'jamb_lga',
                         'app_email',
                         'app_mobile',
                         )
            for f in da_fields:
                da[f] = jamb.get(csv_d[f])
            catalog_entry['email'] = jamb.get(csv_d['app_email'])
            app = student.application
            app_doc = app.getContent()
            #import pdb;pdb.set_trace()
            picture ="%s/import/pictures/%s.jpg" % (i_home,picture_id)
            if os.path.exists(picture):
                file = open(picture)
                if False:
                    img = PIL.Image.open(file)
                    img.thumbnail((150,200),
                                  resample=PIL.Image.ANTIALIAS)
                    # We now need a buffer to write to. It can't be the same
                    # as the inbuffer as the PNG writer will write over itself.
                    outfile = StringIO()
                    img.save(outfile, format=img.format)
                else:
                    outfile = file.read()
                app_doc.manage_addFile('passport',
                                       file=outfile,
                                       title="%s.jpg" % jamb_reg_no)
            app.getContent().edit(mapping=da)
            app.manage_setLocalRoles(sid, ['Owner',])
            #wftool.doActionFor(app,'close')
            dp_fields = (
                         #'pume_eng_score',
                         #'pume_gen_score',
                         'pume_tot_score',
                         )
            dp['pume_tot_score'] = jamb.get(csv_d['pume_tot_score']) or "No Option Shaded"
            pume = student.pume
            pume.getContent().edit(mapping=dp)
            #wftool.doActionFor(pume,'close')
            pume.manage_setLocalRoles(sid, ['Owner',])
            #student.getContent().createSubObjects()
            dp = {}
            if len(names) == 3:
                dp['firstname'] = names[0].capitalize()
                dp['middlename'] = names[1].capitalize()
                dp['lastname'] = names[2].capitalize()
            elif len(names) == 2:
                dp['firstname'] = names[0].capitalize()
                dp['middlename'] = ''
                dp['lastname'] = names[1].capitalize()
            else:
                dp['firstname'] = ''
                dp['middlename'] = ''
                dp['lastname'] = jamb_name
            dp['sex'] = jamb.get(csv_d['jamb_sex']) == 'F'
            catalog_entry['sex'] = dp['sex']
            catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
            student.invokeFactory('StudentPersonal','personal')
            per = student.personal
            per_doc = per.getContent()
            per_doc.edit(mapping = dp)
            per.manage_setLocalRoles(sid, ['Owner',])
            if jamb.get(csv_d['stud_status']) == "Admitted":
                wftool.doActionFor(student,'pume_pass')
                wftool.doActionFor(student,'admit')
            else:
                wftool.doActionFor(student,'pume_fail')
                wftool.doActionFor(student,'reject_admission')
                continue
            #
            # Clearance
            #
            student.invokeFactory('StudentClearance','clearance')
            #wftool.doActionFor(student.clearance,'open')
            dp = {'Title': 'Clearance/Eligibility Record'}
            student.clearance.manage_setLocalRoles(sid, ['Owner',])
            #
            # Study Course
            #
            student.invokeFactory('StudentStudyCourse','study_course')
            study_course = student.study_course
            dsc = {}
            #from_certificate = ['title',
            #                   'max_elect',
            #                   'max_pass',
            #                   'n_core',
            #                   'nr_years',
            #                   'probation_credits',
            #                   'promotion_credits',
            #                   'start_level',
            #                  ]
            #for f in from_certificate:
            #    dsc[f] = getattr(cert_doc,f)
            #dsc['faculty'] = jamb.get(csv_d['faculty'])
            #dsc['department'] = jamb.get(csv_d['department'])
            catalog_entry['faculty'] = jamb.get(csv_d['faculty'])
            catalog_entry['department'] = jamb.get(csv_d['department'])
            catalog_entry['course'] = cert_id
            #catalog_entry['level'] = getattr(cert_doc,'start_level')
            catalog_entry['level'] = '100'
            dsc['study_course'] = cert_id
            dsc['entry_level'] = '100'
            #dsc['entry_session'] = jamb.get(csv_d['session'])
            study_course.getContent().edit(mapping=dsc)
            self.students_catalog.addRecord(**catalog_entry)
            if tr_count > 10:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                             '\n'.join(no_import) + "\n")
                    no_import = []
                em = '%d transactions commited\n' % tr_count
                transaction.commit()
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"createStudents")###(
    def createStudents(self):
        """load Studentdata from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        entry_levels = {'ume_ft':'100',
                        'de_ft': '200',
                        'ug_pt': '100',
                        'de_pt': '200',
                        'pg_ft': '700',
                        'pg_pt': '700',
                        'dp_pt': '100',
                        'dp_ft': '100',
                        }
        csv_d = {'jamb_reg_no': "reg_no",
                 'entry_mode': 'entry_mode',
                 'jamb_firstname': "firstname",
                 'jamb_middlename': "middlename",
                 'jamb_lastname': "lastname",
                 'jamb_sex': "sex",
                 'jamb_state': "state",
                 'birthday': "date_of_birth",
                 'app_email': "email",
                 'study_course': "study_course",
                 'perm_address': "address",
                 }
        csv_fields = [f[1] for f in csv_d.items()]
        tr_count = 0
        total = 0
        #name = 'pume_results'
        name = 'Admitted'
        no_import = []
        s = ','.join(['"%s"' % fn for fn in csv_fields])
        no_import.append('"Error",%s' % s)
        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
        no_certificate = "no certificate %s" % format
        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write('\n'.join(no_import))
        logger = logging.getLogger('Students.StudentsFolder.createStudents')
        logger.info('Start loading from %s.csv' % name)
        certs = {}
        try:
            results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for result in results:
            #result['Error'] = "Processing "
            #logger.info(format % result)
            jamb_reg_no = result.get(csv_d['jamb_reg_no'])
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if res:
                em = 'Student with RegNo %s already exists\n' % jamb_reg_no
                logger.info(em)
                result['Error'] = "Student exists"
                no_import.append(format % result)
                continue
            cert_id = makeCertificateCode(result.get(csv_d['study_course']))
            if cert_id not in certs.keys():
                res = self.portal_catalog(meta_type = "Certificate",id = cert_id)
                if not res:
                    em = 'No Certificate with ID %s \n' % cert_id
                    logger.info(em)
                    result['Error'] = "No Certificate %s" % cert_id
                    no_import.append( format % result)
                    continue
                cert = res[0]
                cert_path = cert.getPath().split('/')
                certificate = certs[cert_id] = {'faculty': cert_path[-4],
                                     'department': cert_path[-3]}
            cert_doc = certs[cert_id]
            catalog_entry = {}
            catalog_entry['jamb_reg_no'] = jamb_reg_no
            firstname = result.get(csv_d['jamb_firstname'])
            middlename = result.get(csv_d['jamb_middlename'])
            lastname = result.get(csv_d['jamb_lastname'])
            sid = self.generateStudentId('x')
            students_folder.invokeFactory('Student', sid)
            catalog_entry['id'] = sid
            tr_count += 1
            logger.info('%(total)s+%(tr_count)s: Creating Student with ID %(sid)s reg_no %(jamb_reg_no)s ' % vars())
            student = getattr(self,sid)
            student.manage_setLocalRoles(sid, ['Owner',])
            student.invokeFactory('StudentApplication','application')
            da = {'Title': 'Application Data'}
            da["jamb_firstname"] = firstname
            da["jamb_middlename"] = middlename
            da["jamb_lastname"] = lastname
            catalog_entry['entry_session'] = da["entry_session"] = self.getSessionId()[-2:]
            catalog_entry['sex'] = sex = result.get(csv_d['jamb_sex']).startswith('F')
            da_fields = ('jamb_reg_no',
                         'jamb_sex',
                         'jamb_state',
                         'entry_mode',
                         'app_email',
                         )
            for f in da_fields:
                da[f] = result.get(csv_d[f])
            catalog_entry['email'] = da['app_email']
            catalog_entry['entry_mode'] = da['entry_mode']
            catalog_entry['entry_level'] = da["entry_level"] = entry_levels.get(da['entry_mode'],'100')
            app = student.application
            app_doc = app.getContent()
            app.getContent().edit(mapping=da)
            picture ="%s/import/pictures/%s.jpg" % (i_home,jamb_reg_no)
            app.manage_setLocalRoles(sid, ['Owner',])

            picture_id = da['jamb_reg_no'].replace('/','_')
            file = None
            for ext in ('jpg','JPG'):
                picture ="%s/import/pictures_admitted_latest/%s.%s" % (i_home,picture_id,ext)
                if os.path.exists(picture):
                    file = open(picture)
                    break
            if file is not None:

                ## file conversion does not work
                #img = PIL.Image.open(file)
                #img.thumbnail((150,200),
                #              resample=PIL.Image.ANTIALIAS)
                #outfile = StringIO()
                #img.save(outfile, format=img.format)

                outfile = file.read()
                app_doc.manage_addFile('passport',
                                       file=outfile,
                                       title="%s.jpg" % jamb_reg_no)
            #wftool.doActionFor(app,'close')
            dp = {}
            dp['firstname'] = firstname
            dp['middlename'] = middlename
            dp['lastname'] = lastname
            dp['email'] = da['app_email']
            dp['sex'] = sex
            catalog_entry['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
            student.invokeFactory('StudentPersonal','personal')
            per = student.personal
            per_doc = per.getContent()
            per_doc.edit(mapping = dp)
            per.manage_setLocalRoles(sid, ['Owner',])
            wftool.doActionFor(student,'pume_pass')
            wftool.doActionFor(student,'admit')
            #
            # Clearance
            #
            student.invokeFactory('StudentClearance','clearance')
            #wftool.doActionFor(student.clearance,'open')
            clearance = student.clearance
            dc = {'Title': 'Clearance/Eligibility Record'}
            clearance = student.clearance
            date_str = result.get(csv_d['birthday'])
            try:
                date = DateTime.DateTime(date_str)
            except:
                #import pdb;pdb.set_trace()
                date = None
            dc['birthday'] = date
            clearance.getContent().edit(mapping=dc)
            clearance.manage_setLocalRoles(sid, ['Owner',])
            #
            # Study Course
            #
            student.invokeFactory('StudentStudyCourse','study_course')
            study_course = student.study_course
            dsc = {}
            #catalog_entry['level'] = getattr(cert_doc,'start_level')
            catalog_entry['session'] = dsc['current_session'] = da['entry_session']
            catalog_entry['level'] = dsc['current_level'] = entry_levels.get(da['entry_mode'],'100')
            catalog_entry['mode'] = dsc['current_mode'] = da['entry_mode']
            catalog_entry['course'] = dsc['study_course'] = cert_id
            catalog_entry['faculty'] = certificate['faculty']
            catalog_entry['department'] = certificate['department']
            catalog_entry['verdict'] = dsc['current_verdict'] = 'N/A'
            catalog_entry['review_state'] = self.portal_workflow.getInfoFor(student,'review_state',None)
            study_course.getContent().edit(mapping=dsc)
            #import pdb;pdb.set_trace()
            self.students_catalog.addRecord(**catalog_entry)
            if tr_count > 10:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                             '\n'.join(no_import) + "\n")
                    no_import = []
                em = '%d transactions commited\n' % tr_count
                transaction.commit()
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported.csv" % (i_home,name),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"importReturningStudents")###(
    def importReturningStudents(self):
        """load Returning Studentdata from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        #students_folder = self.portal_url().getPortalObject().campus.students
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'Returning'
        table = self.returning_import
        no_import = []
        imported = []
        logger = logging.getLogger('Students.StudentsFolder.importReturningStudents')
        try:
            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        l = self.portal_catalog({'meta_type': "Certificate"})
        certs = {}
        cert_docs = {}
        for f in l:
            certs[f.getId] = f.getObject().getContent()
        start = True
        res = table()
        regs = []
        if len(res) > 0:
            regs = [s.matric_no for s in res]
        #import pdb;pdb.set_trace()
        for student in returning:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in student.keys()])
                imported.append(s)
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
                format_error = format + ',"%(Error)s"'
                no_certificate = "no certificate %s" % format
            student['matric_no'] = matric_no = student.get('matric_no').upper()
            student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
            student['Permanent_Address'] = perm_address = student.get('Permanent Address')
            if matric_no == '':
                student['Error'] = "Empty matric_no"
                no_import.append( format_error % student)
                continue
            if matric_no in regs or self.returning_import(matric_no = matric_no):
                student['Error'] = "Duplicate"
                no_import.append( format_error % student)
                continue
            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
            if cert_id not in certs.keys():
                student['Error'] = "No Certificate %s" % cert_id
                no_import.append( format_error % student)
                continue
            try:
                table.addRecord(**student)
            except ValueError:
                student['Error'] = "Duplicate"
                no_import.append( format_error % student)
                continue
            regs.append(student.get('matric_no'))
            imported.append(format % student)
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(no_import) + "\n")
                imported = []
                em = '%d transactions commited total %s\n' % (tr_count,total)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported))
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"fixVerdicts")###(
    def fixVerdicts(self,csv_file=None):
        """fix wrong uploaded verdicts"""
        import transaction
        import random
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        tr_count = 1
        total = 0
        if csv_file is None:
            name = 'Verdicts'
        else:
            name = csv_file
        st_cat = self.students_catalog
        no_import = []
        verdicts_voc = self.portal_vocabularies.verdicts
        rverdicts = {}
        for k,v in verdicts_voc.items():
            rverdicts[v.upper()] = k
        rverdicts['STUDENT ON PROBATION'] = 'C'
        logger = logging.getLogger('Students.StudentsFolder.fixVerdicts')
        try:
            verdicts = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        start = True
        #import pdb;pdb.set_trace()
        for verdict in verdicts:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in verdict.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in verdict.keys()])
                format_error = format + ',"%(Error)s"'
            matric_no = verdict.get('MAT NO')
            if not matric_no:
                continue
            matric_no = matric_no.upper()
            if matric_no == '':
                continue
            verdict_code = rverdicts.get(verdict.get('CATEGORY'),None)
            if verdict_code is None:
                continue
            sres = st_cat(matric_no = matric_no)
            if sres:
                student_id = sres[0].id
                student_obj = getattr(students_folder,student_id,None)
                if student_obj:
                    study_course = getattr(student_obj,'study_course',None)
                    if study_course is None:
                        verdict['Error'] = "Student did not yet log in"
                        no_import.append( format_error % verdict)
                        continue
                st_cat.modifyRecord(id = student_id,
                                    verdict=verdict_code)

                dsc = {}
                dsc['current_verdict'] = verdict_code
                study_course.getContent().edit(mapping=dsc)
            else:
                verdict['Error'] = "Not found in students_catalog"
                no_import.append( format_error % verdict)
                continue
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions commited total %s\n' % (tr_count,total)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        total += tr_count
        em = '%d total transactions commited' % (total)
        logger.info(em)
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"fixAllNames")###(
    def fixAllNames(self):
        "fix all students names"
        import transaction
        response = self.REQUEST.RESPONSE
        logger = logging.getLogger('fixAllNames')
        logger.info('Start')
        students = self.portal_catalog(portal_type='Student')
        count = 0
        total = 0
        for student in students:
            scat_res = self.students_catalog(id = student.getId)
            if not scat_res:
                self.students_catalog.addRecord(id = student.getId)
                scat_res = self.students_catalog(id = student.getId)
            student_entry = scat_res[0]
            old_new = self.fixName(student,student_entry)
            count += 1
            response_write(response,'"%d","%s",%s' % (count + total,student_entry.id,old_new))
            if count > 2000:
                transaction.commit()
                logger.info("%d transactions commited" % count)
                total += count
                count = 0
    ###)

    security.declareProtected(ModifyPortalContent,"fixName")###(
    def fixName(self,student_brain, student_entry):
        "fix the name of a student"
        fix = "first"
        if student_entry.get('name_fixed',None) == fix:
            return "Name already fixed"
        student_id = student_entry.id
        new_student = student_entry.jamb_reg_no.startswith('6')
        student_obj = student_brain.getObject()
        personal = getattr(student_obj,'personal',None)
        invalid = ''
        if personal is None:
            return '"%s","Returning","%s","%s"' % (invalid,student_entry.name,"not logged in")
        per_doc = personal.getContent()
        old_first = per_doc.firstname
        old_middle = per_doc.middlename
        old_last = per_doc.lastname
        new_first = ''
        new_middle = ''
        new_last = ''
        if new_student:
            if not old_first and not old_middle and old_last:
                new_names = [n.capitalize() for n in old_last.split()]
                if len(new_names) > 1:
                    old_first = new_names[0]
                    old_last = new_names[-1]
                    old_middle = ' '.join(new_names[1:-1])
                else:
                    old_last = new_names[0]
                    old_first = ''
                    old_middle = ''
            if old_first:
                new_first = old_first
            if old_middle:
                new_middle = old_middle
            if old_last:
                new_last = old_last
            if old_first.find('<') != -1 or\
               old_first.find('>') != -1 or\
               old_middle.find('<') != -1 or\
               old_middle.find('>') != -1 or\
               old_last.find('<') != -1 or\
               old_last.find('>') != -1:
                   invalid = "invalid characters"
        else:
            new_first = old_first
            if new_first.strip() == '-':
                new_first = ''
            new_middle = old_middle
            if new_middle.strip() == '-':
                new_middle = ''
            new_last = old_last
            if new_last.strip() == '-':
                new_last = ''
        name = "%(new_first)s %(new_middle)s %(new_last)s" % vars()
        if new_student:
            text = "New"
        else:
            text = "Returning"
        old_new = '"%s","%s","%s","%s"' % (invalid,text,
                                           student_entry.name,
                                           name)
        if not invalid:
            self.students_catalog.modifyRecord(id = student_id,
                                      name_fixed = fix,
                                      name = name)
            per_doc.edit(mapping = {'firstname' : new_first,
                                'middlename' : new_middle,
                                'lastname' : new_last,
                                })
        return old_new
    ###)

    security.declareProtected(ModifyPortalContent,"fixAllEntryModeForReturning")###(
    def fixAllEntryModeForReturning(self):
        "read all Returning*.csv"
        ipath = "%s/import/" % i_home
        names = os.listdir(ipath)
        for name in names:
            head,tail = os.path.splitext(name)
            if head.startswith('Returning')\
               and tail == '.csv'\
               and name.find('imported') < 0:
                self.fixEntryModeForReturning(csv_file=head)
    ###)

    security.declareProtected(ModifyPortalContent,"fixEntryModeForReturning")###(
    def fixEntryModeForReturning(self,csv_file=None):
        """load Returning Studentdata from CSV values"""
        import transaction
        import random
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        tr_count = 1
        total = 0
        if csv_file is None:
            name = 'Returning'
        else:
            name = csv_file
        table = self.returning_import
        st_cat = self.students_catalog
        no_import = []
        logger = logging.getLogger('Students.StudentsFolder.fixEntryModeForReturning')
        try:
            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        start = True
        for student in returning:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in student.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
                format_error = format + ',"%(Error)s"'
            matric_no = student.get('matric_no')
            if not matric_no:
                continue
            matric_no = matric_no.upper()
            student['matric_no'] = matric_no
            if matric_no == '':
                continue
            if not table(matric_no = matric_no):
                student['Error'] = "Not imported yet"
                no_import.append( format_error % student)
                continue
            student_id = None
            app = None
            per = None
            if st_cat(matric_no = matric_no):
                student_id = st_cat(matric_no = matric_no)[0].id
                student_obj = getattr(students_folder,student_id,None)
                if student_obj:
                    app = getattr(student_obj,'application',None)
                    if app is not None:
                        app_doc = app.getContent()
                    per = getattr(student_obj,'personal',None)
                    if per is not None:
                        per_doc = per.getContent()
            student['Mode_of_Entry'] = entry_mode = student.get('Mode of Entry').upper()
            student['Permanent_Address'] = perm_address = student.get('Permanent Address')
            #import pdb;pdb.set_trace()
            if not entry_mode:
                student['Error'] = "'Mode of Entry' empty"
                no_import.append( format_error % student)
                continue
            try:
                table.modifyRecord(matric_no = matric_no,
                                   Mode_of_Entry = entry_mode,
                                   Permanent_Address = perm_address)
            except KeyError:
                student['Error'] = "Not found in returning_import"
                no_import.append( format_error % student)
                continue
            if student_id is not None:
                try:
                    st_cat.modifyRecord(id = student_id,
                                        entry_mode=entry_mode)
                except KeyError:
                    student['Error'] = "Not found in students_catalog"
                    no_import.append( format_error % student)
                    continue
            if app is not None:
                da = {}
                da['entry_mode'] = entry_mode
                app_doc.edit(mapping=da)
            if per is not None:
                dp = {}
                dp['perm_address'] = perm_address
                per_doc.edit(mapping=dp)
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions commited total %s\n' % (tr_count,total)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"updateReturningStudents")###(
    def updateReturningStudents(self):
        """load and overwrite Returning Student Data from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'Returning_update'
        table = self.returning_import
        no_import = []
        imported = []
        logger = logging.getLogger('Students.StudentsFolder.updateReturningStudents')
        try:
            returning = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        l = self.portal_catalog({'meta_type': "Certificate"})
        certs = {}
        cert_docs = {}
        for f in l:
            certs[f.getId] = f.getObject().getContent()
        start = True
        res = table()
        regs = []
        if len(res) > 0:
            regs = [s.matric_no for s in res]
        for student in returning:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in student.keys()])
                imported.append(s)
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in student.keys()])
                format_error = format + ',"%(Error)s"'
                no_certificate = "no certificate %s" % format
            matric_no = student.get('matric_no').upper()
            student['matric_no'] = matric_no
            if matric_no == '':
                student['Error'] = "Empty matric_no"
                no_import.append( format_error % student)
                continue
#            if matric_no in regs or self.returning_import(matric_no = matric_no):
#                student['Error'] = "Duplicate"
#                no_import.append( format_error % student)
#                continue
#            cert_id = makeCertificateCode(student.get('Coursemajorcode'))
#            if cert_id not in certs.keys():
#                student['Error'] = "No Certificate %s" % cert_id
#                no_import.append( format_error % student)
#                continue
            try:
                table.modifyRecord(**student)
            except KeyError:
                #import pdb;pdb.set_trace()
                student['Error'] = "no Student found to update"
                no_import.append( format_error % student)
                continue
            #s = self.students_catalog(matric_no=matric_no)
            #if s:
            #    level = "%s" % (int(student.get('Level')) + 100)
            #    self.students_catalog.modifyRecord(id = s[0].id,
            #                                           level=level)

            regs.append(student.get('matric_no'))
            imported.append(format % student)
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(no_import) + "\n")
                imported = []
                em = '%d transactions commited total %s\n' % (tr_count,total)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported))
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"importResults")###(
    def importResults(self):
        """load Returning Students Results from CSV"""
        import transaction
        import random
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'Results'
        table = self.results_import
        no_import = []
        imported = []
        logger = logging.getLogger('Students.StudentsFolder.importResults')
        try:
            results = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        l = self.portal_catalog({'meta_type': "Course"})
        courses = [f.getId for f in l]
        start = True
        res = table()
        regs = []
        if len(res) > 0:
            regs = [s.key for s in res]
        no_course = []
        no_course_list = []
        course_count = 0
        for result in results:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in result.keys()])
                imported.append(s)
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in result.keys()])
                format_error = format + ',"%(Error)s"'
                no_certificate = "no certificate %s" % format
            course_id = result.get('CosCode')
            if not course_id:
                course_id = 'N/A'
                result['CosCode'] = course_id
            matric_no = result.get('matric_no').upper()
            result['matric_no'] = matric_no
            key = matric_no+course_id
            if matric_no == '':
                result['Error'] = "Empty matric_no"
                no_import.append( format_error % result)
                continue
            if key in regs or self.results_import(key = key):
                result['Error'] = "Duplicate"
                no_import.append( format_error % result)
                continue
            if course_id not in courses:
                if course_id not in no_course:
                    course_count +=1
                    no_course.append(course_id)
                    no_course_list.append('"%s"' % course_id)
                    #result['Error'] = "No Course"
                    #logger.info(format_error % result)
            regs.append(key)
            imported.append(format % result)
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import)+'\n')
                    no_import = []
                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported) + '\n')
                imported = []
                if no_course_list:
                    open("%s/import/%sno_courses%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(no_course_list) + '\n')
                    no_course_list = []
                em = '%d transactions commited total %s\n courses not found %s' % (tr_count,total,course_count)
                transaction.commit()
                logger.info(em)
                regs = []
                total += tr_count
                tr_count = 0
        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported))
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        if no_course_list:
            open("%s/import/%sno_courses%s.csv" % (i_home,name,current),"a").write(
                                    '\n'.join(no_course_list))
        em = '%d transactions commited total %s\n courses not found %s' % (tr_count,total,course_count)
        logger.info(em)
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"importPreviousSessionStudents")###(
    def importPreviousSessionStudents(self):
        """load and create previous session students from CSV"""
        import transaction
        import random
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'Previous'
        keys = self.portal_schemas.import_student.keys()
        keys += self.portal_schemas.import_student_level_data.keys()
        not_imported = []
        imported = []
        certificates = {}
        logger = logging.getLogger('Students.StudentsFolder.importPreviousSessionStudents')
        try:
            records = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        start = True
        for record in records:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                false_keys = [k for k in record.keys() if k not in keys]
                right_keys = [k for k in record.keys() if k in keys]
                if false_keys:
                    logger.info('Fields %s not in schema' % false_keys)
                s = ','.join(['"%s"' % k for k in right_keys])
                imported.append(s)
                not_imported.append('%s,"error"' % s)
                format = ','.join(['"%%(%s)s"' % k for k in right_keys])
                format_error = format + ',"%(error)s"'
            study_course = makeCertificateCode(record.get('study_course'))
            matric_no = record.get('matric_no')
            student_res = self.students_catalog(matric_no = matric_no)
            if student_res:
                record['error'] = "Student exists"
                not_imported.append(format_error % record)
                continue
            if study_course not in certificates.keys():
                cert_res = self.portal_catalog(portal_type='Certificate',
                                                   id = study_course)
                if not cert_res:
                    record['error'] = "No such studycourse"
                    not_imported.append(format_error % record)
                    continue
                certificates[cert_res[0].id] = cert_res[0]
            sid = self.generateStudentId('x',students_folder)
            students_folder.invokeFactory('Student', sid)
            #from pdb import set_trace;set_trace()
            record['student_id'] = sid
            student = getattr(self,sid)
            student.manage_setLocalRoles(sid, ['Owner',])
            student.invokeFactory('StudentApplication','application')
            app = student.application
            app_doc = app.getContent()
            dict = {'Title': 'Application Data'}
            dict["jamb_lastname"] = "%(firstname)s %(lastname)s %(middlename)s" % record
            r2d = [
                   ('entry_mode','entry_mode'),
                   ('sex','jamb_sex'),
                   ('jamb_score','jamb_score'),
                   ('jamb_reg_no','jamb_reg_no'),
                   ]
            for r,d in r2d:
                dict[d] = record[r]
            app_doc.edit(mapping=dict)
            app.manage_setLocalRoles(sid, ['Owner',])
            wftool.doActionFor(app,'close')
            record['sex'] = record['sex'] == 'F'
            student.invokeFactory('StudentPersonal','personal')
            dict = {}
            r2d = [('firstname','firstname'),
                   ('middlename','middlename'),
                   ('lastname','lastname'),
                   ('sex','sex'),
                   ('email','email'),
                   ('phone','phone'),
                   ('address','perm_address'),
                   ]
            for r,d in r2d:
                dict[d] = record[r]
            per = student.personal
            per_doc = per.getContent()
            per_doc.edit(mapping = dict)
            per.manage_setLocalRoles(sid, ['Owner',])
            #
            # Clearance
            #
            student.invokeFactory('StudentClearance','clearance')
            #wftool.doActionFor(student.clearance,'open')
            clearance = getattr(student,'clearance')
            dict = {'Title': 'Clearance/Eligibility Record'}
            clearance.getContent().edit(mapping = dict)
            student.clearance.manage_setLocalRoles(sid, ['Owner',])
            #
            # Study Course
            #
            student.invokeFactory('StudentStudyCourse','study_course')
            study_course = student.study_course
            dict = {}
            r2d = [('session','current_session'),
                   ('level','current_level'),
                   ('verdict','current_verdict'),
                   ('study_course','study_course'),
                   ]
            for r,d in r2d:
                dict[d] = record[r]
            study_course.getContent().edit(mapping=dict)
            #
            # Study Level
            #
            level = record['level']
            study_course.invokeFactory('StudentStudyLevel',level)
            study_level = getattr(study_course,level)
            dict = {}
            r2d = [('session','session'),
                   ('level','code'),
                   ('verdict','verdict'),
                   ]
            for r,d in r2d:
                dict[d] = record[r]
            study_level.getContent().edit(mapping=dict)
            wftool.doActionFor(student,'return')
            imported.append(format % record)
            tr_count += 1
            record['tr_count'] = tr_count
            record['total'] = total
            logger.info('%(total)s+%(tr_count)s: Creating Student %(student_id)s %(matric_no)s %(jamb_reg_no)s' % record)
            if tr_count > 1000:
                if len(not_imported) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(not_imported)+'\n')
                    not_imported = []
                open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported) + '\n')
                imported = []
                em = '%d transactions commited total %s' % (tr_count,total)
                transaction.commit()
                logger.info(em)
                regs = []
                total += tr_count
                tr_count = 0
        open("%s/import/%simported%s.csv" % (i_home,name,current),"a").write(
                                            '\n'.join(imported))
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(not_imported))
        em = '%d transactions commited total %d' % (tr_count,total)
        logger.info(em)
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"updateStudyCourse")###(
    def updateStudyCourse(self):
        """update StudyCourse from CSV values"""
        import transaction
        import random
        from pdb import set_trace
        wftool = self.portal_workflow
        students_folder = self.portal_catalog({'meta_type': 'StudentsFolder'})[-1].getObject()
        csv_d = {'jamb_reg_no': "RegNumber",
                 'jamb_lastname': "Name",
                 'session': "Session",
                 'pume_tot_score': "PUME SCORE",
                 'jamb_score': "JambScore",
                 'jamb_sex': "Sex",
                 'jamb_state': "State",
##                 'jamb_first_cos': "AdminCourse",
                 'faculty': "AdminFaculty",
                 'course_code': "AdmitCoscode",
                 'stud_status':"AdmitStatus",
                 'department': "AdmitDept",
                 'jamb_lga': "LGA",
                 'app_email': "email",
                 'app_mobile': "PhoneNumbers",
                 }
        csv_fields = [f[1] for f in csv_d.items()]
        tr_count = 0
        total = 0
        #name = 'pume_results'
        name = 'StudyCourseChange'
        no_import = []
        s = ','.join(['"%s"' % fn for fn in csv_fields])
        no_import.append('"Error",%s' % s)
        format = '"%(Error)s",' + ','.join(['"%%(%s)s"' % fn for fn in csv_fields])
        no_certificate = "no certificate %s" % format
        open("%s/import/%s_not_imported.csv" % (i_home,name),"w").write(
                    '\n'.join(no_import))
        logger = logging.getLogger('Students.StudentsFolder.updateStudyCourse')
        logger.info('Start loading from %s.csv' % name)
        l = self.portal_catalog({'meta_type': "Certificate"})
        try:
            result = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for jamb in result:
            jamb['Error'] = "Processing "
            logger.info(format % jamb)
            jamb_reg_no = jamb.get(csv_d['jamb_reg_no'])
            res = self.portal_catalog({'portal_type': "StudentApplication",
                                     'SearchableText': jamb_reg_no })
            if not res:
                em = 'Student with jamb_reg_no %s does not exists\n' % jamb_reg_no
                logger.info(em)
                jamb['Error'] = "Student does not exist"
                no_import.append(format % jamb)
                continue
            sid = res[0].getPath().split('/')[-2]
            cert_id = makeCertificateCode(jamb.get(csv_d['course_code']))
            res = self.portal_catalog(portal_type = "Certificate", id = cert_id)
            if not res:
                em = 'No Certificate with ID %s \n' % cert_id
                logger.info(em)
                jamb['Error'] = "No Certificate %s" % cert_id
                no_import.append( format % jamb)
                continue
            cert_brain = res[0]
            catalog_entry = {}
            student = getattr(self,sid)
            #
            # Study Course
            #
            study_course = student.study_course
            dsc = {}
            cert_pl = cert_brain.getPath().split('/')
            catalog_entry['id'] = sid
            catalog_entry['faculty'] = cert_pl[-4]
            catalog_entry['department'] = cert_pl[-3]
            catalog_entry['course'] = cert_id
            dsc['study_course'] = cert_id
            study_course.getContent().edit(mapping=dsc)
            self.students_catalog.modifyRecord(**catalog_entry)
            if tr_count > 10:
                if len(no_import) > 1:
                    open("%s/import/%s_not_imported.csv" % (i_home,name),"w+").write(
                             '\n'.join(no_import))
                    no_import = []
                em = '%d transactions commited\n' % tr_count
                transaction.commit()
                logger.info(em)
                total += tr_count
                tr_count = 0
            tr_count += 1
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(View,"fixOwnership") ###(
    def fixOwnership(self):
        """fix Ownership"""
        for s in self.portal_catalog(meta_type = 'Student'):
            student = s.getObject()
            sid = s.getId
            import pdb;pdb.set_trace()
            student.application.manage_setLocalRoles(sid, ['Owner',])
            student.personal.manage_setLocalRoles(sid, ['Owner',])
    ###)

    security.declareProtected(View,"Title") ###(
    def Title(self):
        """compose title"""
        return "Student Section"
    ###)

    def generateStudentId(self,letter,students = None): ###(
        import random
        r = random
        if students is None:
            students = self.portal_url.getPortalObject().campus.students
        if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
        sid = "%c%d" % (letter,r.randint(99999,1000000))
        while hasattr(students, sid):
            sid = "%c%d" % (letter,r.randint(99999,1000000))
        return sid
        #return "%c%d" % (r.choice('ABCDEFGHKLMNPQRSTUVWXY'),r.randint(99999,1000000))
    ###)

InitializeClass(StudentsFolder)

def addStudentsFolder(container, id, REQUEST=None, **kw): ###(
    """Add a Student."""
    ob = StudentsFolder(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)
    ###)

###)

class Student(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'Student'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        reg_nr = self.getId()[1:]
        data = getattr(self,'personal',None)
        if data:
            content = data.getContent()
            return "%s %s %s" % (content.firstname,content.middlename,content.lastname)
        data = getattr(self,'application',None)
        if data:
            content = data.getContent()
            return "%s" % (content.jamb_lastname)
        return self.title

    security.declarePrivate('makeStudentMember') ###(
    def makeStudentMember(self,sid,password='uNsEt'):
        """make the student a member"""
        membership = self.portal_membership
        membership.addMember(sid,
                             password ,
                             roles=('Member',
                                     'Student',
                                     ),
                             domains='',
                             properties = {'memberareaCreationFlag': False,
                                           'homeless': True},)
        member = membership.getMemberById(sid)
        self.portal_registration.afterAdd(member, sid, password, None)
        self.manage_setLocalRoles(sid, ['Owner',])

###)

    security.declareProtected(View,'createSubObjects') ###(
    def createSubObjects(self):
        """make the student a member"""
        dp = {'Title': 'Personal Data'}
        app_doc = self.application.getContent()
        names = app_doc.jamb_lastname.split()
        if len(names) == 3:
            dp['firstname'] = names[0].capitalize()
            dp['middlename'] = names[1].capitalize()
            dp['lastname'] = names[2].capitalize()
        elif len(names) == 2:
            dp['firstname'] = names[0].capitalize()
            dp['lastname'] = names[1].capitalize()
        else:
            dp['lastname'] = app_doc.jamb_lastname
        dp['sex'] = app_doc.jamb_sex == 'F'
        dp['lga'] = "%s/%s" % (app_doc.jamb_state,app_doc.jamb_lga )
        proxy = self.aq_parent
        proxy.invokeFactory('StudentPersonal','personal')
        per = proxy.personal
        per_doc = per.getContent()
        per_doc.edit(mapping = dp)
        per.manage_setLocalRoles(proxy.getId(), ['Owner',])
        #self.portal_workflow.doActionFor(per,'open',dest_container=per)

###)

InitializeClass(Student)

def addStudent(container, id, REQUEST=None, **kw):
    """Add a Student."""
    ob = Student(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentAccommodation(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentAccommodation'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        content = self.getContent()
        #return "Accommodation Data for %s %s" % (content.firstname,content.lastname)
        return "Accommodation Data for Session %s" % content.session


InitializeClass(StudentAccommodation)

def addStudentAccommodation(container, id, REQUEST=None, **kw):
    """Add a Students personal data."""
    ob = StudentAccommodation(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentPersonal(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentPersonal'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        content = self.getContent()
        #return "Personal Data for %s %s" % (content.firstname,content.lastname)
        return "Personal Data"


InitializeClass(StudentPersonal)

def addStudentPersonal(container, id, REQUEST=None, **kw):
    """Add a Students personal data."""
    ob = StudentPersonal(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentClearance(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentClearance'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        content = self.getContent()
        #return "Clearance/Eligibility Record for %s %s" % (content.firstname,content.lastname)
        return "Clearance/Eligibility Record"


InitializeClass(StudentClearance)

def addStudentClearance(container, id, REQUEST=None, **kw):
    """Add a Students personal data."""
    ob = StudentClearance(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentStudyLevel(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentStudyLevel'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        return "Level %s" % self.aq_parent.getId()

    def create_course_results(self,cert_id,current_level): ###(
        "create all courses in a level"
        aq_portal = self.portal_catalog.evalAdvancedQuery
        res = self.portal_catalog(portal_type="Certificate", id = cert_id)
        l = []
        import transaction
        if res:
            cert = res[0]
            path = cert.getPath()
            query = Eq("path","%s/%s" % (path,current_level)) &\
                    Eq('portal_type','CertificateCourse')
            courses = aq_portal(query)
            #from pdb import set_trace;set_trace()
            self_proxy = self.aq_parent
            for c in courses:
                d = self.getCourseInfo(c.getId)
                cr_id = self_proxy.invokeFactory('StudentCourseResult',c.getId)
                course_result = getattr(self_proxy,cr_id)
                self.portal_workflow.doActionFor(course_result,'open')
                d['core_or_elective'] = getattr(c.getObject().getContent(),'core_or_elective')
                course_result.getContent().edit(mapping=d)
                transaction.commit()
    ###)

InitializeClass(StudentStudyLevel)

def addStudentStudyLevel(container, id, REQUEST=None, **kw):
    """Add a Students personal data."""
    ob = StudentStudyLevel(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentStudyCourse(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentStudyCourse'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        content = self.getContent()
        return "Study Course"


InitializeClass(StudentStudyCourse)

def addStudentStudyCourse(container, id, REQUEST=None, **kw):
    """Add a Students personal data."""
    ob = StudentStudyCourse(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)

###)

class StudentApplication(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentApplication'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        return "Application Data"


InitializeClass(StudentApplication)

def addStudentApplication(container, id, REQUEST=None, **kw):
    """Add a Students eligibility data."""
    ob = StudentApplication(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)
###)

class StudentPume(CPSDocument): ###(
    """
    WAeUP Student container for the various student data
    """
    meta_type = 'StudentPume'
    portal_type = meta_type
    security = ClassSecurityInfo()

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        return "PUME Results"


InitializeClass(StudentPume)

def addStudentPume(container, id, REQUEST=None, **kw):
    """Add a Students PUME data."""
    ob = StudentPume(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)
###)

##class StudentSemester(CPSDocument): ###(
##    """
##    WAeUP StudentSemester containing the courses and students
##    """
##    meta_type = 'StudentSemester'
##    portal_type = meta_type
##    security = ClassSecurityInfo()
##
##InitializeClass(StudentSemester)
##
##def addStudentSemester(container, id, REQUEST=None, **kw):
##    """Add a StudentSemester."""
##    ob = StudentSemester(id, **kw)
##    return CPSBase_adder(container, ob, REQUEST=REQUEST)
##
#####)

##class Semester(CPSDocument): ###(
##    """
##    WAeUP Semester containing the courses and students
##    """
##    meta_type = 'Semester'
##    portal_type = meta_type
##    security = ClassSecurityInfo()
##
##InitializeClass(Semester)
##
##def addSemester(container, id, REQUEST=None, **kw):
##    """Add a Semester."""
##    ob = Semester(id, **kw)
##    return CPSBase_adder(container, ob, REQUEST=REQUEST)
##
#####)

class StudentCourseResult(CPSDocument): ###(
    """
    WAeUP StudentCourseResult
    """
    meta_type = 'StudentCourseResult'
    portal_type = meta_type
    security = ClassSecurityInfo()

    def getCourseEntry(self,cid):
        res = self.portal_catalog({'meta_type': "Course",
                                           'id': cid})
        if res:
            return res[-1]
        else:
            return None

    security.declareProtected(View,"Title")
    def Title(self):
        """compose title"""
        cid = self.aq_parent.getId()
        ce = self.getCourseEntry(cid)
        if ce:
            return "%s" % ce.Title
        return "No course with id %s" % cid

InitializeClass(StudentCourseResult)

def addStudentCourseResult(container, id, REQUEST=None, **kw):
    """Add a StudentCourseResult."""
    ob = StudentCourseResult(id, **kw)
    return CPSBase_adder(container, ob, REQUEST=REQUEST)
###)

# Backward Compatibility StudyLevel

from Products.WAeUP_SRP.Academics import StudyLevel

from Products.WAeUP_SRP.Academics import addStudyLevel