#-*- mode: python; mode: fold -*-
# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
# Author: Joachim Schmitz (js@aixtraware.de)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Id: WAeUPTool.py 1846 2007-06-02 16:16:22Z henrik $
"""The WAeUP Tool Box.
"""

from AccessControl import ClassSecurityInfo
from Acquisition import aq_inner
from Acquisition import aq_parent
from Globals import DTMLFile
from Globals import InitializeClass
from OFS.SimpleItem import SimpleItem

from Products.CPSSchemas.DataStructure import DataStructure
from Products.CPSSchemas.DataModel import DataModel
from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
from Products.CMFCore.ActionProviderBase import ActionProviderBase
from Products.CMFCore.permissions import View
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFCore.utils import UniqueObject
from Products.CMFCore.URLTool import URLTool
from Products.CMFCore.utils import getToolByName
from Students import makeCertificateCode
from Globals import package_home,INSTANCE_HOME
p_home = package_home(globals())
i_home = INSTANCE_HOME
import DateTime
import logging
import transaction
import csv,re,os
from Products.AdvancedQuery import Eq, Between, Le,In

def getObject(object,name):
    if object.hasObject(name):
        return getattr(object,name)
    return None

class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
    """WAeUP tool"""

    id = 'waeup_tool'
    meta_type = 'WAeUP Tool'
    _actions = ()

    security = ClassSecurityInfo()
    security.declareObjectProtected(View)

    manage_options = ( ActionProviderBase.manage_options
                     + SimpleItem.manage_options
                     )

    def rwrite(self,s): ###(
        response = self.REQUEST.RESPONSE
        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
        response.write("%s<br />\r\n" % s)
    ###)

    def sleep(self,secs): ###(
        "sleep"
        import time
        time.sleep(secs)
        return

###)

    security.declareProtected(ModifyPortalContent,'openLog') ###(
    def openLog(self,name):
        """open a log file"""
        version = 1
        path = "%s/log/%s_%d.log" % (i_home,name,version)
        while os.path.exists(path):
            version += 1
            path = "%s/log/%s_%d.log" % (i_home,name,version)
        log = open(path,"w")
        return log

###)

    security.declareProtected(ModifyPortalContent,'writeLog') ###(
    def writeLog(self,logfile,s):
        """write to the log file"""
        logfile.write(s)

###)

    def generateStudentId(self,letter): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if letter == '?':
            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
        sid = "%c%d" % (letter,r.randint(99999,1000000))
        students = self.portal_url.getPortalObject().campus.students
##        while hasattr(students, sid):
##            sid = "%c%d" % (letter,r.randint(99999,1000000))
        while self.students_catalog(id = sid):
            sid = "%c%d" % (letter,r.randint(99999,1000000))
        return sid
    ###)

    def generatePassword(self,s=None): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if s is None:
            s = 'abcdefghklmnpqrstuvwxy23456789'
        pw = ''
        while len(pw) < 6:
            pw += r.choice(s)
        return pw
    ###)

    security.declareProtected(ModifyPortalContent,'getCredential') ###(
    def getCredential(self,student_id):
        "return a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return None
        return getattr(student_entry,"password","not set")
    ###)

    security.declarePublic('checkPassword') ###(
    def checkPassword(self,student_id,password):
        "return a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return False
        return getattr(student_entry,"password","not set") == password
    ###)

    security.declarePublic('editPassword') ###(
    def editPassword(self,student_id,password):
        "edit a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return
        setattr(student_entry,'password',password)
    ###)

    security.declareProtected(View,'doCommit') ###(
    def doCommit(self,logger=None):
        "commit some transactions"
        transaction.commit()
    ###)

    security.declarePublic('loadStudentFoto') ###(
    def loadStudentFoto(self,student,filename,folder):
        "return a student passport picture"
        app_doc = student.application.getContent()
        #clear = student.clearance
        #clear_doc = clear.getContent()
        #matric_no = clear_doc.matric_no.upper()
        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
        #import pdb;pdb.set_trace()
        if os.path.exists(picture1):
            file = open(picture1)
        elif os.path.exists(picture2):
            file = open(picture2)
        else:
            return "passport picture not found %s" % picture1

        outfile = file.read()
        app_doc.manage_addFile('passport',
                               file=outfile,
                               title="%s.jpg" % filename)
        return "successfully loaded passport picture"
    ###)

    security.declareProtected(ModifyPortalContent,'createOne') ###(
    def createOne(self,students_folder,student_brain,letter,commit=False):
        sid = self.waeup_tool.generateStudentId(letter)
        students_folder.invokeFactory('Student', sid)
        student = getattr(students_folder,sid)
        self.portal_workflow.doActionFor(student,'return')
        student.manage_setLocalRoles(sid, ['Owner',])
        matric_no = student_brain.matric_no
        jamb_reg_no = student_brain.Entryregno
        self.students_catalog.addRecord(id = sid,
                                           matric_no = matric_no,
                                           jamb_reg_no = jamb_reg_no,
                                           sex = student_brain.Sex == "F",
                                           name = "%s %s %s" % (student_brain.Firstname,
                                                                student_brain.Middlename,
                                                                student_brain.Lastname)
                                        )
        if commit:
            transaction.commit()
        return sid,jamb_reg_no
    ###)

    security.declareProtected(ModifyPortalContent,'addStudent') ###(
    def addStudent(self,dict):
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        password = self.generatePassword()
        self.makeStudentMember(sid,password)
        f2t = self.student_field2types
        #from pdb import set_trace; set_trace()
        d = {}
        d['jamb_sex']  = 'M'
        if dict.get('sex'):
            d['jamb_sex']  = 'F'
        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = dict.get(field,'')
            sub_doc.edit(mapping = d)
            new_state = f2t[pt]['wf_transition']
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        entry_session = dict.get('entry_session')
        wfaction = 'return'
        if entry_session == self.getSessionId():
            wfaction = 'admit'
        self.portal_workflow.doActionFor(student_obj,wfaction)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,password
    ###)

    security.declarePublic('getCertificateBrain') ###(
    def getCertificateBrain(self,cert_id):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog,
                                {'portal_type':"Certificate",
                                      'id': cert_id})
        if res:
            return res[0]
        return None
    ###)

    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
    def get_csv_filenames(self):
        "do it"
        files = [file for file in os.listdir("%s/import/" % (i_home))
                 if file.endswith('.csv') and file.find('imported') == -1]
        return files
    ###)

    security.declarePublic('findStudentByMatricelNo') ###(
    def findStudentByMatricelNo(self,matric_no):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog,
                                {'portal_type':"StudentClearance",
                                 'SearchableText': matric_no})
        if res:
            return res[0]
        return None
    ###)

    security.declarePublic('makeStudentMember') ###(
    def makeStudentMember(self,sid,password='uNsEt'):
        """make the student a member"""
        membership = self.portal_membership
        membership.addMember(sid,
                             password ,
                             roles=('Member',
                                     'Student',
                                     ),
                             domains='',
                             properties = {'memberareaCreationFlag': False,
                                           'homeless': True},)
        member = membership.getMemberById(sid)
        self.portal_registration.afterAdd(member, sid, password, None)
        #self.manage_setLocalRoles(sid, ['Owner',])
    ###)

    security.declarePublic('makeStudentData') ###(
    def makeStudentData(self,student_id,email=None,phone_nr=None):
        "create Datastructure for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentData')
        students_folder = self.portal_url.getPortalObject().campus.students
        #res = self.students_catalog(id=student_id)
        #if res:
        #    st = res[0]
        #res = self.returning_import(matric_no = st.matric_no)
        res = self.returning_import(id = student_id)
        if res:
            student = res[0]
        else:
            logger.info('Id %s not found in returning_import' % student_id)
            return
        logger.info('%s creates data structure' % student_id)
        s_results = self.results_import(matric_no = student.matric_no)
        if s_results:
            lnr = self.getLevelFromResultsCosCode(s_results)
            level = "%d00" % lnr
            verdict,eligible = self.getVerdict(s_results[0].Verdict)
            if eligible:
                level = "%d00" % (lnr + 1)
        else:
            logger.info('matric_no %s not found in results_import' % student.matric_no)
            level = 0
            verdict = 'N/A'
        #student should not be allowed to perform this transition
        #wftool = self.portal_workflow
        #wftool.doActionFor(student,'return')
        certcode_org = student.Coursemajorcode
        certcode = makeCertificateCode(certcode_org)
        certificate_brain = self.getCertificateBrain(certcode)
        if not certificate_brain:
            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
            logger.info(em)
        matric_no = student.matric_no
        sid = student_id
        student_obj = getattr(students_folder,sid)
        student_obj.invokeFactory('StudentApplication','application')
        application = student_obj.application
        self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        student_obj.invokeFactory('StudentPersonal','personal')
        da['jamb_reg_no'] = student.Entryregno
        em = self.getEntryMode(student.Entryregno)
##        em = student.Mode_of_Entry
##        if em in ('DIRECT', 'DIRECT ENTRY',):
##            em = 'DE'
##        elif em in ('U.M.E', 'UNE',):
##            em = 'UME'
##        elif not em:
##            em = "unknown"
        da['entry_mode'] = em
        personal = student_obj.personal
        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        student_obj.invokeFactory('StudentClearance','clearance')
        clearance = student_obj.clearance
        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['matric_no'] = matric_no
        state = student.State
        lga = student.LGA
        if state and lga:
            lga =  state + ' / ' + lga
        else:
            lga = "None"
        da['jamb_lga'] = dc['lga'] = lga
        da['app_email'] = dp['email'] = email
        da['app_mobile'] = dp['phone'] = phone_nr
        dp['firstname'] = student.Firstname
        dp['middlename'] = student.Middlename
        dp['lastname'] = student.Lastname
        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
        da['jamb_sex'] = student.Sex
        dp['sex'] = student.Sex == 'F'
        dp['perm_address'] = student.Permanent_Address
        application.getContent().edit(mapping=da)
        self.portal_workflow.doActionFor(application,'close',dest_container=application)
        personal.getContent().edit(mapping=dp)
        clearance.getContent().edit(mapping=dc)
        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
##        catd = {}
##        catd['id'] = sid
##        catd['entry_mode']= da['entry_mode']
##        catd['matric_no'] = matric_no
##        catd['jamb_reg_no'] = da['jamb_reg_no']
##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
##        catd['sex'] = dp['sex']
##        catd['level'] = level
##        catd['verdict'] = verdict
##        if certificate_brain:
##            cpath = certificate_brain.getPath().split('/')
##            catd['faculty'] = cpath[-4]
##            catd['department'] = cpath[-3]
##            catd['course'] = certcode
##        self.students_catalog.modifyRecord(**catd)
        #
        # Study Course
        #
        student_obj.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_obj.study_course
        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = certcode
        dsc['current_level'] = level
        dsc['current_verdict'] = verdict
        dsc['current_mode'] = em
        dsc['current_session'] = '06'
        studycourse.getContent().edit(mapping=dsc)
        #
        # Level
        #
##        l = getattr(studycourse,level,None)
##        if l is None:
##            studycourse.invokeFactory('StudentStudyLevel', level)
##            l = getattr(studycourse, level)
##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
##            l.getContent().edit(mapping={'Title': "Level %s" % level})
###)

    security.declarePublic('makeStudentLevel') ###(
    def makeStudentLevel(self,student_id):
        "create the StudyLevel for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
        students_folder = self.portal_url.getPortalObject().campus.students
        res = self.students_catalog(id=student_id)
        if res:
            st = res[0]
        course = st.course
        matric_no = st.matric_no
        level = st.level
        res = self.results_import(matric_no = matric_no)
        if res:
            results = res
        logger.info('%s creating Level %s' % (student_id,level))
        #
        # Level
        #
        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        studycourse = getattr(student_obj,"study_course",None)
        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
        l = getattr(studycourse,level,None)
        if l is None:
            studycourse.invokeFactory('StudentStudyLevel', level)
            l = getattr(studycourse, level)
            self.portal_workflow.doActionFor(l,'open',dest_container=l)
            l.getContent().edit(mapping={'Title': "Level %s" % level})
        ###)

    security.declarePublic('getAccommodationInfo') ###(
    def getAccommodationInfo(self,bed):
        """return Accommodation Info"""
        info = {}
        hall,block,room,letter = bed.split('_')
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
        if res and len(res) == 1:
            hall_brain = res[0]
            hall_doc = hall_brain.getObject().getContent()
        else:
            return info
        info['hall_title'] = hall_brain.Title
        info['maintenance_code'] = hall_doc.maintenance_code
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
        batch_doc = None
        for brain in res:
            if brain.id.startswith(info['maintenance_code']):
                batch_doc = brain.getObject().getContent()
                break
        if batch_doc is None:
            info['maintenance_fee'] = None
        else:
            info['maintenance_fee'] = batch_doc.cost
        return info
    ###)

    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
    def deleteAllCourses(self,department="All"):
        ''' delete the courses'''
        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        if str(member) not in ("henrik","joachim"):
            return "not possible"
        if department == "All":
            res = self.portal_catalog({'meta_type': 'Department'})
        if len(res) < 1:
            return "No Departments found"

        deleted = []
        for dep in res:
            cf = dep.getObject().courses
            if cf:
                cf.manage_delObjects(ids=cf.objectIds())
                deleted.append("deleted Courses in %s" % dep.getId)
        return "\r".join(deleted)
    ###)

    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
    def getLogfileLines(self,filename="event.log",numlines=20):
        """Get last NUMLINES lines of logfile FILENAME.

        Return last lines' of a file in the instances logfile directory as
        a list. The number of returned lines equals `numlines' or less. If
        less than `numlines' lines are available, the whole file ist
        returned. If the file can not be opened or some other error
        occurs, empty list is returend.
        """
        result = []
        lines_hit = 0

        # We only handle files in instances' log directory...
        logpath = os.path.join(i_home, "log")
        filename = str(os.path.abspath( os.path.join( logpath, filename )))
        if not filename.startswith( logpath ):
            # Attempt to access file outside log-dir...
            return []

        try:
            fd = file( filename, "rb" )
        except IOError:
            return []
        if not fd:
            return []

        if os.linesep == None:
            linesep = '\n'
        else:
            linesep = os.linesep

        # Try to find 'numlines' times a lineseparator, searching from end
        # and moving to the beginning of file...
        fd.seek( 0, 2) # Move to end of file...
        while lines_hit < numlines:
            if fd.read(1) == linesep[-1]: # This moves filedescriptor
                                          # one step forward...
                lines_hit += 1
            try:
                fd.seek( -2, 1) # Go two bytes back from current pos...
            except IOError:
                # We cannot go back two bytes. Maybe the file is too small...
                break
        fd.seek(2,1)

        # Read all lines from current position...
        result = fd.readlines()
        # Remove line endings...
        result = [x.strip() for x in result]
        fd.close()
        return result
    ###)

    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
    def getCallbacksFromLog(self,filename):
        """fix Online Payment Transactions from Z2.log entries"""
        import transaction
        import random
        from cgi import parse_qs
        from urlparse import urlparse
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
        data = re.compile(s)
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        #name = 'epaymentsuccessful_z2log2'
        name = filename
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
        try:
            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
        except:
            logger.error('Error reading %s' % name)
            return
        tas = []
        for line in transactions:
            dict = {}
            items = data.search(line)
            dict['idict'] = idict = items.groupdict()
            #print idict
            #from pdb import set_trace;set_trace()
            urlparsed = urlparse(idict['get'][4:])
            #print urlparsed
            path = urlparsed[2].split('/')
            dict['student_id'] = student_id = path[8]
            dict['payment_id'] = payment_id = path[10]
            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
            tas.append(dict)
            tr_count += 1
        return tas
    ###)

    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
    def importOnlinePaymentTransactions(self):
        """load Online Payment Transactions from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        opt = self.online_payments_import
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'OnlineTransactions'
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
        try:
            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for pay_transaction in transactions:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
                format_error = format + ',"%(Error)s"'
            data = {}

            # format of the first file sent by Tayo
            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = pay_transaction['Resp Code']
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the second file sent by Tayo
            #data['datetime'] = date = 0
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = '00'
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the third file sent by Kehinde
            data['datetime'] = date = 0
            data['student_id'] = student_id = pay_transaction['customer_id']
            data['order_id'] = order_id = pay_transaction['merchant_reference']
            data['response_code'] = response_code = '00'
            data['amount'] = amount = pay_transaction['Amount']

            dup = False
            if response_code == "12":
                continue
            try:
                opt.addRecord(**data)
            except ValueError:
                dup = True
            #from pdb import set_trace;set_trace()
            if dup:
                if response_code == "00":
                    try:
                        opt.modifyRecord(**data)
                    except:
                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
                        continue
                else:
                    pay_transaction['Error'] = "Duplicate order_id"
                    no_import.append( format_error % pay_transaction)
                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
                    continue
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions committed\n' % (tr_count)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)


    # all workflow transitions refer to students in state returning
    student_field2types = {   ###(
                      'StudentApplication':
                          {'id': 'application',
                           'title': 'Application Data',
                           'wf_transition': 'close',
                           'fields':
                             ('jamb_reg_no',
                              'entry_mode',
                              'entry_session',
                              'jamb_score',
                              'app_email',
                              )
                              },
                      #'StudentPume':
                      #    {'id': 'pume',
                      #     'title': 'Pume Data',
                      #     'wf_transition': 'close',
                      #     'fields':
                      #       ('pume_score',
                      #        )
                      #        },
                      'StudentClearance':
                          {'id': 'clearance',
                           'title': 'Clearance Data',
                           'wf_transition': 'close',
                           'fields':
                             ('matric_no',
                              'nationality',
                              'lga',
                              'birthday',
                              )
                              },
                         'StudentPersonal':
                          {'id': 'personal',
                           'title': 'Personal Data',
                           'wf_transition': 'open',
                           'fields':
                             ('firstname',
                              'middlename',
                              'lastname',
                              'sex',
                              'email',
                              'phone',
                              'perm_address',
                              )
                              },
                         'StudentStudyCourse':
                          {'id': 'study_course',
                           'title': 'Study Course',
                           'wf_transition': 'open',
                           'fields':
                             ('study_course',
                              'current_level',
                              'current_session',
                              'current_mode',
                              'current_verdict',
                              )
                              },
                         'PaymentsFolder':
                          {'id': 'payments',
                           'title': 'Payments',
                           'wf_transition': 'open',
                           'fields':
                             ()
                              },
                         }
    ###)


    security.declareProtected(ModifyPortalContent,'importStudent') ###(
    def importStudent(self,mapping):
        "create a students record due import"
        logger = logging.getLogger('WAeUPTool.importStudent')
        students_folder = self.portal_url.getPortalObject().campus.students
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        if jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if res:
                return '',"jamb_reg_no exists"
        matric_no = mapping.get('matric_no',None)
        if matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if res:
                return '',"matric_no exists"
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.student_field2types
        d = {}
        d['jamb_sex']  = 'M'
        if mapping.get('sex'):
            d['jamb_sex']  = 'F'
        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = mapping.get(field,'')
            if pt == "StudyCourse":
                for von,zu in (('entry_mode','current_mode'),
                               ('entry_session','current_session')):
                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
                        d[zu] = mapping[von]
            sub_doc.edit(mapping = d)
            new_state = f2t[pt]['wf_transition']
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        wfaction = 'return'
        self.portal_workflow.doActionFor(student_obj,wfaction)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,''
    ###)

    security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
    def importEditStudent(self,mapping):
        "edit a students record due import"
        logger = logging.getLogger('WAeUPTool.importEditStudent')
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = mapping.get('id',None)
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        matric_no = mapping.get('matric_no',None)
        if sid:
            res = self.students_catalog(id = sid)
            if not res:
                return '',"no student with id %s" % sid
            elif matric_no and res[0].matric_no and\
              matric_no != res[0].matric_no:
                return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
            elif jamb_reg_no and res[0].jamb_reg_no and\
              jamb_reg_no != res[0].jamb_reg_no:
                return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
        elif jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if not res:
                return '',"no student with jamb_reg_no %s" % jamb_reg_no
        elif matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if not res:
                return '',"no student with matric_no %s" % matric_no
            #elif jamb_reg_no and res[0].jamb_reg_no and\
            #  jamb_reg_no != res[0].jamb_reg_no:
            #    return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no

        sid = res[0].id
        student_obj = getattr(students_folder,sid)
        f2t = self.student_field2types
        d = {}
        d['jamb_sex']  = 'M'
        if mapping.get('sex'):
            d['jamb_sex']  = 'F'
        for pt in f2t.keys():
            sub_obj = getattr(student_obj,f2t[pt]['id'],None)
            if sub_obj is None:
                try:
                    student_obj.invokeFactory(pt,f2t[pt]['id'])
                except:
                    continue
                sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            # self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                if not mapping.get(field,None):
                    continue
                d[field] = mapping.get(field,'')
            sub_doc.edit(mapping = d)
            # new_state = f2t[pt]['wf_transition']
            # if new_state != "remain":
            #     self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        # wfaction = 'return'
        # self.portal_workflow.doActionFor(student_obj,wfaction)
        # student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,''
    ###)

    security.declareProtected(ModifyPortalContent,"importData")###(
    def importData(self,filename,name,edit=False):
        """load data from CSV values"""
        import transaction
        import random

        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        logger = logging.getLogger('WAeUPTool.importData')
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total_imported = 0
        total_not_imported = 0
        total = 0
        iname = "import_%s" % name
        stool = getToolByName(self, 'portal_schemas')
        ltool = getToolByName(self, 'portal_layouts')
        schema = stool._getOb(iname)
        if schema is None:
            em = 'No such schema %s' % iname
            logger.error('No such schema %s' % iname)
            return em
        layout = ltool._getOb(iname)
        if layout is None:
            em = 'No such layout %s' % iname
            logger.error(em)
            return em
        validators = {}
        for widget in layout.keys():
            validators[widget] = layout[widget].validate
        if edit:
            importer_name = "importEdit%s" % name.capitalize()
        else:
            importer_name = "import%s" % name.capitalize()
        importer = getattr(self, '%s' % importer_name,None)
        if importer is None:
            em = 'No importer function %s' % importer_name
            logger.error(em)
            return em
        not_imported = []
        imported = []
        try:
            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
        except:
            em = 'Error reading %s.csv' % filename
            logger.error(em)
            return em
        for item in items:
            tr_count += 1
            if start:
                start = False
                adapters = [MappingStorageAdapter(schema, item)]
                dm = DataModel(item, adapters,context=self)
                logger.info('%s starts import from %s.csv' % (member,filename))
                import_keys = [k for k in item.keys() if not k.startswith('ignore')]
                diff2schema = set(import_keys).difference(set(schema.keys()))
                diff2layout = set(import_keys).difference(set(layout.keys()))
                if diff2schema:
                    em = "not ignorable key(s) %s found in heading" % diff2schema
                    return em
                s = ','.join(['"%s"' % fn for fn in import_keys])
                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
                s = '"id",' + s
                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
                format_error = format + ',"%(Error)s"'
                format = '"%(id)s",'+ format
            ds = DataStructure(data=item,datamodel=dm)
            error_string = ""
            for k in import_keys:
                if not validators[k](ds):
                    error_string += " %s : %s" % (k,ds.getError(k))
            if not error_string:
                item.update(dm)
                item['id'],error = importer(item)
                if error:
                    error_string += error
            if error_string:
                item['Error'] = error_string
                not_imported.append(format_error % item)
                total_not_imported += 1
            else:
                em = format % item
                imported.append(em)
                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
                total_imported += 1
            total += 1
            if tr_count > 1000:
                if len(not_imported) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(not_imported) + '\n')
                    not_imported = []
                if len(imported) > 0:
                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(imported) + '\n')
                    imported = []
                em = '%d transactions committed\n' % (tr_count)
                transaction.commit()
                regs = []
                logger.info(em)
                tr_count = 0
        if len(imported) > 0:
            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(imported))
        if len(not_imported) > 0:
            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(not_imported))
        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
        logger.info(em)
        return em
    ###)

InitializeClass(WAeUPTool)