#-*- mode: python; mode: fold -*-
# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
# Author: Joachim Schmitz (js@aixtraware.de)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Id: WAeUPTool.py 2320 2007-10-05 19:11:47Z joachim $
"""The WAeUP Tool Box.
"""

from AccessControl import ClassSecurityInfo
from Acquisition import aq_inner
from Acquisition import aq_parent
from Globals import DTMLFile
from Globals import InitializeClass
from OFS.SimpleItem import SimpleItem
from zExceptions import BadRequest

from Products.CMFCore.utils import getToolByName
from Products.CPSSchemas.DataStructure import DataStructure
from Products.CPSSchemas.DataModel import DataModel
from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
from Products.CMFCore.ActionProviderBase import ActionProviderBase
from Products.CMFCore.permissions import View
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFCore.permissions import ManagePortal
from Products.CMFCore.utils import UniqueObject
from Products.CMFCore.URLTool import URLTool
from Products.CMFCore.utils import getToolByName
from Students import makeCertificateCode
from Globals import package_home,INSTANCE_HOME
p_home = package_home(globals())
i_home = INSTANCE_HOME
import DateTime,time
import logging
import transaction
import csv,re,os,sys
from Products.AdvancedQuery import Eq, Between, Le,In

def getObject(object,name):
    if object.hasObject(name):
        return getattr(object,name)
    return None

class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
    """WAeUP tool"""

    id = 'waeup_tool'
    meta_type = 'WAeUP Tool'
    _actions = ()
    security = ClassSecurityInfo()
    security.declareObjectProtected(View)
    manage_options = ( ActionProviderBase.manage_options
                     + SimpleItem.manage_options
                     )

    def rwrite(self,s): ###(
        response = self.REQUEST.RESPONSE
        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
        response.write("%s<br />\r\n" % s)
    ###)

    def sleep(self,secs): ###(
        "sleep"
        import time
        time.sleep(secs)
        return

###)

    security.declareProtected(ModifyPortalContent,'openLog') ###(
    def openLog(self,name):
        """open a log file"""
        version = 1
        path = "%s/log/%s_%d.log" % (i_home,name,version)
        while os.path.exists(path):
            version += 1
            path = "%s/log/%s_%d.log" % (i_home,name,version)
        log = open(path,"w")
        return log
    ###)

    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
    def bypassQueueCatalog(self,enable=True):
        """bypass the QueueCatalog by setting all indexes to process imediate,
        if enable is True (default) the old settings are restored
        """

    ###)

    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
        """measure Object access Time"""
        import random
        if hasattr(self,'portal_catalog_real'):
            aq_portal = self.portal_catalog_real.evalAdvancedQuery
        else:
            aq_portal = self.portal_catalog.evalAdvancedQuery
        nr_pts = int(nr_pts)
        probe = int(probe)
        intervall = probe/10
        objects = ("application","clearance","personal")
        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
        #i = random.randrange(num_objects)
        count = 0
        found = 0
        not_found = 0
        t_found = 0
        t_not_found = 0
        time_found = time_not_found = 0.0
        t_time_found = t_time_not_found = 0.0
        accessed = []
        t_min = 1000
        t_max = 0
        #import pdb;pdb.set_trace()
        students = self.portal_catalog(portal_type="Student")
        num_students = len(students)
        if method == "d":
            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
            res = aq_portal(query)
            brains = {}
            for r in res:
                sid = r.relative_path.split('/')[-2]
                if brains.has_key(sid):
                    brains[sid][r.portal_type] = r
                else:
                    brains[sid] = {r.portal_type : r}
            brains_list = brains.keys()
            num_objects = len(brains_list)
        else:
            num_objects = num_students
        print "="*40
        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
                                                                        probe,
                                                                        nr_pts,
                                                                        num_objects)
        print "nr found/not time found/not min/max"
        elapse = time.time()
        i_elapse = time.time()
        c_elapse = time.clock()
        for c in range(1,probe + 1):
            i = random.randrange(num_objects)
            if method in ('a','b','c'):
                student_brain = students[i]
            elif method == "d":
                #import pdb;pdb.set_trace()
                student_brain = brains[brains_list[i]]
            if method == "c":
                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
                res = aq_portal(query)
                this_portal_types = [r.portal_type for r in res]
            for i in range(nr_pts):
                oid = objects[i]
                if method == "a":
                    try:
                        student_path = student_brain.getPath()
                        path = "%s/%s" % (student_path,oid)
                        doc = self.unrestrictedTraverse(path).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        pass
                elif method == "b":
                    try:
                        student_object = student_brain.getObject()
                        doc = getattr(student_object,oid).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        not_found += 1
                        pass
                elif method == "c":
                    if portal_types[i] in this_portal_types:
                        found += 1
                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                elif method == "d":
                    if student_brain.has_key(portal_types[i]):
                        found += 1
                        doc = student_brain[portal_types[i]].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                i_elapse = time.time()
            if c and (c % intervall == 0):
                #i_time = time.time() - i_elapse
                t_per = 0.0
                if found:
                    t_per = time_found/found
                if t_per > t_max:
                    t_max = t_per
                if t_per > 0.0 and t_per < t_min:
                    t_min = t_per
                itf = 0.0
                if found:
                    itf = time_found/found
                itnf = 0.0
                if not_found :
                    itnf = time_not_found / not_found
                interval_time = time_found + time_not_found
                s = "%(c)d: %(found)d/%(not_found)d " % vars()
                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
                print s
                t_found += found
                t_not_found += not_found
                t_time_found += time_found
                t_time_not_found += time_not_found
                time_found = time_not_found = 0.0
                found = not_found = 0
        # t_found += found
        # t_not_found += not_found
        elapse = time.time() - elapse
        itf = 0.0
        if t_found:
            itf = t_time_found/t_found
        itnf = 0.0
        if t_not_found:
            itnf = t_time_not_found / t_not_found
        #c_elapse = time.clock() - c_elapse
        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
        print "-"*40
        print s
        rel_found = float(t_found)/probe
        rel_not_found = float(t_not_found)/probe
        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
        print estimated_total_time
    ###)

    security.declareProtected(ModifyPortalContent,'writeLog') ###(
    def writeLog(self,logfile,s):
        """write to the log file"""
        logfile.write(s)

###)

    def generateStudentId(self,letter): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if letter == '?':
            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
        sid = "%c%d" % (letter,r.randint(99999,1000000))
        students = self.portal_url.getPortalObject().campus.students
##        while hasattr(students, sid):
##            sid = "%c%d" % (letter,r.randint(99999,1000000))
        while self.students_catalog(id = sid):
            sid = "%c%d" % (letter,r.randint(99999,1000000))
        return sid
    ###)

    def generatePassword(self,s=None): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if s is None:
            s = 'abcdefghklmnpqrstuvwxy23456789'
        pw = ''
        while len(pw) < 6:
            pw += r.choice(s)
        return pw
    ###)

    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
    def removeDeletedDocIds(self, max=1000):
        """
        remove deleted docids from repository commit after max
        """
        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
        repository = getToolByName(self, 'portal_repository')
        pxtool = getToolByName(self, 'portal_proxies')
        pxtool_infos = pxtool.getRevisionsUsed()

        nb_revs = 0
        docids_d = {} # all docids
        unused_docids_d = {} # all docids that are unused
        ids_unused_revs_docids = [] # ids for revs of unused docids
        ids_unused_revs = [] # ids for unused revs
        total = 0
        idlist = repository.objectIds()
        for id in idlist:
            docid, rev = repository._splitId(id)
            if docid is None:
                logger.info("invalid doc_id %s" % docid)
                continue
            nb_revs += 1
            docids_d[docid] = None
            if not pxtool_infos.has_key(docid):
                unused_docids_d[docid] = None
                ids_unused_revs_docids.append(id)
                ids_unused_revs.append(id)
            elif not pxtool_infos[docid].has_key(rev):
                ids_unused_revs.append(id)
            if len(ids_unused_revs) >= max:
                repository.manage_delObjects(ids_unused_revs)
                #import pdb;pdb.set_trace()
                transaction.commit()
                total += max
                logger.info('removed %d total %d unused docids ' % (max,total))
        anz = len(ids_unused_revs)
        if anz > 0:
            repository.manage_delObjects(ids_unused_revs)
            transaction.commit()
            total += anz
            logger.info('finished removing %d unused docids ' % (total))


###)

    security.declareProtected(View,'getCredential') ###(
    def getCredential(self,student_id):
        "return a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return None
        return getattr(student_entry,"password","not set")
    ###)

    security.declarePublic('checkPassword') ###(
    def checkPassword(self,student_id,password):
        "return a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return False
        return getattr(student_entry,"password","not set") == password
    ###)

    security.declarePublic('editPassword') ###(
    def editPassword(self,student_id,password):
        "edit a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return
        setattr(student_entry,'password',password)
    ###)

    security.declareProtected(View,'doCommit') ###(
    def doCommit(self,logger=None):
        "commit some transactions"
        transaction.commit()
    ###)

    security.declarePublic('loadStudentFoto') ###(
    def loadStudentFoto(self,student,filename,folder):
        "return a student passport picture"
        app = student.application
        app_doc = app.getContent()
        #clear = student.clearance
        #clear_doc = clear.getContent()
        #matric_no = clear_doc.matric_no.upper()
        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
        #import pdb;pdb.set_trace()
        if os.path.exists(picture1):
            file = open(picture1)
        elif os.path.exists(picture2):
            file = open(picture2)
        else:
            return "passport picture not found %s" % picture1
        reopened = False
        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
            self.portal_workflow.doActionFor(app,'open')
            reopened = True
        outfile = file.read()
        app_doc.manage_addFile('passport',
                               file=outfile,
                               title="%s.jpg" % filename)
        if reopened:
            self.portal_workflow.doActionFor(app,'close')
        return "successfully loaded passport picture"
    ###)

    security.declareProtected(ModifyPortalContent,'createOne') ###(
    def createOne(self,students_folder,student_brain,letter,commit=False):
        sid = self.waeup_tool.generateStudentId(letter)
        students_folder.invokeFactory('Student', sid)
        student = getattr(students_folder,sid)
        self.portal_workflow.doActionFor(student,'return')
        student.manage_setLocalRoles(sid, ['Owner',])
        matric_no = student_brain.matric_no
        jamb_reg_no = student_brain.Entryregno
        self.students_catalog.addRecord(id = sid,
                                           matric_no = matric_no,
                                           jamb_reg_no = jamb_reg_no,
                                           sex = student_brain.Sex == "F",
                                           name = "%s %s %s" % (student_brain.Firstname,
                                                                student_brain.Middlename,
                                                                student_brain.Lastname)
                                        )
        if commit:
            transaction.commit()
        return sid,jamb_reg_no
    ###)

    security.declareProtected(ModifyPortalContent,'addStudent') ###(
    def addStudent(self,dict):
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.student_field2types
        #from pdb import set_trace; set_trace()
        d = {}
        #d['jamb_sex']  = 'M'
        #if dict.get('sex'):
        #    d['jamb_sex']  = 'F'

        entry_session = dict.get('entry_session')
        if entry_session == self.getSessionId()[-2:]:
            wfaction = 'admit'
            wft = 'wf_transition_admit'
            password = None
        else:
            wfaction = 'return'
            wft = 'wf_transition_return'
            password = self.generatePassword()
            self.makeStudentMember(sid,password)

        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = dict.get(field,'')
            sub_doc.edit(mapping = d)
            new_state = f2t[pt][wft]
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        self.portal_workflow.doActionFor(student_obj,wfaction)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,password
    ###)

    security.declarePublic('getCertificateBrain') ###(
    def getCertificateBrain(self,cert_id):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"Certificate",
                                      'id': cert_id})
        if res:
            return res[0]
        return None
    ###)

    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
    def get_csv_filenames(self):
        "do it"
        files = [file for file in os.listdir("%s/import/" % (i_home))
                 if file.endswith('.csv') and file.find('imported') == -1]
        return files
    ###)

    security.declarePublic('findStudentByMatricelNo') ###(
    def findStudentByMatricelNo(self,matric_no):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"StudentClearance",
                                 'SearchableText': matric_no})
        if res:
            return res[0]
        return None
    ###)

    security.declarePublic('makeStudentMember') ###(
    def makeStudentMember(self,sid,password='uNsEt'):
        """make the student a member"""
        membership = self.portal_membership
        membership.addMember(sid,
                             password ,
                             roles=('Member',
                                     'Student',
                                     ),
                             domains='',
                             properties = {'memberareaCreationFlag': False,
                                           'homeless': True},)
        member = membership.getMemberById(sid)
        self.portal_registration.afterAdd(member, sid, password, None)
        #self.manage_setLocalRoles(sid, ['Owner',])
    ###)

    security.declarePublic('makeStudentData') ###(
    def makeStudentData(self,student_id,email=None,phone_nr=None):
        "create Datastructure for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentData')
        students_folder = self.portal_url.getPortalObject().campus.students
        #res = self.students_catalog(id=student_id)
        #if res:
        #    st = res[0]
        #res = self.returning_import(matric_no = st.matric_no)
        res = self.returning_import(id = student_id)
        if res:
            student = res[0]
        else:
            logger.info('Id %s not found in returning_import' % student_id)
            return
        logger.info('%s creates data structure' % student_id)
        s_results = self.results_import(matric_no = student.matric_no)
        if s_results:
            lnr = self.getLevelFromResultsCosCode(s_results)
            level = "%d00" % lnr
            verdict,eligible = self.getVerdict(s_results[0].Verdict)
            if eligible:
                level = "%d00" % (lnr + 1)
        else:
            logger.info('matric_no %s not found in results_import' % student.matric_no)
            level = 0
            verdict = 'N/A'
        #student should not be allowed to perform this transition
        #wftool = self.portal_workflow
        #wftool.doActionFor(student,'return')
        certcode_org = student.Coursemajorcode
        certcode = makeCertificateCode(certcode_org)
        certificate_brain = self.getCertificateBrain(certcode)
        if not certificate_brain:
            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
            logger.info(em)
        matric_no = student.matric_no
        sid = student_id
        student_obj = getattr(students_folder,sid)
        student_obj.invokeFactory('StudentApplication','application')
        application = student_obj.application
        self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        student_obj.invokeFactory('StudentPersonal','personal')
        da['jamb_reg_no'] = student.Entryregno
        em = self.getEntryMode(student.Entryregno)
##        em = student.Mode_of_Entry
##        if em in ('DIRECT', 'DIRECT ENTRY',):
##            em = 'DE'
##        elif em in ('U.M.E', 'UNE',):
##            em = 'UME'
##        elif not em:
##            em = "unknown"
        da['entry_mode'] = em
        personal = student_obj.personal
        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        student_obj.invokeFactory('StudentClearance','clearance')
        clearance = student_obj.clearance
        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['matric_no'] = matric_no
        state = student.State
        lga = student.LGA
        if state and lga:
            lga =  state + ' / ' + lga
        else:
            lga = "None"
        da['jamb_lga'] = dc['lga'] = lga
        da['app_email'] = dp['email'] = email
        da['app_mobile'] = dp['phone'] = phone_nr
        dp['firstname'] = student.Firstname
        dp['middlename'] = student.Middlename
        dp['lastname'] = student.Lastname
        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
        da['jamb_sex'] = student.Sex
        dp['sex'] = student.Sex == 'F'
        dp['perm_address'] = student.Permanent_Address
        application.getContent().edit(mapping=da)
        self.portal_workflow.doActionFor(application,'close',dest_container=application)
        personal.getContent().edit(mapping=dp)
        clearance.getContent().edit(mapping=dc)
        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
##        catd = {}
##        catd['id'] = sid
##        catd['entry_mode']= da['entry_mode']
##        catd['matric_no'] = matric_no
##        catd['jamb_reg_no'] = da['jamb_reg_no']
##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
##        catd['sex'] = dp['sex']
##        catd['level'] = level
##        catd['verdict'] = verdict
##        if certificate_brain:
##            cpath = certificate_brain.getPath().split('/')
##            catd['faculty'] = cpath[-4]
##            catd['department'] = cpath[-3]
##            catd['course'] = certcode
##        self.students_catalog.modifyRecord(**catd)
        #
        # Study Course
        #
        student_obj.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_obj.study_course
        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = certcode
        dsc['current_level'] = level
        dsc['current_verdict'] = verdict
        dsc['current_mode'] = em
        dsc['current_session'] = '06'
        studycourse.getContent().edit(mapping=dsc)
        #
        # Level
        #
##        l = getattr(studycourse,level,None)
##        if l is None:
##            studycourse.invokeFactory('StudentStudyLevel', level)
##            l = getattr(studycourse, level)
##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
##            l.getContent().edit(mapping={'Title': "Level %s" % level})
###)

    security.declarePublic('makeStudentLevel') ###(
    def makeStudentLevel(self,student_id):
        "create the StudyLevel for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
        students_folder = self.portal_url.getPortalObject().campus.students
        res = self.students_catalog(id=student_id)
        if res:
            st = res[0]
        course = st.course
        matric_no = st.matric_no
        level = st.level
        res = self.results_import(matric_no = matric_no)
        if res:
            results = res
        logger.info('%s creating Level %s' % (student_id,level))
        #
        # Level
        #
        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        studycourse = getattr(student_obj,"study_course",None)
        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
        l = getattr(studycourse,level,None)
        if l is None:
            studycourse.invokeFactory('StudentStudyLevel', level)
            l = getattr(studycourse, level)
            self.portal_workflow.doActionFor(l,'open',dest_container=l)
            l.getContent().edit(mapping={'Title': "Level %s" % level})
        ###)

    security.declarePublic('getHallInfo') ###(
    def getHallInfo(self,bed):
        """return Hall Info"""
        info = {}
        hall,block,room,letter = bed.split('_')
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
        if res and len(res) == 1:
            hall_brain = res[0]
            hall_doc = hall_brain.getObject().getContent()
        else:
            return info
        info['hall_title'] = hall_brain.Title
        info['maintenance_code'] = hall_doc.maintenance_code
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
        batch_doc = None
        for brain in res:
            if brain.id.startswith(info['maintenance_code']):
                batch_doc = brain.getObject().getContent()
                break
        if batch_doc is None:
            info['maintenance_fee'] = None
        else:
            info['maintenance_fee'] = batch_doc.cost
        return info
    ###)

    security.declarePublic('showFsPicture') ###(
    def showFsPicture(self,path):
        """return a picture from the filesystem"""
        picture_path = "%s/import/%s" % (i_home,path)
        if os.path.exists(picture_path):
            return open(picture_path).read()

    ###)

    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
    def deleteAllCourses(self,department="All"):
        ''' delete the courses'''
        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        if str(member) not in ("henrik","joachim"):
            return "not possible"
        if department == "All":
            res = self.portal_catalog({'meta_type': 'Department'})
        if len(res) < 1:
            return "No Departments found"

        deleted = []
        for dep in res:
            cf = dep.getObject().courses
            if cf:
                cf.manage_delObjects(ids=cf.objectIds())
                deleted.append("deleted Courses in %s" % dep.getId)
        return "\r".join(deleted)
    ###)

    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
    def getLogfileLines(self,filename="event.log",numlines=20):
        """Get last NUMLINES lines of logfile FILENAME.

        Return last lines' of a file in the instances logfile directory as
        a list. The number of returned lines equals `numlines' or less. If
        less than `numlines' lines are available, the whole file ist
        returned. If the file can not be opened or some other error
        occurs, empty list is returend.
        """
        result = []
        lines_hit = 0

        # We only handle files in instances' log directory...
        logpath = os.path.join(i_home, "log")
        filename = str(os.path.abspath( os.path.join( logpath, filename )))
        if not filename.startswith( logpath ):
            # Attempt to access file outside log-dir...
            return []

        try:
            fd = file( filename, "rb" )
        except IOError:
            return []
        if not fd:
            return []

        if os.linesep == None:
            linesep = '\n'
        else:
            linesep = os.linesep

        # Try to find 'numlines' times a lineseparator, searching from end
        # and moving to the beginning of file...
        fd.seek( 0, 2) # Move to end of file...
        while lines_hit < numlines:
            if fd.read(1) == linesep[-1]: # This moves filedescriptor
                                          # one step forward...
                lines_hit += 1
            try:
                fd.seek( -2, 1) # Go two bytes back from current pos...
            except IOError:
                # We cannot go back two bytes. Maybe the file is too small...
                break
        fd.seek(2,1)

        # Read all lines from current position...
        result = fd.readlines()
        # Remove line endings...
        result = [x.strip() for x in result]
        fd.close()
        return result
    ###)

    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
    def getCallbacksFromLog(self,filename):
        """fix Online Payment Transactions from Z2.log entries"""
        import transaction
        import random
        from cgi import parse_qs
        from urlparse import urlparse
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
        data = re.compile(s)
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        #name = 'epaymentsuccessful_z2log2'
        name = filename
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
        try:
            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
        except:
            logger.error('Error reading %s' % name)
            return
        tas = []
        for line in transactions:
            dict = {}
            items = data.search(line)
            dict['idict'] = idict = items.groupdict()
            #print idict
            #from pdb import set_trace;set_trace()
            urlparsed = urlparse(idict['get'][4:])
            #print urlparsed
            path = urlparsed[2].split('/')
            dict['student_id'] = student_id = path[8]
            dict['payment_id'] = payment_id = path[10]
            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
            tas.append(dict)
            tr_count += 1
        return tas
    ###)

    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
    def importOnlinePaymentTransactions(self):
        """load Online Payment Transactions from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        opt = self.online_payments_import
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'OnlineTransactions'
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
        try:
            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for pay_transaction in transactions:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
                format_error = format + ',"%(Error)s"'
            data = {}

            # format of the first file sent by Tayo
            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = pay_transaction['Resp Code']
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the second file sent by Tayo
            #data['datetime'] = date = 0
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = '00'
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the third file sent by Kehinde
            data['datetime'] = date = 0
            data['student_id'] = student_id = pay_transaction['customer_id']
            data['order_id'] = order_id = pay_transaction['merchant_reference']
            data['response_code'] = response_code = '00'
            data['amount'] = amount = pay_transaction['Amount']

            dup = False
            if response_code == "12":
                continue
            try:
                opt.addRecord(**data)
            except ValueError:
                dup = True
            #from pdb import set_trace;set_trace()
            if dup:
                if response_code == "00":
                    try:
                        opt.modifyRecord(**data)
                    except:
                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
                        continue
                else:
                    pay_transaction['Error'] = "Duplicate order_id"
                    no_import.append( format_error % pay_transaction)
                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
                    continue
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions committed\n' % (tr_count)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
    def mass_create_faculty(self,mapping):
        "create a faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['code']
        if getattr(academics_folder,fid,None) is not None:
            return '', "Faculty with ID: %s exists" % fid
        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
        try:
            academics_folder.invokeFactory('Faculty', fid)
        except BadRequest,E:
            return '', "%s" % E
        f = getattr(academics_folder,fid,None)
        f.getContent().edit(mapping=mapping)
        return fid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
    def mass_edit_faculty(self,mapping):
        "edit a faculty"
        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['code']
        f = getattr(academics_folder,fid,None)
        if f is None:
            return '', "Faculty with ID: %s does not exist" % fid
        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
        f.getContent().edit(mapping=mapping)
        return fid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
    def mass_create_department(self,mapping):
        "create a department in the correct faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_department')
        fid = mapping['faculty_code']
        if getattr(self,'_v_faculties',None) is None:
            res = self.portal_catalog(portal_type = "Faculty")
            self._v_faculties = {}
            for f in res:
                self._v_faculties[f.getId] = f.getObject()
        f = self._v_faculties.get(fid,None)
        if f is None:
            return '', "No Faculty with ID: %s" % fid
        else:
            did = mapping.get('code')
            d = getattr(f,did,None)
            if d is None or d.portal_type == "Faculty":
                logger.info('Creating Department %(code)s, %(title)s' % mapping)
                try:
                    f.invokeFactory('Department', did)
                except BadRequest,E:
                    return '', "%s" % E
                d = getattr(f,did)
                d.invokeFactory('CoursesFolder','courses')
                courses = getattr(d,'courses')
                dict = {'Title': 'Courses'}
                courses.getContent().edit(mapping=dict)
                d.invokeFactory('CertificatesFolder','certificates')
                certificates = getattr(d,'certificates')
                dict = {'Title': 'Certificates'}
                certificates.getContent().edit(mapping=dict)
            d.getContent().edit(mapping=mapping)
        return did,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
    def mass_edit_department(self,mapping):
        "create a department in the correct faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_department')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['faculty_code']
        did = mapping.get('code')
        try:
            d = getattr(getattr(academics_folder,fid),did,None)
        except KeyError:
            return '', "Department %s or Faculty %s wrong" % (did,fid)
        else:
            if d is None or d.portal_type == "Faculty":
                logger.info('Editing Department %(code)s, %(title)s' % mapping)
            d.getContent().edit(mapping=mapping)
        return did,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
    def mass_create_course(self,mapping):
        #import pdb;pdb.set_trace()
        if getattr(self,'_v_course_list',None) is None:
            self._v_course_list = []
        if getattr(self,'_v_departments',None) is None:
            res = self.portal_catalog(portal_type = "Department")
            self._v_department_courses = {}
            for d in res:
                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
        did = mapping['department_code']
        d = self._v_department_courses.get(did,None)
        if d is None:
            return '', "No Department with ID: %s" % did
        course_id = mapping.get('code')
        if course_id in self._v_course_list:
            return '', "Duplicate Course ID: %s" % did
        c = getattr(d,course_id,None)
        if c is not None:
            return '', "Duplicate Course ID: %s" % did
        try:
            d.invokeFactory('Course', course_id)
        except BadRequest,E:
            return '', "%s" % E
        self._v_course_list.append(course_id)
        c = getattr(d,course_id)
        c.getContent().edit(mapping=mapping)
        return course_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
    def mass_edit_course(self,mapping):
        #import pdb;pdb.set_trace()
        course_id = mapping.get('code')
        res = self.portal_catalog(id=course_id)
        if not res:
            return '', "No Course with ID: %s" % course_id
        c = res[0].getObject()
        c.getContent().edit(mapping=mapping)
        return course_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
    def mass_create_certificate(self,mapping):
        if getattr(self,'_v_certificate_list',None) is None:
            self._v_certificate_list = []
        if getattr(self,'_v_department_certificates',None) is None:
            res = self.portal_catalog(portal_type = "Department")
            self._v_department_certificates = {}
            for d in res:
                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
        did = mapping['department_code']
        d = self._v_department_certificates.get(did,None)
        if d is None:
            return '', "No Department with ID: %s" % did
        certificate_id = mapping.get('code')
        if certificate_id in self._v_certificate_list:
            return '', "Duplicate Certificate ID: %s" % did
        c = getattr(d,certificate_id,None)
        if c is not None:
            return '', "Duplicate Certificate ID: %s" % did
        try:
            d.invokeFactory('Certificate', certificate_id)
        except BadRequest,E:
            return '', "%s" % E
        self._v_certificate_list.append(certificate_id)
        c = getattr(d,certificate_id)
        c.getContent().edit(mapping=mapping)
        return certificate_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
    def mass_edit_certificate(self,mapping):
        #import pdb;pdb.set_trace()
        certificate_id = mapping.get('code')
        res = self.portal_catalog(id=certificate_id)
        if not res:
            return '', "No Certificate with ID: %s" % did
        c = res[0].getObject()
        c.getContent().edit(mapping=mapping)
        return certificate_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
    def mass_create_certificate_course(self,mapping):
        if getattr(self,'_v_courses',None) is None:
            res = self.courses_catalog()
            self._v_courses= [course.code for course in res]  
        if getattr(self,'_v_ceritficates',None) is None:
            res = self.portal_catalog(portal_type = "Certificate")
            self._v_certificates = {}
            for cert in res:
                self._v_certificates[cert.getId] = cert.getObject()
        certificate_course_id = mapping.get('code')
        if certificate_course_id not in self._v_courses: 
            return '', "No Course with ID: %s" % certificate_course_id
        cert_id = mapping['certificate_code']
        cert = self._v_certificates.get(cert_id,None)
        if cert is None:
            return '', "No Certificate with ID: %s" % cert_id
        level_id = mapping.get('level')
        level = getattr(cert,level_id,None)
        if level is None:
            cert.invokeFactory('StudyLevel', level_id)
            level = getattr(cert,level_id,None)
        elif hasattr(level,certificate_course_id):
            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
            (certificate_course_id,cert_id,level_id)
        level.invokeFactory('CertificateCourse', certificate_course_id)
        c = getattr(level,certificate_course_id)
        c.getContent().edit(mapping=mapping)
        return certificate_course_id,''
    ###)

    field2types_student = {   ###(
                      'StudentApplication':
                          {'id': 'application',
                           'title': 'Application Data',
                           'wf_transition_return': 'close',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('jamb_reg_no',
                              'entry_mode',
                              'entry_session',
                              'jamb_score',
                              'app_email',
                              'jamb_age',
                              #'jamb_state',
                              #'jamb_lga',
                              )
                              },
                      #'StudentPume':
                      #    {'id': 'pume',
                      #     'title': 'Pume Data',
                      #     'wf_transition_return': 'close',
                      #     'wf_transition_admit': 'close',
                      #     'fields':
                      #       ('pume_score',
                      #        )
                      #        },
                      'StudentClearance':
                          {'id': 'clearance',
                           'title': 'Clearance/Eligibility Record',
                           'wf_transition_return': 'close',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('matric_no',
                              'nationality',
                              'lga',
                              'birthday',
                              )
                              },
                         'StudentPersonal':
                          {'id': 'personal',
                           'title': 'Personal Data',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('firstname',
                              'middlename',
                              'lastname',
                              'sex',
                              'email',
                              'phone',
                              'perm_address',
                              )
                              },
                         'StudentStudyCourse':
                          {'id': 'study_course',
                           'title': 'Study Course',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('study_course',
                              'current_level',
                              'current_session',
                              'current_mode',
                              'current_verdict',
                              )
                              },

                         'PaymentsFolder':
                          {'id': 'payments',
                           'title': 'Payments',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'open',
                           'fields':
                             ()
                              },
                         }
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
    def mass_create_student(self,mapping):
        "create a students record due import"
        logger = logging.getLogger('WAeUPTool.mass_create_student')
        students_folder = self.portal_url.getPortalObject().campus.students
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        if jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if res:
                return '',"jamb_reg_no exists"
        matric_no = mapping.get('matric_no',None)
        if matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if res:
                return '',"matric_no exists"
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        d = {}
        d['jamb_sex']  = 'M'
        if mapping.get('sex'):
            d['jamb_sex']  = 'F'
        transition = mapping.get('reg_transition','admit')
        if transition not in ('admit','return'):
            return '',"no valid transition provided"
        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = mapping.get(field,'')
            if pt == "StudyCourse":
                for von,zu in (('entry_mode','current_mode'),
                               ('entry_session','current_session')):
                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
                        d[zu] = mapping[von]
            sub_doc.edit(mapping = d)

            #import pdb;pdb.set_trace()
            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        self.portal_workflow.doActionFor(student_obj,transition)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
    def mass_edit_student(self,mapping):
        "edit a students record due import"
        logger = logging.getLogger('WAeUPTool.mass_edit_student')
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = mapping.get('id',None)
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        matric_no = mapping.get('matric_no',None)
        editable_keys = mapping.keys()
        if sid:
            res = self.students_catalog(id = sid)
            if not res:
                return '',"no student with id %s" % sid
            if matric_no and res[0].matric_no and\
              matric_no != res[0].matric_no:
                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
            if jamb_reg_no and res[0].jamb_reg_no and\
              jamb_reg_no != res[0].jamb_reg_no:
                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
        elif jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if not res:
                return '',"no student with jamb_reg_no %s" % jamb_reg_no
            editable_keys.remove('jamb_reg_no')
        elif matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if not res:
                return '',"no student with matric_no %s" % matric_no
            editable_keys.remove('matric_no')

        ## included only to change wf state from admitted to returning
        #if res[0].review_state not in ('admitted','objection_raised'):
        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
        ## end inclusion

        sid = res[0].id
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        d = {}
        #import pdb;pdb.set_trace()
        any_change = False
        for pt in f2t.keys():
            if pt == "student_application":
                d['jamb_sex']  = 'M'
                if mapping.get('sex'):
                    d['jamb_sex']  = 'F'
            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
            if intersect:
                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
                if sub_obj is None:
                    try:
                        student_obj.invokeFactory(pt,f2t[pt]['id'])
                    except:
                        continue
                    sub_obj = getattr(student_obj,f2t[pt]['id'])
                    d['Title'] = f2t[pt]['title']
                sub_doc = sub_obj.getContent()
                for field in intersect:
                    changed = False
                    if getattr(sub_doc,field) != mapping.get(field,''):
                        any_change = True
                        changed = True
                        d[field] = mapping.get(field,'')
                    if changed:
                        sub_doc.edit(mapping = d)


        ## included only to change wf state from admitted to returning
        #    if res[0].review_state in ('admitted','objection_raised'):
        #        new_state = f2t[pt]['wf_transition_return']
        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
        #        if sub_obj and new_state != "remain":
        #            try:
        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        #            except:
        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
        #                pass
        #if res[0].review_state in ('admitted','objection_raised'):
        #    wfaction = 'return'
        #    try:
        #        self.portal_workflow.doActionFor(student_obj,wfaction)
        #        logger.info('%s, wf state changed' % sid)
        #        any_change = True
        #    except:
        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
        #        pass
        ## end inclusion


        if any_change:
            return sid,''
        else:
            return sid,'not modified'
    ###)

    security.declareProtected(ModifyPortalContent,"importData")###(
    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
        """load data from CSV values"""
        import transaction
        import random

        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        logger = logging.getLogger('WAeUPTool.importData')
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total_imported = 0
        total_not_imported = 0
        total = 0
        iname = "import_%s" % name
        stool = getToolByName(self, 'portal_schemas')
        ltool = getToolByName(self, 'portal_layouts')
        schema = stool._getOb(iname)
        if schema is None:
            em = 'No such schema %s' % iname
            logger.error('No such schema %s' % iname)
            return em
        layout = ltool._getOb(iname)
        if layout is None:
            em = 'No such layout %s' % iname
            logger.error(em)
            return em
        validators = {}
        for widget in layout.keys():
            validators[widget] = layout[widget].validate
        #import pdb;pdb.set_trace()
        mode = "create"
        if edit:
            mode = "edit"
        importer_name = "mass_%(mode)s_%(name)s" % vars()
        importer = getattr(self, '%s' % importer_name,None)
        if importer is None:
            em = 'No importer function %s' % importer_name
            logger.error(em)
            return em
        not_imported = []
        imported = []
        try:
            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
        except:
            em = 'Error reading %s.csv' % filename
            logger.error(em)
            return em
        for item in items:
            if start:
                start = False
                adapters = [MappingStorageAdapter(schema, item)]
                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
                import_keys = [k for k in attrs if not k.startswith('ignore')]
                diff2schema = set(import_keys).difference(set(schema.keys()))
                diff2layout = set(import_keys).difference(set(layout.keys()))
                if diff2schema:
                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
                    return em
                s = ','.join(['"%s"' % fn for fn in import_keys])
                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
                s = '"id",' + s
                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
                format_error = format + ',"%(Error)s"'
                format = '"%(id)s",'+ format

            dm = DataModel(item, adapters,context=self)
            ds = DataStructure(data=item,datamodel=dm)
            error_string = ""
            for k in import_keys:
                #import pdb;pdb.set_trace()
                if not validators[k](ds,mode=mode):
                    error_string += " %s : %s" % (k,
                                                  self.translation_service(ds.getError(k),
                                                                           ds.getErrorMapping(k)))
            if not error_string:
                item.update(dm)
                item['id'],error = importer(item)
                if error:
                    error_string += error
            if error_string:
                item['Error'] = error_string
                not_imported.append(format_error % item)
                total_not_imported += 1
            else:
                em = format % item
                imported.append(em)
                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
                tr_count += 1
                total_imported += 1
            total += 1
            if total and not total % 100:
                transaction.commit()
                if len(not_imported) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(not_imported) + '\n')
                    not_imported = []
                if len(imported) > 0:
                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(imported) + '\n')
                    imported = []
                em = '%d transactions committed\n' % (tr_count)
                regs = []
                logger.info(em)
                tr_count = 0
        if len(imported) > 0:
            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(imported))
        if len(not_imported) > 0:
            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(not_imported))
        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
        return em
    ###)

    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
    def moveImagesToFS(self,student_id="O738726"):
        "move the images to the filesystem"
        images_dir = os.path.join("%s/images" % i_home,student_id)
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key)
                if not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))

    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
    def movePassportToFS(self,student_id="O738726"):
        "move the passports to the filesystem"
        images_dir = os.path.join("%s" % i_home,'passports')
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   #'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key)
                if not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))

InitializeClass(WAeUPTool)