#-*- mode: python; mode: fold -*-
# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
# Author: Joachim Schmitz (js@aixtraware.de)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Id: WAeUPTool.py 3033 2008-01-22 21:00:09Z joachim $
"""The WAeUP Tool Box.
"""

from AccessControl import ClassSecurityInfo
from Acquisition import aq_inner
from Acquisition import aq_parent
from Globals import DTMLFile
from Globals import InitializeClass
from OFS.SimpleItem import SimpleItem
from zExceptions import BadRequest

from Products.CMFCore.utils import getToolByName
from Products.CPSSchemas.DataStructure import DataStructure
from Products.CPSSchemas.DataModel import DataModel
from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
from Products.CMFCore.ActionProviderBase import ActionProviderBase
from Products.CMFCore.permissions import View
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFCore.permissions import ManagePortal
from Products.CMFCore.utils import UniqueObject
from Products.CMFCore.URLTool import URLTool
from Products.CMFCore.utils import getToolByName
from Students import makeCertificateCode
from Globals import package_home,INSTANCE_HOME
import DateTime,time
import logging
import transaction
import csv,re,os,sys
from shutil import copy2
from Products.AdvancedQuery import Eq, Between, Le,In

p_home = package_home(globals())
i_home = INSTANCE_HOME
images_base = os.path.join(i_home,"images")

def getImagesDir(student_id):
    return os.path.join("%s" % images_base,student_id[0],student_id)

def getObject(object,name):
    if object.hasObject(name):
        return getattr(object,name)
    return None


class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
    """WAeUP tool"""

    id = 'waeup_tool'
    meta_type = 'WAeUP Tool'
    _actions = ()
    security = ClassSecurityInfo()
    security.declareObjectProtected(View)
    manage_options = ( ActionProviderBase.manage_options
                     + SimpleItem.manage_options
                     )

    security.declareProtected(View,'re_split') ###(
    def re_split(self,split_string,string):
        return re.split(split_string,string)
    ###)

    security.declareProtected(View,'difference') ###(
    def difference(self,l1,l2):
        return set(l1).difference(set(l2))
    ###)

    def rwrite(self,s): ###(
        response = self.REQUEST.RESPONSE
        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
        response.write("%s<br />\r\n" % s)
    ###)

    def addtodict(self,d,key,item): ###(
        d[key].append(item)
        return d[key]
    ###)

    def sleep(self,secs): ###(
        "sleep"
        import time
        time.sleep(secs)
        return

###)

    security.declareProtected(View,'updateRoleMappingsFor') ###(
    def updateRoleMappingsFor(self,wf_definition,ob):
        "do so for public"
        wf_def = getattr(self.portal_workflow,wf_definition)
        wf_def.updateRoleMappingsFor(ob)
    ###)

    security.declareProtected(View,'getStatesLgas') ###(
    def getStatesLgas(self):
        """return lga info"""
        voc = getattr(self.portal_vocabularies,'local_gov_areas')
        states = []
        lgas  = []
        d = {}
        wd = {}
        for k,v in voc.items():
            parts = v.split(' / ')
            if len(parts) == 1:
                state = parts[0].lower()
                lga = ""
            elif len(parts) == 2:
                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
            else:
                continue
            if state not in states:
                states.append(state)
            if lga not in lgas:
                lgas.append(lga)
            words = re.split('[^a-zA-Z0-9/]',k)
            words.sort()
            wd[k] = words
            d[k] = v
        mapping = {}
        mapping['word_dict'] = wd
        mapping['lga_dict'] = d
        mapping['states'] = states
        mapping['lgas'] = lgas
        return mapping
    ###)

    security.declareProtected(View,'findLga') ###(
    def findLga(self,words,words_dict):
        words = re.split('[^a-zA-Z0-9/]',words)
        lga_words = []
        for word in words:
            if word:
                lga_words += word.strip().lower(),
        lga_words.sort()
        state_lga = ''
        while not state_lga:
            for k,l in words_dict.items():
                if lga_words == l:
                    state_lga = k
                    break
            break
        return state_lga
    ###)
    security.declareProtected(View,'getAccessInfo') ###(
    def getAccessInfo(self,context):
        "return a dict with access_info"
        logger = logging.getLogger('WAeUPTool.getAccessInfo')
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        member_id = str(member)
        info = {}
        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
        is_staff = info['is_staff'] = not is_anonymous and not is_student
        roles = member.getRolesInContext(context)
        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
                                                                            "SectionManager" in roles or
                                                                            "Manager" in roles)
        is_allowed = info['is_allowed'] = not is_anonymous
        requested_id = context.getStudentId()
        student_id  = None
        if is_allowed:
            if not is_student and requested_id:
                student_id  = requested_id
            elif is_anonymous or member_id != requested_id:
                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
            else:
                student_id = member_id
        info['student_id'] = student_id
        return info
    ###)

    security.declareProtected(ModifyPortalContent,'openLog') ###(
    def openLog(self,name):
        """open a log file"""
        version = 1
        path = "%s/log/%s_%d.log" % (i_home,name,version)
        while os.path.exists(path):
            version += 1
            path = "%s/log/%s_%d.log" % (i_home,name,version)
        log = open(path,"w")
        return log
    ###)

    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
    def bypassQueueCatalog(self,enable=True):
        """bypass the QueueCatalog by setting all indexes to process imediate,
        if enable is True (default) the old settings are restored
        """

    ###)

    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
        """measure Object access Time"""
        import random
        if hasattr(self,'portal_catalog_real'):
            aq_portal = self.portal_catalog_real.evalAdvancedQuery
        else:
            aq_portal = self.portal_catalog.evalAdvancedQuery
        nr_pts = int(nr_pts)
        probe = int(probe)
        intervall = probe/10
        objects = ("application","clearance","personal")
        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
        #i = random.randrange(num_objects)
        count = 0
        found = 0
        not_found = 0
        t_found = 0
        t_not_found = 0
        time_found = time_not_found = 0.0
        t_time_found = t_time_not_found = 0.0
        accessed = []
        t_min = 1000
        t_max = 0
        #import pdb;pdb.set_trace()
        students = self.portal_catalog(portal_type="Student")
        num_students = len(students)
        if method == "d":
            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
            res = aq_portal(query)
            brains = {}
            for r in res:
                sid = r.relative_path.split('/')[-2]
                if brains.has_key(sid):
                    brains[sid][r.portal_type] = r
                else:
                    brains[sid] = {r.portal_type : r}
            brains_list = brains.keys()
            num_objects = len(brains_list)
        else:
            num_objects = num_students
        print "="*40
        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
                                                                        probe,
                                                                        nr_pts,
                                                                        num_objects)
        print "nr found/not time found/not min/max"
        elapse = time.time()
        i_elapse = time.time()
        c_elapse = time.clock()
        for c in range(1,probe + 1):
            i = random.randrange(num_objects)
            if method in ('a','b','c'):
                student_brain = students[i]
            elif method == "d":
                #import pdb;pdb.set_trace()
                student_brain = brains[brains_list[i]]
            if method == "c":
                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
                res = aq_portal(query)
                this_portal_types = [r.portal_type for r in res]
            for i in range(nr_pts):
                oid = objects[i]
                if method == "a":
                    try:
                        student_path = student_brain.getPath()
                        path = "%s/%s" % (student_path,oid)
                        doc = self.unrestrictedTraverse(path).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        pass
                elif method == "b":
                    try:
                        student_object = student_brain.getObject()
                        doc = getattr(student_object,oid).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        not_found += 1
                        pass
                elif method == "c":
                    if portal_types[i] in this_portal_types:
                        found += 1
                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                elif method == "d":
                    if student_brain.has_key(portal_types[i]):
                        found += 1
                        doc = student_brain[portal_types[i]].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                i_elapse = time.time()
            if c and (c % intervall == 0):
                #i_time = time.time() - i_elapse
                t_per = 0.0
                if found:
                    t_per = time_found/found
                if t_per > t_max:
                    t_max = t_per
                if t_per > 0.0 and t_per < t_min:
                    t_min = t_per
                itf = 0.0
                if found:
                    itf = time_found/found
                itnf = 0.0
                if not_found :
                    itnf = time_not_found / not_found
                interval_time = time_found + time_not_found
                s = "%(c)d: %(found)d/%(not_found)d " % vars()
                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
                print s
                t_found += found
                t_not_found += not_found
                t_time_found += time_found
                t_time_not_found += time_not_found
                time_found = time_not_found = 0.0
                found = not_found = 0
        # t_found += found
        # t_not_found += not_found
        elapse = time.time() - elapse
        itf = 0.0
        if t_found:
            itf = t_time_found/t_found
        itnf = 0.0
        if t_not_found:
            itnf = t_time_not_found / t_not_found
        #c_elapse = time.clock() - c_elapse
        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
        print "-"*40
        print s
        rel_found = float(t_found)/probe
        rel_not_found = float(t_not_found)/probe
        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
        print estimated_total_time
    ###)

    security.declareProtected(ModifyPortalContent,'writeLog') ###(
    def writeLog(self,logfile,s):
        """write to the log file"""
        logfile.write(s)

###)

    def generateStudentId(self,letter): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if letter == '?':
            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
        sid = "%c%d" % (letter,r.randint(99999,1000000))
        students = self.portal_url.getPortalObject().campus.students
##        while hasattr(students, sid):
##            sid = "%c%d" % (letter,r.randint(99999,1000000))
        while self.students_catalog(id = sid):
            sid = "%c%d" % (letter,r.randint(99999,1000000))
        return sid
    ###)

    def generatePassword(self,s=None): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if s is None:
            s = 'abcdefghklmnpqrstuvwxy23456789'
        pw = ''
        while len(pw) < 6:
            pw += r.choice(s)
        return pw
    ###)

    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
    def dumpSchoolfeePayments(self):
        "dump paid schoolfees"
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
        aq_student = self.students_catalog.evalAdvancedQuery
        query = In('review_state',('schoolfee_paid',
                                   'courses_registered',
                                   'courses_validated',
                                   ))
        res = aq_student(query)
        #import pdb;pdb.set_trace()
        l = []
        logger.info("start for %d" % len(res))
        count = 1
        log_after = 100
        for student in res:
            if not count % log_after:
                logger.info("processed %d total %d" % (log_after,count))
            count += 1
            fee_dict =self.getSchoolFee(student)
            fulltime = student.mode.endswith('_ft')
            d = {}
            d['student_id'] = student.id
            d['name'] = student.name
            d['amount'] = fee_dict.get(new_returning)
            l += d,
        csv_name = self.dumpListToCSV(l,'payments')
        logger.info('%s dumped payments to %s' % (member,export_file))
    ###)

    security.declarePublic('dumpListToCSV') ###(
    def dumpListToCSV(self,l,filename,fields=None):
        """dump a list of dicts to a CSV file"""
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
        if fields is None:
            fields = l[0].keys()
        headline = ','.join(fields)
        out = open(export_file,"wb")
        out.write(headline +'\n')
        out.close()
        out = open(export_file,"a")
        csv_writer = csv.DictWriter(out,fields,)
        csv_writer.writerows(l)
        return export_file
    ###)

    security.declareProtected(ManagePortal, 'listMembers') ###(
    def listMembers(self):
        "list all members"
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        logger = logging.getLogger('WAeUPTool.listMembers')
        if str(member) not in ('admin','joachim'):
            logger.info('%s tried to list members' % (member))
            return None
        members = self.portal_directories.members
        all = members.listEntryIdsAndTitles()
        l = []
        for user_id,name in all:
            d = {}
            d['user_id'] = user_id
            d['name'] = name
            d['pw'] = getattr(getattr(members,user_id),'password')
            d['email'] = getattr(getattr(members,user_id),'email')
            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
            l += d,
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
        logger.info('%s dumped memberlist to %s' % (member,export_file))
        fields = l[0].keys()
        headline = ','.join(fields)
        out = open(export_file,"wb")
        out.write(headline +'\n')
        out.close()
        out = open(export_file,"a")
        csv_writer = csv.DictWriter(out,fields,)
        csv_writer.writerows(l)
    ###)

    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
    def removeDeletedDocIds(self, max=1000):
        """
        remove deleted docids from repository commit after max
        """
        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
        repository = getToolByName(self, 'portal_repository')
        pxtool = getToolByName(self, 'portal_proxies')
        logger.info('start')
        pxtool_infos = pxtool.getRevisionsUsed()
        logger.info('found  %d used revisions ' % (len(pxtool_infos)))

        nb_revs = 0
        docids_d = {} # all docids
        unused_docids_d = {} # all docids that are unused
        ids_unused_revs_docids = [] # ids for revs of unused docids
        unused_ids = [] # ids for unused revs
        total = 0
        idlist = repository.objectIds()
        to_delete = 0
        found = False
        for id in idlist:
            docid, rev = repository._splitId(id)
            if docid is None:
                logger.info("invalid doc_id %s" % docid)
                continue
            nb_revs += 1
            if not pxtool_infos.has_key(docid):
                found = True
                to_delete += 1
                unused_ids.append(id)
            elif not pxtool_infos[docid].has_key(rev):
                found = True
                to_delete += 1
                unused_ids.append(id)
            if found and not to_delete % max:
                found = False
                #import pdb;pdb.set_trace()
                repository.manage_delObjects(unused_ids)
                transaction.commit()
                logger.info('removed %d total %d unused docids ' % (max,to_delete))
        else:
            if unused_ids:
                repository.manage_delObjects(unused_ids)
                transaction.commit()
        logger.info('finished removing %d unused docids ' % (to_delete))


###)

    security.declareProtected(View,'getCredential') ###(
    def getCredential(self,student_id):
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if not self.isStaff():
            mtool = self.portal_membership
            member = mtool.getAuthenticatedMember()
            logger = logging.getLogger('WAeUPTool.getCredential')
            logger.info('%s tried to access password of %s' % (member,student_id))
            return None
        if student_entry is None:
            return None
        return getattr(student_entry,"password","not set")
    ###)

    security.declarePublic('checkPassword') ###(
    def checkPassword(self,student_id,password):
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return False
        return getattr(student_entry,"password","not set") == password
    ###)

    security.declarePublic('checkGenericPassword') ###(
    def checkGenericPassword(self,member_id):
        member_entry = getattr(self.portal_directories.members,member_id,None)
        if member_entry is None:
            return False
        ltool = getToolByName(self, 'portal_layouts')
        unsecure_words = ltool._getOb('members')['w__password'].check_words
        password = getattr(member_entry,"password","not set")
        is_unsecure = password in unsecure_words
        if is_unsecure:
            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
        return is_unsecure
    ###)

    security.declareProtected(ModifyPortalContent,'editPassword') ###(
    def editPassword(self,student_id,password):
        "edit a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return
        setattr(student_entry,'password',password)
    ###)

    security.declareProtected(ModifyPortalContent,'doCommit') ###(
    def doCommit(self,logger=None):
        "commit some transactions"
        transaction.commit()
    ###)

    security.declarePublic('loadStudentFoto') ###(
    def loadStudentFoto(self,student,filename,folder):
        "return a student passport picture"
        #import pdb;pdb.set_trace()
        picture ="%s/import/%s/%s" % (i_home,folder,filename)
        student_id = student.getId()
        images_dir = getImagesDir(student_id)
        if not os.path.exists(images_dir):
            os.mkdir(images_dir)
        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
        for extension in ('.jpg','.JPG'):
            fullname = "%(picture)s%(extension)s" % vars()
            if os.path.exists(fullname):
                copy2(fullname,image_name)
                return "successfully copied passport picture"
        return "passport picture not found: %s.jpg or .JPG" % picture
    ###)

    def old____loadStudentFoto(self,student,filename,folder): ###(
        "return a student passport picture"
        app = student.application
        app_doc = app.getContent()
        #clear = student.clearance
        #clear_doc = clear.getContent()
        #matric_no = clear_doc.matric_no.upper()
        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
        #import pdb;pdb.set_trace()
        if os.path.exists(picture1):
            file = open(picture1)
        elif os.path.exists(picture2):
            file = open(picture2)
        else:
            return "passport picture not found %s" % picture1
        reopened = False
        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
            self.portal_workflow.doActionFor(app,'open')
            reopened = True
        outfile = file.read()
        app_doc.manage_addFile('passport',
                               file=outfile,
                               title="%s.jpg" % filename)
        if reopened:
            self.portal_workflow.doActionFor(app,'close')
        return "successfully loaded passport picture"
    ###)

    security.declareProtected(ModifyPortalContent,'createOne') ###(
    def createOne(self,students_folder,student_brain,letter,commit=False):
        sid = self.waeup_tool.generateStudentId(letter)
        students_folder.invokeFactory('Student', sid)
        student = getattr(students_folder,sid)
        self.portal_workflow.doActionFor(student,'return')
        student.manage_setLocalRoles(sid, ['Owner',])
        matric_no = student_brain.matric_no
        jamb_reg_no = student_brain.Entryregno
        self.students_catalog.addRecord(id = sid,
                                           matric_no = matric_no,
                                           jamb_reg_no = jamb_reg_no,
                                           sex = student_brain.Sex == "F",
                                           name = "%s %s %s" % (student_brain.Firstname,
                                                                student_brain.Middlename,
                                                                student_brain.Lastname)
                                        )
        if commit:
            transaction.commit()
        return sid,jamb_reg_no
    ###)

    security.declareProtected(ModifyPortalContent,'addStudent') ###(
    def addStudent(self,dict):
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        #from pdb import set_trace; set_trace()
        d = {}
        #d['jamb_sex']  = 'M'
        #if dict.get('sex'):
        #    d['jamb_sex']  = 'F'

        entry_session = dict.get('entry_session')
        if entry_session == self.getSessionId()[0]:
            wfaction = 'admit'
            wft = 'wf_transition_admit'
            password = None
        else:
            wfaction = 'return'
            wft = 'wf_transition_return'
            password = self.generatePassword()
            self.makeStudentMember(sid,password)

        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = dict.get(field,'')
            sub_doc.edit(mapping = d)
            new_state = f2t[pt][wft]
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        self.portal_workflow.doActionFor(student_obj,wfaction)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,password
    ###)

    security.declarePublic('getCertificateBrain') ###(
    def getCertificateBrain(self,cert_id):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"Certificate",
                                      'id': cert_id})
        if res:
            return res[0]
        return None
    ###)

    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
    def get_csv_filenames(self):
        "do it"
        files = [file for file in os.listdir("%s/import/" % (i_home))
                 if file.endswith('.csv') and file.find('imported') == -1]
        return files
    ###)

    security.declarePublic('findStudentByMatricelNo') ###(
    def findStudentByMatricelNo(self,matric_no):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"StudentClearance",
                                 'SearchableText': matric_no})
        if res:
            return res[0]
        return None
    ###)

    security.declarePublic('makeStudentMember') ###(
    def makeStudentMember(self,sid,password='uNsEt'):
        """make the student a member"""
        membership = self.portal_membership
        membership.addMember(sid,
                             password ,
                             roles=('Member',
                                     'Student',
                                     ),
                             domains='',
                             properties = {'memberareaCreationFlag': False,
                                           'homeless': True},)
        member = membership.getMemberById(sid)
        self.portal_registration.afterAdd(member, sid, password, None)
        #self.manage_setLocalRoles(sid, ['Owner',])
    ###)

    security.declareProtected(View,'makeStudentData') ###(
    def makeStudentData(self,student_id,email=None,phone_nr=None):
        "create Datastructure for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentData')
        students_folder = self.portal_url.getPortalObject().campus.students
        #res = self.students_catalog(id=student_id)
        #if res:
        #    st = res[0]
        #res = self.returning_import(matric_no = st.matric_no)
        res = self.returning_import(id = student_id)
        if res:
            student = res[0]
        else:
            logger.info('Id %s not found in returning_import' % student_id)
            return
        logger.info('%s creates data structure' % student_id)
        s_results = self.results_import(matric_no = student.matric_no)
        if s_results:
            lnr = self.getLevelFromResultsCosCode(s_results)
            level = "%d00" % lnr
            verdict,eligible = self.getVerdict(s_results[0].Verdict)
            #if eligible:
            #    level = "%d00" % (lnr + 1)
        else:
            logger.info('matric_no %s not found in results_import' % student.matric_no)
            level = ''
            verdict = ''
        #student should not be allowed to perform this transition
        #wftool = self.portal_workflow
        #wftool.doActionFor(student,'return')
        certcode_org = student.Coursemajorcode
        certcode = makeCertificateCode(certcode_org)
        certificate_brain = self.getCertificateBrain(certcode)
        if not certificate_brain:
            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
            logger.info(em)
        matric_no = student.matric_no
        sid = student_id
        student_obj = getattr(students_folder,sid)
        student_obj.invokeFactory('StudentApplication','application')
        application = student_obj.application
        self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        student_obj.invokeFactory('StudentPersonal','personal')
        da['jamb_reg_no'] = student.Entryregno
        em = self.getEntryMode(student.Entryregno)
        da['entry_mode'] = em
        personal = student_obj.personal
        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        student_obj.invokeFactory('StudentClearance','clearance')
        clearance = student_obj.clearance
        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['matric_no'] = matric_no
        state = student.State
        lga = student.LGA
        if state and lga:
            lga =  state + ' / ' + lga
        else:
            lga = "None"
        da['jamb_lga'] = dc['lga'] = lga
        da['app_email'] = dp['email'] = email
        da['app_mobile'] = dp['phone'] = phone_nr
        dp['firstname'] = student.Firstname
        dp['middlename'] = student.Middlename
        dp['lastname'] = student.Lastname
        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
        da['jamb_sex'] = student.Sex
        dp['sex'] = student.Sex == 'F'
        dp['perm_address'] = student.Permanent_Address
        application.getContent().edit(mapping=da)
        self.portal_workflow.doActionFor(application,'close',dest_container=application)
        personal.getContent().edit(mapping=dp)
        clearance.getContent().edit(mapping=dc)
        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
        #
        # Study Course
        #
        student_obj.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_obj.study_course
        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = certcode
        dsc['current_level'] = level
        dsc['current_verdict'] = verdict
        dsc['current_mode'] = em
        dsc['current_session'] = '05'
        studycourse.getContent().edit(mapping=dsc)
        #
        # Level
        #
##        l = getattr(studycourse,level,None)
##        if l is None:
##            studycourse.invokeFactory('StudentStudyLevel', level)
##            l = getattr(studycourse, level)
##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
##            l.getContent().edit(mapping={'Title': "Level %s" % level})
###)

    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
    def admitOneStudent(self,brain,entry_session,pin_password):
        "create Datastructure for an admitted Student"
        #import pdb;pdb.set_trace()
        if not hasattr(self,"_v_certificates"):
            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
        students_folder = self.portal_url.getPortalObject().campus.students
        logger = logging.getLogger('WAeUPTool.admitOneStudent')
        if brain.status != "admitted":
            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
            return
        pin_parts = brain.pin.split('-')
        if pin_parts and len(pin_parts) != 3:
            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
            return
        if brain.course_admitted not in self._v_certificates:
            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
            return
        if brain.sex not in (True,False):
            logger.info('sex of %s not available' % (brain.reg_no))
            return
        student_id = self.generateStudentId('?')
        students_folder.invokeFactory('Student', student_id)
        student_object = getattr(students_folder,student_id)
        if pin_password:
            password = pin_parts[2]
            self.makeStudentMember(student_id,password = password)
        student_object.manage_setLocalRoles(student_id, ['Owner',])
        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
        #
        # application
        #
        student_object.invokeFactory('StudentApplication','application')
        application = student_object.application
        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        da['jamb_reg_no'] = brain.reg_no

        sex = 'M'
        if brain.sex:
            sex = 'F'
        da['jamb_sex'] = sex
        da['jamb_age'] = brain.jamb_age
        #da['app_ac_pin'] = brain.pin
        da['jamb_lga'] = brain.jamb_lga
        da['jamb_state'] = brain.jamb_state
        da['jamb_score'] = brain.aggregate
        da['app_email'] = brain.email
        da['app_mobile'] = brain.phone
        if brain.entry_mode:
            da['entry_mode'] = brain.entry_mode
        elif brain.screening_type == 'pume':
            da['entry_mode'] = 'ume_ft'
        elif brain.screening_type == 'pde':
            da['entry_mode'] = 'de_ft'
        elif brain.screening_type == 'pce':
            da['entry_mode'] = 'pce'
        elif brain.screening_type == 'prence':
            da['entry_mode'] = 'prence'
        else:
            da['entry_mode'] = ''
        da['entry_session'] = entry_session
        da['jamb_lastname'] = brain.lastname
        da['jamb_middlename'] = brain.middlenames   # different field names!
        da['jamb_firstname'] = brain.firstname
        da['screening_application_date'] = brain.application_date
        da['date_of_birth'] = brain.date_of_birth
        da['jamb_first_cos'] = brain.course1
        da['jamb_second_cos'] = brain.course2
        da['course3'] = brain.course3
        da['screening_type'] = brain.screening_type
        da['screening_score'] = brain.screening_score
        da['screening_date'] = brain.screening_date
        da['hq_type'] = brain.hq_type
        da['hq_grade'] = brain.hq_grade
        da['aos'] = brain.aos

        application.getContent().edit(mapping=da)
        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
        #
        # personal
        #
        student_object.invokeFactory('StudentPersonal','personal')
        personal = student_object.personal
        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        dp['sex'] = brain.sex
        dp['email'] = brain.email
        dp['phone'] = brain.phone
        dp['lastname'] = brain.lastname
        dp['middlename'] = brain.middlenames   # different field names!
        dp['firstname'] = brain.firstname
        personal.getContent().edit(mapping=dp)
        #
        # clearance
        #
        student_object.invokeFactory('StudentClearance','clearance')
        clearance = student_object.clearance
        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['lga'] = brain.lga
        dc['birthday'] = brain.date_of_birth
        clearance.getContent().edit(mapping=dc)
        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
        #
        # study Course
        #
        student_object.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_object.study_course
        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = brain.course_admitted
        dsc['current_verdict'] = ''
        dsc['current_mode'] = da['entry_mode']
        if da['entry_mode'].startswith('de'):
            dsc['current_level'] = '200'
        else:
            dsc['current_level'] = '100'
        dsc['current_session'] = entry_session
        studycourse.getContent().edit(mapping=dsc)
        #
        # payments folder
        student_object.invokeFactory('PaymentsFolder','payments')
        payments = getattr(student_object,'payments')
        dpay = {}
        dpay['Title'] = 'Payments'
        payments.getContent().edit(mapping=dpay)
        self.portal_workflow.doActionFor(payments,'open')
        #
        # passport foto
        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
                                                              brain.screening_type,
                                                              brain.reg_no)
        images_dir = getImagesDir(student_id)
        #images_dir = os.path.join("%s" % images_base,student_id)
        letter_dir,student_dir = os.path.split(images_dir)
        if not os.path.exists(letter_dir):
            os.mkdir(letter_dir)
        if not os.path.exists(images_dir):
            os.mkdir(images_dir)
        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
        if os.path.exists(app_picture):
            copy2(app_picture,image_name)
        else:
            logger.info('passport of %s/%s not found: %s' % (student_id,
                                                             brain.reg_no,
                                                             app_picture))

        return student_id
    ###)

    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
    def makeStudentLevel(self,student_id):
        "create the StudyLevel for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
        students_folder = self.portal_url.getPortalObject().campus.students
        res = self.students_catalog(id=student_id)
        if res:
            st = res[0]
        course = st.course
        matric_no = st.matric_no
        level = st.level
        res = self.results_import(matric_no = matric_no)
        if res:
            results = res
        logger.info('%s creating Level %s' % (student_id,level))
        #
        # Level
        #
        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        studycourse = getattr(student_obj,"study_course",None)
        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
        l = getattr(studycourse,level,None)
        if l is None:
            studycourse.invokeFactory('StudentStudyLevel', level)
            l = getattr(studycourse, level)
            self.portal_workflow.doActionFor(l,'open',dest_container=l)
            l.getContent().edit(mapping={'Title': "Level %s" % level})
        ###)

    security.declarePublic('getHallInfo') ###(
    def getHallInfo(self,bed):
        """return Hall Info"""
        info = {}
        hall,block,room,letter = bed.split('_')
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
        if res and len(res) == 1:
            hall_brain = res[0]
            hall_doc = hall_brain.getObject().getContent()
        else:
            return info
        info['hall_title'] = hall_brain.Title
        info['maintenance_code'] = hall_doc.maintenance_code
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
        batch_doc = None
        for brain in res:
            if brain.id.startswith(info['maintenance_code']):
                batch_doc = brain.getObject().getContent()
                break
        if batch_doc is None:
            info['maintenance_fee'] = ''
        else:
            info['maintenance_fee'] = batch_doc.cost
        return info
    ###)

    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
    def removePictureFolder(self,student_id):
        """remove picture_folder by renaming it"""
        path = 'images'
        picture_path = os.path.join(i_home,path,student_id)
        if not os.path.exists(picture_path):
            return False
        os.rename(picture_path,picture_path + "_removed")
        return True
    ###)

    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
    def restorePictureFolder(self,student_id):
        """restore picture_folder by renaming it"""
        path = 'images'
        picture_path = os.path.join(i_home,path,student_id)
        if not os.path.exists(picture_path + "_removed"):
            return False
        os.rename(picture_path + "_removed",picture_path)
        return True
    ###)

    security.declarePublic('picturesExist') ###(
    def picturesExist(self, ids,student_id=None):
        """check if pictures exist in the filesystem"""
        if student_id is None:
            student_id = self.getStudentId()
        if student_id is None:
            return False
        picture_path = getImagesDir(student_id)
        #picture_path = os.path.join(images_base,student_id)
        if not os.path.exists(picture_path):
            return False
        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
        return set(ids).issubset(set(pictures))
    ###)

    security.declarePublic('picturesList') ###(
    def picturesList(self):
        """check if pictures exist in the filesystem"""
        path = 'images'
        student_id = self.getStudentId()
        #picture_path = os.path.join(i_home,path,student_id)
        picture_path = getImagesDir(student_id)
        if not os.path.exists(picture_path):
            return []
        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
    ###)

    security.declarePublic('showFsPicture') ###(
    def showFsPicture(self,path):
        """return a picture from the filesystem"""
        #picture_path = os.path.join(i_home,path)
        picture_path = os.path.join(images_base,path)
        response = self.REQUEST.RESPONSE
        #import pdb;pdb.set_trace()
        registry = getToolByName(self, 'mimetypes_registry')
        mimetype = str(registry.lookupExtension(path.lower()) or
                    registry.lookupExtension('file.bin'))
        if os.path.exists(picture_path):
            response.setHeader('Content-type',mimetype)
            return open(picture_path).read()
        picture_path = os.path.join(i_home,'import',path)
        if os.path.exists(picture_path):
            return open(picture_path).read()
    ###)

    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
    def deleteAllCourses(self,department="All"):
        ''' delete the courses'''
        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        if str(member) not in ("henrik","joachim"):
            return "not possible"
        if department == "All":
            res = self.portal_catalog({'meta_type': 'Department'})
        if len(res) < 1:
            return "No Departments found"

        deleted = []
        for dep in res:
            cf = dep.getObject().courses
            if cf:
                cf.manage_delObjects(ids=cf.objectIds())
                deleted.append("deleted Courses in %s" % dep.getId)
        return "\r".join(deleted)
    ###)

    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
    def getLogfileLines(self,filename="event.log",numlines=20):
        """Get last NUMLINES lines of logfile FILENAME.

        Return last lines' of a file in the instances logfile directory as
        a list. The number of returned lines equals `numlines' or less. If
        less than `numlines' lines are available, the whole file ist
        returned. If the file can not be opened or some other error
        occurs, empty list is returend.
        """
        result = []
        lines_hit = 0

        # We only handle files in instances' log directory...
        logpath = os.path.join(i_home, "log")
        filename = str(os.path.abspath( os.path.join( logpath, filename )))
        if not filename.startswith( logpath ):
            # Attempt to access file outside log-dir...
            return []

        try:
            fd = file( filename, "rb" )
        except IOError:
            return []
        if not fd:
            return []

        if os.linesep == None:
            linesep = '\n'
        else:
            linesep = os.linesep

        # Try to find 'numlines' times a lineseparator, searching from end
        # and moving to the beginning of file...
        fd.seek( 0, 2) # Move to end of file...
        while lines_hit < numlines:
            if fd.read(1) == linesep[-1]: # This moves filedescriptor
                                          # one step forward...
                lines_hit += 1
            try:
                fd.seek( -2, 1) # Go two bytes back from current pos...
            except IOError:
                # We cannot go back two bytes. Maybe the file is too small...
                break
        fd.seek(2,1)

        # Read all lines from current position...
        result = fd.readlines()
        # Remove line endings...
        result = [x.strip() for x in result]
        fd.close()
        return result
    ###)

    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
    def getCallbacksFromLog(self,filename):
        """fix Online Payment Transactions from Z2.log entries"""
        import transaction
        import random
        from cgi import parse_qs
        from urlparse import urlparse
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
        data = re.compile(s)
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        #name = 'epaymentsuccessful_z2log2'
        name = filename
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
        try:
            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
        except:
            logger.error('Error reading %s' % name)
            return
        tas = []
        for line in transactions:
            dict = {}
            items = data.search(line)
            dict['idict'] = idict = items.groupdict()
            #print idict
            #from pdb import set_trace;set_trace()
            urlparsed = urlparse(idict['get'][4:])
            #print urlparsed
            path = urlparsed[2].split('/')
            dict['student_id'] = student_id = path[8]
            dict['payment_id'] = payment_id = path[10]
            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
            tas.append(dict)
            tr_count += 1
        return tas
    ###)

    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
    def importOnlinePaymentTransactions(self):
        """load Online Payment Transactions from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        opt = self.online_payments_import
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'OnlineTransactions'
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
        try:
            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for pay_transaction in transactions:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
                format_error = format + ',"%(Error)s"'
            data = {}

            # format of the first file sent by Tayo
            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = pay_transaction['Resp Code']
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the second file sent by Tayo
            #data['datetime'] = date = 0
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = '00'
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the third file sent by Kehinde
            data['datetime'] = date = 0
            data['student_id'] = student_id = pay_transaction['customer_id']
            data['order_id'] = order_id = pay_transaction['merchant_reference']
            data['response_code'] = response_code = '00'
            data['amount'] = amount = pay_transaction['Amount']

            dup = False
            if response_code == "12":
                continue
            try:
                opt.addRecord(**data)
            except ValueError:
                dup = True
            #from pdb import set_trace;set_trace()
            if dup:
                if response_code == "00":
                    try:
                        opt.modifyRecord(**data)
                    except:
                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
                        continue
                else:
                    pay_transaction['Error'] = "Duplicate order_id"
                    no_import.append( format_error % pay_transaction)
                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
                    continue
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions committed\n' % (tr_count)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
    def mass_create_faculty(self,mapping):
        "create a faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['code']
        if getattr(academics_folder,fid,None) is not None:
            return '', "Faculty with ID: %s exists" % fid
        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
        try:
            academics_folder.invokeFactory('Faculty', fid)
        except BadRequest,E:
            return '', "%s" % E
        f = getattr(academics_folder,fid,None)
        f.getContent().edit(mapping=mapping)
        return fid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
    def mass_edit_faculty(self,mapping):
        "edit a faculty"
        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['code']
        f = getattr(academics_folder,fid,None)
        if f is None:
            return '', "Faculty with ID: %s does not exist" % fid
        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
        f.getContent().edit(mapping=mapping)
        return fid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
    def mass_create_department(self,mapping):
        "create a department in the correct faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_department')
        fid = mapping['faculty_code']
        if getattr(self,'_v_faculties',None) is None:
            res = self.portal_catalog(portal_type = "Faculty")
            self._v_faculties = {}
            for f in res:
                self._v_faculties[f.getId] = f.getObject()
        f = self._v_faculties.get(fid,None)
        if f is None:
            return '', "No Faculty with ID: %s" % fid
        else:
            did = mapping.get('code')
            d = getattr(f,did,None)
            if d is None or d.portal_type == "Faculty":
                logger.info('Creating Department %(code)s, %(title)s' % mapping)
                try:
                    f.invokeFactory('Department', did)
                except BadRequest,E:
                    return '', "%s" % E
                d = getattr(f,did)
                d.invokeFactory('CoursesFolder','courses')
                courses = getattr(d,'courses')
                dict = {'Title': 'Courses'}
                courses.getContent().edit(mapping=dict)
                d.invokeFactory('CertificatesFolder','certificates')
                certificates = getattr(d,'certificates')
                dict = {'Title': 'Certificates'}
                certificates.getContent().edit(mapping=dict)
            d.getContent().edit(mapping=mapping)
        return did,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
    def mass_edit_department(self,mapping):
        "create a department in the correct faculty"
        logger = logging.getLogger('WAeUPTool.mass_create_department')
        academics_folder = self.portal_url.getPortalObject().campus.academics
        fid = mapping['faculty_code']
        did = mapping.get('code')
        try:
            d = getattr(getattr(academics_folder,fid),did,None)
        except KeyError:
            return '', "Department %s or Faculty %s wrong" % (did,fid)
        else:
            if d is None or d.portal_type == "Faculty":
                logger.info('Editing Department %(code)s, %(title)s' % mapping)
            d.getContent().edit(mapping=mapping)
        return did,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
    def mass_create_course(self,mapping):
        #import pdb;pdb.set_trace()
        if getattr(self,'_v_course_list',None) is None:
            self._v_course_list = []
        if getattr(self,'_v_departments',None) is None:
            res = self.portal_catalog(portal_type = "Department")
            self._v_department_courses = {}
            for d in res:
                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
        did = mapping['department_code']
        d = self._v_department_courses.get(did,None)
        if d is None:
            return '', "No Department with ID: %s" % did
        course_id = mapping.get('code')
        if course_id in self._v_course_list:
            return '', "Duplicate Course ID: %s" % did
        c = getattr(d,course_id,None)
        if c is not None:
            return '', "Duplicate Course ID: %s" % did
        try:
            d.invokeFactory('Course', course_id)
        except BadRequest,E:
            return '', "%s" % E
        self._v_course_list.append(course_id)
        c = getattr(d,course_id)
        c.getContent().edit(mapping=mapping)
        return course_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
    def mass_edit_course(self,mapping):
        #import pdb;pdb.set_trace()
        course_id = mapping.get('code')
        res = self.portal_catalog(id=course_id)
        if not res:
            return '', "No Course with ID: %s" % course_id
        c = res[0].getObject()
        c.getContent().edit(mapping=mapping)
        return course_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
    def mass_create_certificate(self,mapping):
        if getattr(self,'_v_certificate_list',None) is None:
            self._v_certificate_list = []
        if getattr(self,'_v_department_certificates',None) is None:
            res = self.portal_catalog(portal_type = "Department")
            self._v_department_certificates = {}
            for d in res:
                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
        did = mapping['department_code']
        d = self._v_department_certificates.get(did,None)
        if d is None:
            return '', "No Department with ID: %s" % did
        certificate_id = mapping.get('code')
        if certificate_id in self._v_certificate_list:
            return '', "Duplicate Certificate ID: %s" % did
        c = getattr(d,certificate_id,None)
        if c is not None:
            return '', "Duplicate Certificate ID: %s" % did
        try:
            d.invokeFactory('Certificate', certificate_id)
        except BadRequest,E:
            return '', "%s" % E
        self._v_certificate_list.append(certificate_id)
        c = getattr(d,certificate_id)
        c.getContent().edit(mapping=mapping)
        return certificate_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
    def mass_edit_certificate(self,mapping):
        #import pdb;pdb.set_trace()
        certificate_id = mapping.get('code')
        res = self.portal_catalog(id=certificate_id)
        if not res:
            return '', "No Certificate with ID: %s" % did
        c = res[0].getObject()
        c.getContent().edit(mapping=mapping)
        return certificate_id,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
    def mass_create_application(self,mapping):
        #import pdb;pdb.set_trace()
        reg_no = mapping.get('reg_no')
        try:
            self.applicants_catalog.addRecord(**mapping)
        except ValueError:
            return '', "applicant record with reg_no %s already exists" % reg_no
        return reg_no,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
    def mass_edit_application(self,mapping):
        #import pdb;pdb.set_trace()
        reg_no = mapping.get('reg_no')
        status = mapping.get('status')
        res = self.applicants_catalog(reg_no = reg_no)
        if len(res):
            if res[0].status == 'created' and status != 'created':
                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
            elif status == 'created' and res[0].status != 'created':
                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
            else:
                self.applicants_catalog.modifyRecord(**mapping)
                return reg_no,''
        else:
            return '', "applicant record with reg_no %s does not exist" % reg_no

    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
    def mass_create_course_result(self,mapping):
        #import pdb;pdb.set_trace()
        students_folder = self.portal_url.getPortalObject().campus.students
        if getattr(self,'_v_courses',None) is None:
            res = self.courses_catalog()
            self._v_courses = {}
            for brain in res:
                self._v_courses[brain.code] = brain
        if getattr(self,'_v_level_created',None) is None:
            self._v_level_created = []
        course_id = mapping.get('code')
        if course_id not in self._v_courses.keys():
            return '', "No course with ID: %s" % course_id
        id_key = ''
        for id_key in ('student_id','matric_no'):
            id_field = mapping.get(id_key,None)
            if id_field is not None:
                student_id = id_field
                break
        query = Eq(id_key,id_field)
        res = self.students_catalog.evalAdvancedQuery(query)
        if not res:
            return '', "no student with %(id_key)s %(id_field)s" % vars()
        student_rec = res[0]
        if id_field != "student_id":
            mapping['student_id'] = student_rec.id
        student_id = student_rec.id
        level_id = mapping['level_id']
        code = mapping['code']
        if student_id not in self._v_level_created:
            try:
                context = getattr(getattr(students_folder,
                                          "%(student_id)s" % vars()),
                                  'study_course')
            except:
                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
            if level_id not in context.objectIds():
                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
                level = getattr(context,"%(level_id)s" % vars())
                self.portal_workflow.doActionFor(level,'open')
                # the session string must not be copied into the level object
                current_verdict = getattr(student_rec,'current_verdict','')
                current_session = getattr(student_rec,'current_session','')
                if current_verdict and student_rec.current_level == level_id:
                    level.getContent().edit(mapping={'verdict': "%s" %
                                                     current_verdict,
                                                     'session': "%s" %
                                                     current_session,
                                                    })
                    self.portal_workflow.doActionFor(level,'close')
            self._v_level_created += student_id,
        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
        for k in ('semester','credits',):
            mapping[k] = getattr(self._v_courses[course_id],k)
        try:
            self.course_results.addRecord(**mapping)
        except ValueError:
            return '', "course result already exists: %s" % key
        return key,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
    def mass_edit_course_result(self,mapping):
        #import pdb;pdb.set_trace()
        # if getattr(self,'_v_courses',None) is None:
        #     res = self.courses_catalog()
        #     self._v_courses = {}
        #     for brain in res:
        #         self._v_courses[brain.code] = brain
        # course_id = mapping.get('code')
        # if course_id not in self._v_courses.keys():
        #     return '', "No course with ID: %s" % did
        id_key = ''
        for id_key in ('student_id','matric_no'):
            id_field = mapping.get(id_key,None)
            if id_field is not None:
                student_id = id_field
                break
        query = Eq(id_key,id_field)
        res = self.students_catalog.evalAdvancedQuery(query)
        if not res:
            return '', "No student with %(id_field)s: %(id_key)s" % vars()
        if id_field != "student_id":
            mapping['student_id'] = res[0].id
        if not res:
            return '', "no student with %(id_key)s %(id_field)s" % vars()
        student_rec = res[0]
        student_id = student_rec.id
        level_id = mapping['level_id']
        code = mapping['code']
        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
        # for k in ('semester',):
        #     mapping[k] = getattr(self._v_courses[course_id],k)
        try:
            self.course_results.modifyRecord(**mapping)
        except KeyError:
            return '', "No course result to edit: %s" % key
        return key,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
    def mass_create_certificate_course(self,mapping):
        if getattr(self,'_v_courses',None) is None:
            res = self.courses_catalog()
            self._v_courses= [course.code for course in res]
        if getattr(self,'_v_ceritficates',None) is None:
            res = self.portal_catalog(portal_type = "Certificate")
            self._v_certificates = {}
            for cert in res:
                self._v_certificates[cert.getId] = cert.getObject()
        certificate_course_id = mapping.get('code')
        if certificate_course_id not in self._v_courses:
            return '', "No Course with ID: %s" % certificate_course_id
        cert_id = mapping['certificate_code']
        cert = self._v_certificates.get(cert_id,None)
        if cert is None:
            return '', "No Certificate with ID: %s" % cert_id
        level_id = mapping.get('level')
        level = getattr(cert,level_id,None)
        if level is None:
            cert.invokeFactory('StudyLevel', level_id)
            level = getattr(cert,level_id,None)
        elif hasattr(level,certificate_course_id):
            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
            (certificate_course_id,cert_id,level_id)
        level.invokeFactory('CertificateCourse', certificate_course_id)
        c = getattr(level,certificate_course_id)
        c.getContent().edit(mapping=mapping)
        return certificate_course_id,''
    ###)

    field2types_student = {   ###(
                      'StudentApplication':
                          {'id': 'application',
                           'title': 'Application Data',
                           'wf_transition_return': 'close',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('jamb_reg_no',
                              'entry_mode',
                              'entry_session',
                              'jamb_score',
                              'app_email',
                              'jamb_age',
                              'jamb_state',
                              'jamb_lga',
                              'jamb_sex',
                              )
                              },
                      #'StudentPume':
                      #    {'id': 'pume',
                      #     'title': 'Pume Data',
                      #     'wf_transition_return': 'close',
                      #     'wf_transition_admit': 'close',
                      #     'fields':
                      #       ('pume_score',
                      #        )
                      #        },
                      'StudentClearance':
                          {'id': 'clearance',
                           'title': 'Clearance/Eligibility Record',
                           'wf_transition_return': 'close',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('matric_no',
                              'nationality',
                              'lga',
                              'birthday',
                              )
                              },
                         'StudentPersonal':
                          {'id': 'personal',
                           'title': 'Personal Data',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('firstname',
                              'middlename',
                              'lastname',
                              'sex',
                              'email',
                              'phone',
                              'perm_address',
                              )
                              },
                         'StudentStudyCourse':
                          {'id': 'study_course',
                           'title': 'Study Course',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'remain',
                           'fields':
                             ('study_course',
                              'current_level',
                              'current_session',
                              'current_mode',
                              'current_verdict',
                              'previous_verdict',
                              )
                              },
                         # 'StudentStudyLevel':
                         #  {'id': 'current_level',
                         #   'title': '',
                         #   'wf_transition_return': 'open',
                         #   'wf_transition_admit': 'remain',
                         #   'fields':
                         #     ('verdict',
                         #      'session',
                         #      )
                         #      },
                         'PaymentsFolder':
                          {'id': 'payments',
                           'title': 'Payments',
                           'wf_transition_return': 'open',
                           'wf_transition_admit': 'open',
                           'fields':
                             ()
                              },
                         }
    ###)

    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
    def mass_create_student(self,mapping):
        "create a students record due import"
        logger = logging.getLogger('WAeUPTool.mass_create_student')
        students_folder = self.portal_url.getPortalObject().campus.students
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        if jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if res:
                return '',"jamb_reg_no exists"
        matric_no = mapping.get('matric_no',None)
        if matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if res:
                return '',"matric_no exists"
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        d = {}
        transition = mapping.get('reg_transition','admit')
        if transition not in ('admit','return'):
            return '',"no valid transition provided"
        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = mapping.get(field,'')

            if pt == "StudentApplication":
                #d['jamb_sex']  = 'M'
                #if mapping.get('sex'):
                #    d['jamb_sex']  = 'F'
                d['jamb_firstname'] = mapping.get('firstname',None)
                d['jamb_middlename'] = mapping.get('middlename',None)
                d['jamb_lastname'] = mapping.get('lastname',None)

            # if pt == "StudyCourse":
            #     for von,zu in (('entry_mode','current_mode'),
            #                    ('entry_session','current_session')):
            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
            #             d[zu] = mapping[von]
            sub_doc.edit(mapping = d)

            #import pdb;pdb.set_trace()
            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        self.portal_workflow.doActionFor(student_obj,transition)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,''
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
    def mass_edit_student(self,mapping):
        wftool = self.portal_workflow
        "edit a students record due import"
        logger = logging.getLogger('WAeUPTool.mass_edit_student')
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = mapping.get('id',None)
        jamb_reg_no = mapping.get('jamb_reg_no',None)
        matric_no = mapping.get('matric_no',None)
        editable_keys = mapping.keys()
        if sid:
            res = self.students_catalog(id = sid)
            if not res:
                return '',"no student with id %s" % sid
            if matric_no and res[0].matric_no and\
              matric_no != res[0].matric_no:
                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
            if jamb_reg_no and res[0].jamb_reg_no and\
              jamb_reg_no != res[0].jamb_reg_no:
                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
        elif jamb_reg_no:
            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
            if not res:
                return '',"no student with jamb_reg_no %s" % jamb_reg_no
            editable_keys.remove('jamb_reg_no')
        elif matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if not res:
                return '',"no student with matric_no %s" % matric_no
            editable_keys.remove('matric_no')

        ## included only to change wf state from admitted to returning
        #if res[0].review_state not in ('admitted','objection_raised'):
        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
        ## end inclusion

        sid = res[0].id
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        d = {}
        #import pdb;pdb.set_trace()
        any_change = False
        #special treatment for StudentStudyLevel
        d['verdict']  = mapping.get('current_verdict','')
        d['session']  = mapping.get('current_session','')
        current_level = mapping.get('current_level','')
        while d['session'] and d['verdict'] and current_level:
            sub_obj = getattr(student_obj,'study_course',None)
            if sub_obj is None:
                break
            level_obj = getattr(sub_obj,current_level,None)
            if  level_obj is None:
                break
            any_change = True
            level_obj.getContent().edit(mapping = d)
            try:
                wftool.doActionFor(level_obj,'close')
            except:
                pass
            break
        for pt in f2t.keys():
            #if pt == "StudentApplication":
            #    d['jamb_sex']  = 'M'
            #    if mapping.get('sex'):
            #        d['jamb_sex']  = 'F'
            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
            if intersect and pt not in ('StudentStudyLevel',):
                object_id = f2t[pt]['id']
                sub_obj = getattr(student_obj,object_id,None)
                if sub_obj is None:
                    try:
                        student_obj.invokeFactory(pt,object_id)
                    except:
                        continue
                    sub_obj = getattr(student_obj,object_id)
                    if f2t[pt]['title'] != '':
                        d['Title'] = f2t[pt]['title']
                sub_doc = sub_obj.getContent()
                for field in intersect:
                    changed = False
                    if getattr(sub_doc,field,None) != mapping.get(field,''):
                        any_change = True
                        changed = True
                        d[field] = mapping.get(field,'')
                    if changed:
                        sub_doc.edit(mapping = d)


        ## included only to change wf state from admitted to returning
        #    if res[0].review_state in ('admitted','objection_raised'):
        #        new_state = f2t[pt]['wf_transition_return']
        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
        #        if sub_obj and new_state != "remain":
        #            try:
        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        #            except:
        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
        #                pass
        #if res[0].review_state in ('admitted','objection_raised'):
        #    wfaction = 'return'
        #    try:
        #        self.portal_workflow.doActionFor(student_obj,wfaction)
        #        logger.info('%s, wf state changed' % sid)
        #        any_change = True
        #    except:
        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
        #        pass
        ## end inclusion


        if any_change:
            return sid,''
        else:
            return sid,'not modified'
    ###)

    security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
    def mass_edit_verdict(self,mapping):
        wftool = self.portal_workflow
        "edit a students verdict"
        logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
        students_folder = self.portal_url.getPortalObject().campus.students
        student_id = mapping.get('id',None)
        matric_no = mapping.get('matric_no',None)
        editable_keys = mapping.keys()
        if student_id:
            student_record = self.students_catalog.getRecordById(student_id)
            if student_record is None:
                return '',"no student with id %s" % student_id
            if matric_no and student_record.matric_no and\
              matric_no != student_record.matric_no:
                logger.info("%s, old matric_no %s overwritten with %s" % (student_record.id,student_record.matric_no,matric_no))
        elif matric_no:
            res = self.students_catalog(matric_no = matric_no)
            if not res:
                return '',"no student with matric_no %s" % matric_no
            student_record = res[0]
            editable_keys.remove('matric_no')

        student_id = student_record.id
        student_review_state =  student_record.review_state
        if student_review_state not in ('courses_validated',):
            return '',"student %s in wrong review_state %s" % (student_id, student_review_state)
        student_obj = getattr(students_folder,student_id)
        f2t = self.field2types_student
        d = {}
        #import pdb;pdb.set_trace()
        any_change = False
        #special treatment for StudentStudyLevel
        current_session = mapping.get('current_session','')
        if current_session and student_record.session != current_session:
            return '','student_id: %s imported session %s does not match session %s' % (student_id,
                                                                                        current_session,
                                                                                        student_record.session)
                                                                                        
        current_level = mapping.get('current_level','')
        if current_level and student_record.level != current_level:
            return '','student_id %s: imported level %s does not match level %s' % (student_id,
                                                                                    current_level,
                                                                                    student_record.level)
        sub_obj = getattr(student_obj,'study_course',None)
        if sub_obj is None:
            return '','student_id %s: no study_course-object' % student_id
        level_obj = getattr(sub_obj,current_level,None)
        if  level_obj is None:
            return '','student_id %s: no study_level-object for level %s' % (student_id,
                                                                             current_level)
        verdict = d['verdict']  = mapping.get('current_verdict','')
        if verdict == student_record.verdict:
            return '','student_id %s: verdict already set to %s' % (student_id,
                                                                    verdict)
            
        level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
        if level_review_state != "closed":
            return '','student_id %s: level %s is not closed' % (student_id,
                                                                 current_level)
        
        level_obj.getContent().edit(mapping = d)
        try:
            wftool.doActionFor(level_obj,'close')
        except:
            pass
        return student_id,''
    ###)

    security.declareProtected(ModifyPortalContent,"importData")###(
    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
        """load data from CSV values"""
        import transaction
        import random

        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        logger = logging.getLogger('WAeUPTool.importData')
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 0
        total_imported = 0
        total_not_imported = 0
        total = 0
        iname = "import_%s" % name
        if iname in ('import_application','import_course_result',):
            commit_after = 2000
        else:
            commit_after = 100
        stool = getToolByName(self, 'portal_schemas')
        ltool = getToolByName(self, 'portal_layouts')
        schema = stool._getOb(iname)
        if schema is None:
            em = 'No such schema %s' % iname
            logger.error('No such schema %s' % iname)
            return em
        layout = ltool._getOb(iname)
        if layout is None:
            em = 'No such layout %s' % iname
            logger.error(em)
            return em
        validators = {}
        for widget in layout.keys():
            validators[widget] = layout[widget].validate
        mode = "create"
        if edit:
            mode = "edit"
        importer_name = "mass_%(mode)s_%(name)s" % vars()
        importer = getattr(self, '%s' % importer_name,None)
        if importer is None:
            em = 'No importer function %s' % importer_name
            logger.error(em)
            return em
        not_imported = []
        imported = []
        try:
            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
        except:
            em = 'Error reading %s.csv' % filename
            logger.error(em)
            return em
        for item in items:
            #import pdb;pdb.set_trace()
            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
            if start:
                start = False
                adapters = [MappingStorageAdapter(schema, item)]
                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
                import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
                diff2schema = set(import_keys).difference(set(schema.keys()))
                diff2layout = set(import_keys).difference(set(layout.keys()))
                if diff2schema:
                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
                    return em
                if mode == "create":
                    required_keys = [layout.getIdUnprefixed(id)
                                     for id,widget in layout.objectItems()
                                     if widget.is_required]
                    if not set(required_keys).issubset(set(import_keys)):
                        diff2import = set(required_keys).difference(set(import_keys))
                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
                        return em
                s = ','.join(['"%s"' % fn for fn in import_keys])
                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
                s = '"id",' + s
                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
                format_error = format + ',"%(Error)s"'
                format = '"%(id)s",'+ format

            dm = DataModel(item, adapters,context=self)
            ds = DataStructure(data=item,datamodel=dm)
            error_string = ""
            total += 1
            error_count = 0
            for k in import_keys:
                if not validators[k](ds,mode=mode):
                    if error_count:
                        error_string += ' ++ '
                    error_string += "%s: %s" % (k,
                                                  self.translation_service(ds.getError(k),
                                                                           ds.getErrorMapping(k)))
                    error_count += 1
            if not error_string:
                temp_item = item.copy()
                temp_item.update(dm)
                temp_item['id'],error = importer(temp_item)
                if error:
                    error_string += error
                else:
                    item = temp_item
            if error_string:
                item['Error'] = error_string
                not_imported.append(format_error % item)
                total_not_imported += 1
            else:
                em = format % item
                imported.append(em)
                tr_count += 1
                total_imported += 1
                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())

            if total and not total % commit_after:
                transaction.commit()
                if len(not_imported) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(not_imported) + '\n')
                    not_imported = []
                if len(imported) > 0:
                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                             '\n'.join(imported) + '\n')
                    imported = []
                em = '%d transactions committed\n' % (tr_count)
                regs = []
                logger.info(em)
                tr_count = 0
                #if total > 100:
                #    return
        if len(imported) > 0:
            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(imported))
        if len(not_imported) > 0:
            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
                                                '\n'.join(not_imported))
        em = "Finished import from %s.csv: %d imported, %d not imported (of total %d)" % (filename,total_imported,total_not_imported,total)
        logger.info(em)
        return em
    ###)

    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
    def moveImagesToFS(self,student_id="O738726"):
        "move the images to the filesystem"
        images_dir = getImagesDir(student_id)
        #images_dir = os.path.join("%s" % images_base,student_id)
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key,None)
                if not image or not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))
    ###)

    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
    def movePassportToFS(self,student_id="O738726"):
        "move the passports to the filesystem"
        images_dir = os.path.join("%s" % i_home,'passports')
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   #'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key)
                if not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))
    ###)

InitializeClass(WAeUPTool)