source: WAeUP_SRP/base/WAeUPTool.py @ 3255

# -*- mode: python; mode: fold; -*-
# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
# Author: Joachim Schmitz (js@aixtraware.de)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Id: WAeUPTool.py 3251 2008-02-28 09:52:42Z henrik $
"""The WAeUP Tool Box.
"""

from AccessControl import ClassSecurityInfo
from Acquisition import aq_inner
from Acquisition import aq_parent
from Globals import DTMLFile
from Globals import InitializeClass
from OFS.SimpleItem import SimpleItem
from zExceptions import BadRequest

from Products.CMFCore.utils import getToolByName
from Products.CPSSchemas.DataStructure import DataStructure
from Products.CPSSchemas.DataModel import DataModel
from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
from Products.CMFCore.ActionProviderBase import ActionProviderBase
from Products.CMFCore.permissions import View
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFCore.permissions import ManagePortal
from Products.CMFCore.utils import UniqueObject
from Products.CMFCore.URLTool import URLTool
from Products.CMFCore.utils import getToolByName
from Students import makeCertificateCode
from Globals import package_home,INSTANCE_HOME
from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
from WAeUPImport import CourseImport,CourseResultImport
from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
from utils import makeDigest
import DateTime,time
import logging
import transaction
import csv,re,os,sys
import md5
from shutil import copy2
from Products.AdvancedQuery import Eq, Between, Le,In

p_home = package_home(globals())
i_home = INSTANCE_HOME
images_base = os.path.join(i_home,"images")
EMPTY = 'XXX'

def getImagesDir(student_id):
    return os.path.join("%s" % images_base,student_id[0],student_id)

def getObject(object,name):
    if object.hasObject(name):
        return getattr(object,name)
    return None

class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
    """WAeUP tool"""

    id = 'waeup_tool'
    meta_type = 'WAeUP Tool'
    _actions = ()
    security = ClassSecurityInfo()
    security.declareObjectProtected(View)
    manage_options = ( ActionProviderBase.manage_options
                     + SimpleItem.manage_options
                     )

    security.declareProtected(View,'re_split') ###(
    def re_split(self,split_string,string):
        return re.split(split_string,string)
    ###)

    security.declareProtected(View,'difference') ###(
    def difference(self,l1,l2):
        return set(l1).difference(set(l2))
    ###)

    def rwrite(self,s): ###(
        response = self.REQUEST.RESPONSE
        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
        response.write("%s<br />\r\n" % s)
    ###)

    def addtodict(self,d,key,item): ###(
        d[key].append(item)
        return d[key]
    ###)

    def sleep(self,secs): ###(
        "sleep"
        import time
        time.sleep(secs)
        return

###)

    security.declareProtected(View,'updateRoleMappingsFor') ###(
    def updateRoleMappingsFor(self,wf_definition,ob):
        "do so for public"
        wf_def = getattr(self.portal_workflow,wf_definition)
        wf_def.updateRoleMappingsFor(ob)
    ###)

    security.declareProtected(View,'getStatesLgas') ###(
    def getStatesLgas(self):
        """return lga info"""
        voc = getattr(self.portal_vocabularies,'local_gov_areas')
        states = []
        lgas  = []
        d = {}
        wd = {}
        for k,v in voc.items():
            parts = v.split(' / ')
            if len(parts) == 1:
                state = parts[0].lower()
                lga = ""
            elif len(parts) == 2:
                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
            else:
                continue
            if state not in states:
                states.append(state)
            if lga not in lgas:
                lgas.append(lga)
            words = re.split('[^a-zA-Z0-9/]',k)
            words.sort()
            wd[k] = words
            d[k] = v
        mapping = {}
        mapping['word_dict'] = wd
        mapping['lga_dict'] = d
        mapping['states'] = states
        mapping['lgas'] = lgas
        return mapping
    ###)

    security.declareProtected(View,'findLga') ###(
    def findLga(self,words,words_dict):
        words = re.split('[^a-zA-Z0-9/]',words)
        lga_words = []
        for word in words:
            if word:
                lga_words += word.strip().lower(),
        lga_words.sort()
        state_lga = ''
        while not state_lga:
            for k,l in words_dict.items():
                if lga_words == l:
                    state_lga = k
                    break
            break
        return state_lga
    ###)
    security.declareProtected(View,'getAccessInfo') ###(
    def getAccessInfo(self,context):
        "return a dict with access_info"
        logger = logging.getLogger('WAeUPTool.getAccessInfo')
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        member_id = str(member)
        info = {}
        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
        is_staff = info['is_staff'] = not is_anonymous and not is_student
        roles = member.getRolesInContext(context)
        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
                                                                            "SectionManager" in roles or
                                                                            "Manager" in roles)
        is_allowed = info['is_allowed'] = not is_anonymous
        requested_id = context.getStudentId()
        student_id  = None
        if is_allowed:
            if not is_student and requested_id:
                student_id  = requested_id
            elif is_anonymous or member_id != requested_id:
                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
            else:
                student_id = member_id
        info['student_id'] = student_id
        return info
    ###)

    security.declareProtected(ModifyPortalContent,'openLog') ###(
    def openLog(self,name):
        """open a log file"""
        version = 1
        path = "%s/log/%s_%d.log" % (i_home,name,version)
        while os.path.exists(path):
            version += 1
            path = "%s/log/%s_%d.log" % (i_home,name,version)
        log = open(path,"w")
        return log
    ###)

    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
    def bypassQueueCatalog(self,enable=True):
        """bypass the QueueCatalog by setting all indexes to process imediate,
        if enable is True (default) the old settings are restored
        """

    ###)

    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
        """measure Object access Time"""
        import random
        if hasattr(self,'portal_catalog_real'):
            aq_portal = self.portal_catalog_real.evalAdvancedQuery
        else:
            aq_portal = self.portal_catalog.evalAdvancedQuery
        nr_pts = int(nr_pts)
        probe = int(probe)
        intervall = probe/10
        objects = ("application","clearance","personal")
        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
        #i = random.randrange(num_objects)
        count = 0
        found = 0
        not_found = 0
        t_found = 0
        t_not_found = 0
        time_found = time_not_found = 0.0
        t_time_found = t_time_not_found = 0.0
        accessed = []
        t_min = 1000
        t_max = 0
        #import pdb;pdb.set_trace()
        students = self.portal_catalog(portal_type="Student")
        num_students = len(students)
        if method == "d":
            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
            res = aq_portal(query)
            brains = {}
            for r in res:
                sid = r.relative_path.split('/')[-2]
                if brains.has_key(sid):
                    brains[sid][r.portal_type] = r
                else:
                    brains[sid] = {r.portal_type : r}
            brains_list = brains.keys()
            num_objects = len(brains_list)
        else:
            num_objects = num_students
        print "="*40
        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
                                                                        probe,
                                                                        nr_pts,
                                                                        num_objects)
        print "nr found/not time found/not min/max"
        elapse = time.time()
        i_elapse = time.time()
        c_elapse = time.clock()
        for c in range(1,probe + 1):
            i = random.randrange(num_objects)
            if method in ('a','b','c'):
                student_brain = students[i]
            elif method == "d":
                #import pdb;pdb.set_trace()
                student_brain = brains[brains_list[i]]
            if method == "c":
                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
                res = aq_portal(query)
                this_portal_types = [r.portal_type for r in res]
            for i in range(nr_pts):
                oid = objects[i]
                if method == "a":
                    try:
                        student_path = student_brain.getPath()
                        path = "%s/%s" % (student_path,oid)
                        doc = self.unrestrictedTraverse(path).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        pass
                elif method == "b":
                    try:
                        student_object = student_brain.getObject()
                        doc = getattr(student_object,oid).getContent()
                        found += 1
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    except:
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                        not_found += 1
                        pass
                elif method == "c":
                    if portal_types[i] in this_portal_types:
                        found += 1
                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                elif method == "d":
                    if student_brain.has_key(portal_types[i]):
                        found += 1
                        doc = student_brain[portal_types[i]].getObject().getContent()
                        i_time = time.time() - i_elapse
                        time_found += i_time
                    else:
                        not_found += 1
                        i_time = time.time() - i_elapse
                        time_not_found += i_time
                i_elapse = time.time()
            if c and (c % intervall == 0):
                #i_time = time.time() - i_elapse
                t_per = 0.0
                if found:
                    t_per = time_found/found
                if t_per > t_max:
                    t_max = t_per
                if t_per > 0.0 and t_per < t_min:
                    t_min = t_per
                itf = 0.0
                if found:
                    itf = time_found/found
                itnf = 0.0
                if not_found :
                    itnf = time_not_found / not_found
                interval_time = time_found + time_not_found
                s = "%(c)d: %(found)d/%(not_found)d " % vars()
                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
                print s
                t_found += found
                t_not_found += not_found
                t_time_found += time_found
                t_time_not_found += time_not_found
                time_found = time_not_found = 0.0
                found = not_found = 0
        # t_found += found
        # t_not_found += not_found
        elapse = time.time() - elapse
        itf = 0.0
        if t_found:
            itf = t_time_found/t_found
        itnf = 0.0
        if t_not_found:
            itnf = t_time_not_found / t_not_found
        #c_elapse = time.clock() - c_elapse
        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
        print "-"*40
        print s
        rel_found = float(t_found)/probe
        rel_not_found = float(t_not_found)/probe
        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
        print estimated_total_time
    ###)

    security.declareProtected(ModifyPortalContent,'writeLog') ###(
    def writeLog(self,logfile,s):
        """write to the log file"""
        logfile.write(s)

###)

    def generateStudentId(self,letter): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if letter == '?':
            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
        sid = "%c%d" % (letter,r.randint(99999,1000000))
        students = self.portal_url.getPortalObject().campus.students
##        while hasattr(students, sid):
##            sid = "%c%d" % (letter,r.randint(99999,1000000))
        while self.students_catalog(id = sid):
            sid = "%c%d" % (letter,r.randint(99999,1000000))
        return sid
    ###)

    def generatePassword(self,s=None): ###(
        import random
        r = random
        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
        if s is None:
            s = 'abcdefghklmnpqrstuvwxy23456789'
        pw = ''
        while len(pw) < 6:
            pw += r.choice(s)
        return pw
    ###)

    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
    def dumpSchoolfeePayments(self):
        "dump paid schoolfees"
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
        aq_student = self.students_catalog.evalAdvancedQuery
        query = In('review_state',('schoolfee_paid',
                                   'courses_registered',
                                   'courses_validated',
                                   ))
        res = aq_student(query)
        #import pdb;pdb.set_trace()
        l = []
        logger.info("start for %d" % len(res))
        count = 1
        log_after = 100
        for student in res:
            if not count % log_after:
                logger.info("processed %d total %d" % (log_after,count))
            count += 1
            fee_dict =self.getSchoolFee(student)
            fulltime = student.mode.endswith('_ft')
            d = {}
            d['student_id'] = student.id
            d['name'] = student.name
            d['amount'] = fee_dict.get(new_returning)
            l += d,
        csv_name = self.dumpListToCSV(l,'payments')
        logger.info('%s dumped payments to %s' % (member,export_file))
    ###)

    security.declarePublic('dumpListToCSV') ###(
    def dumpListToCSV(self,l,filename,fields=None):
        """dump a list of dicts to a CSV file"""
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
        if fields is None:
            fields = l[0].keys()
        headline = ','.join(fields)
        out = open(export_file,"wb")
        out.write(headline +'\n')
        out.close()
        out = open(export_file,"a")
        csv_writer = csv.DictWriter(out,fields,)
        csv_writer.writerows(l)
        return export_file
    ###)

    security.declareProtected(ManagePortal, 'listMembers') ###(
    def listMembers(self):
        "list all members"
        mtool = self.portal_membership
        member = mtool.getAuthenticatedMember()
        logger = logging.getLogger('WAeUPTool.listMembers')
        if str(member) not in ('admin','joachim'):
            logger.info('%s tried to list members' % (member))
            return None
        members = self.portal_directories.members
        all = members.listEntryIdsAndTitles()
        l = []
        for user_id,name in all:
            d = {}
            d['user_id'] = user_id
            d['name'] = name
            d['pw'] = getattr(getattr(members,user_id),'password')
            d['email'] = getattr(getattr(members,user_id),'email')
            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
            l += d,
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
        logger.info('%s dumped memberlist to %s' % (member,export_file))
        fields = l[0].keys()
        headline = ','.join(fields)
        out = open(export_file,"wb")
        out.write(headline +'\n')
        out.close()
        out = open(export_file,"a")
        csv_writer = csv.DictWriter(out,fields,)
        csv_writer.writerows(l)
    ###)

    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
    def removeDeletedDocIds(self, max=1000):
        """
        remove deleted docids from repository commit after max
        """
        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
        repository = getToolByName(self, 'portal_repository')
        pxtool = getToolByName(self, 'portal_proxies')
        logger.info('start')
        pxtool_infos = pxtool.getRevisionsUsed()
        logger.info('found  %d used revisions ' % (len(pxtool_infos)))

        nb_revs = 0
        docids_d = {} # all docids
        unused_docids_d = {} # all docids that are unused
        ids_unused_revs_docids = [] # ids for revs of unused docids
        unused_ids = [] # ids for unused revs
        total = 0
        idlist = repository.objectIds()
        to_delete = 0
        found = False
        for id in idlist:
            docid, rev = repository._splitId(id)
            if docid is None:
                logger.info("invalid doc_id %s" % docid)
                continue
            nb_revs += 1
            if not pxtool_infos.has_key(docid):
                found = True
                to_delete += 1
                unused_ids.append(id)
            elif not pxtool_infos[docid].has_key(rev):
                found = True
                to_delete += 1
                unused_ids.append(id)
            if found and not to_delete % max:
                found = False
                #import pdb;pdb.set_trace()
                repository.manage_delObjects(unused_ids)
                transaction.commit()
                logger.info('removed %d total %d unused docids ' % (max,to_delete))
        else:
            if unused_ids:
                repository.manage_delObjects(unused_ids)
                transaction.commit()
        logger.info('finished removing %d unused docids ' % (to_delete))


###)

    security.declareProtected(View,'getCredential') ###(
    def getCredential(self,student_id):
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if not self.isStaff():
            mtool = self.portal_membership
            member = mtool.getAuthenticatedMember()
            logger = logging.getLogger('WAeUPTool.getCredential')
            logger.info('%s tried to access password of %s' % (member,student_id))
            return None
        if student_entry is None:
            return None
        return getattr(student_entry,"password","not set")
    ###)

    security.declarePublic('checkPassword') ###(
    def checkPassword(self,student_id,password):
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return False
        return getattr(student_entry,"password","not set") == password
    ###)

    security.declarePublic('checkGenericPassword') ###(
    def checkGenericPassword(self,member_id):
        member_entry = getattr(self.portal_directories.members,member_id,None)
        if member_entry is None:
            return False
        ltool = getToolByName(self, 'portal_layouts')
        unsecure_words = ltool._getOb('members')['w__password'].check_words
        password = getattr(member_entry,"password","not set")
        is_unsecure = password in unsecure_words
        if is_unsecure:
            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
        return is_unsecure
    ###)

    security.declareProtected(ModifyPortalContent,'editPassword') ###(
    def editPassword(self,student_id,password):
        "edit a student password"
        student_entry = getattr(self.portal_directories.students,student_id,None)
        if student_entry is None:
            return
        setattr(student_entry,'password',password)
    ###)

    security.declareProtected(ModifyPortalContent,'doCommit') ###(
    def doCommit(self,logger=None):
        "commit some transactions"
        transaction.commit()
    ###)

    security.declarePublic('loadStudentFoto') ###(
    def loadStudentFoto(self,student,filename,folder):
        "return a student passport picture"
        #import pdb;pdb.set_trace()
        picture ="%s/import/%s/%s" % (i_home,folder,filename)
        student_id = student.getId()
        images_dir = getImagesDir(student_id)
        if not os.path.exists(images_dir):
            os.mkdir(images_dir)
        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
        for extension in ('.jpg','.JPG'):
            fullname = "%(picture)s%(extension)s" % vars()
            if os.path.exists(fullname):
                copy2(fullname,image_name)
                return "successfully copied passport picture"
        return "passport picture not found: %s.jpg or .JPG" % picture
    ###)

    def old____loadStudentFoto(self,student,filename,folder): ###(
        "return a student passport picture"
        app = student.application
        app_doc = app.getContent()
        #clear = student.clearance
        #clear_doc = clear.getContent()
        #matric_no = clear_doc.matric_no.upper()
        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
        #import pdb;pdb.set_trace()
        if os.path.exists(picture1):
            file = open(picture1)
        elif os.path.exists(picture2):
            file = open(picture2)
        else:
            return "passport picture not found %s" % picture1
        reopened = False
        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
            self.portal_workflow.doActionFor(app,'open')
            reopened = True
        outfile = file.read()
        app_doc.manage_addFile('passport',
                               file=outfile,
                               title="%s.jpg" % filename)
        if reopened:
            self.portal_workflow.doActionFor(app,'close')
        return "successfully loaded passport picture"
    ###)

    security.declareProtected(ModifyPortalContent,'createOne') ###(
    def createOne(self,students_folder,student_brain,letter,commit=False):
        sid = self.waeup_tool.generateStudentId(letter)
        students_folder.invokeFactory('Student', sid)
        student = getattr(students_folder,sid)
        self.portal_workflow.doActionFor(student,'return')
        student.manage_setLocalRoles(sid, ['Owner',])
        matric_no = student_brain.matric_no
        jamb_reg_no = student_brain.Entryregno
        self.students_catalog.addRecord(id = sid,
                                           matric_no = matric_no,
                                           jamb_reg_no = jamb_reg_no,
                                           sex = student_brain.Sex == "F",
                                           name = "%s %s %s" % (student_brain.Firstname,
                                                                student_brain.Middlename,
                                                                student_brain.Lastname)
                                        )
        if commit:
            transaction.commit()
        return sid,jamb_reg_no
    ###)

    security.declareProtected(ModifyPortalContent,'addStudent') ###(
    def addStudent(self,dict):
        students_folder = self.portal_url.getPortalObject().campus.students
        sid = self.waeup_tool.generateStudentId('?')
        students_folder.invokeFactory('Student', sid)
        student_obj = getattr(students_folder,sid)
        f2t = self.field2types_student
        #from pdb import set_trace; set_trace()
        d = {}
        #d['jamb_sex']  = 'M'
        #if dict.get('sex'):
        #    d['jamb_sex']  = 'F'

        entry_session = dict.get('entry_session')
        if entry_session == self.getSessionId()[0]:
            wfaction = 'admit'
            wft = 'wf_transition_admit'
            password = None
        else:
            wfaction = 'return'
            wft = 'wf_transition_return'
            password = self.generatePassword()
            self.makeStudentMember(sid,password)

        for pt in f2t.keys():
            student_obj.invokeFactory(pt,f2t[pt]['id'])
            sub_obj = getattr(student_obj,f2t[pt]['id'])
            sub_doc = sub_obj.getContent()
            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
            d['Title'] = f2t[pt]['title']
            for field in f2t[pt]['fields']:
                d[field] = dict.get(field,'')
            sub_doc.edit(mapping = d)
            new_state = f2t[pt][wft]
            if new_state != "remain":
                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
        self.portal_workflow.doActionFor(student_obj,wfaction)
        student_obj.manage_setLocalRoles(sid, ['Owner',])
        return sid,password
    ###)

    security.declarePublic('getCertificateBrain') ###(
    def getCertificateBrain(self,cert_id):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"Certificate",
                                      'id': cert_id})
        if res:
            return res[0]
        return None
    ###)

    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
    def get_csv_filenames(self):
        "do it"
        files = [file for file in os.listdir("%s/import/" % (i_home))
                 if file.endswith('.csv') and (file.find('imported') == -1 and
                                               file.find('pending') == -1)]
        return files
    ###)

    security.declarePublic('findStudentByMatricelNo') ###(
    def findStudentByMatricelNo(self,matric_no):
        "do it"
        res = ZCatalog.searchResults(self.portal_catalog_real,
                                {'portal_type':"StudentClearance",
                                 'SearchableText': matric_no})
        if res:
            return res[0]
        return None
    ###)

    security.declarePublic('makeStudentMember') ###(
    def makeStudentMember(self,sid,password='uNsEt'):
        """make the student a member"""
        membership = self.portal_membership
        membership.addMember(sid,
                             password ,
                             roles=('Member',
                                     'Student',
                                     ),
                             domains='',
                             properties = {'memberareaCreationFlag': False,
                                           'homeless': True},)
        member = membership.getMemberById(sid)
        self.portal_registration.afterAdd(member, sid, password, None)
        #self.manage_setLocalRoles(sid, ['Owner',])
    ###)

    security.declareProtected(View,'makeStudentData') ###(
    def makeStudentData(self,student_id,email=None,phone_nr=None):
        "create Datastructure for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentData')
        students_folder = self.portal_url.getPortalObject().campus.students
        #res = self.students_catalog(id=student_id)
        #if res:
        #    st = res[0]
        #res = self.returning_import(matric_no = st.matric_no)
        res = self.returning_import(id = student_id)
        if res:
            student = res[0]
        else:
            logger.info('Id %s not found in returning_import' % student_id)
            return
        logger.info('%s creates data structure' % student_id)
        s_results = self.results_import(matric_no = student.matric_no)
        if s_results:
            lnr = self.getLevelFromResultsCosCode(s_results)
            level = "%d00" % lnr
            verdict,eligible = self.getVerdict(s_results[0].Verdict)
            #if eligible:
            #    level = "%d00" % (lnr + 1)
        else:
            logger.info('matric_no %s not found in results_import' % student.matric_no)
            level = ''
            verdict = ''
        #student should not be allowed to perform this transition
        #wftool = self.portal_workflow
        #wftool.doActionFor(student,'return')
        certcode_org = student.Coursemajorcode
        certcode = makeCertificateCode(certcode_org)
        certificate_brain = self.getCertificateBrain(certcode)
        if not certificate_brain:
            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
            logger.info(em)
        matric_no = student.matric_no
        sid = student_id
        student_obj = getattr(students_folder,sid)
        if not getattr(student_obj,'application'):
            student_obj.invokeFactory('StudentApplication','application')
        application = student_obj.application
        self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        student_obj.invokeFactory('StudentPersonal','personal')
        da['jamb_reg_no'] = student.Entryregno
        em = self.getEntryMode(student.Entryregno)
        da['entry_mode'] = em
        personal = student_obj.personal
        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        student_obj.invokeFactory('StudentClearance','clearance')
        clearance = student_obj.clearance
        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['matric_no'] = matric_no
        state = student.State
        lga = student.LGA
        if state and lga:
            lga =  state + ' / ' + lga
        else:
            lga = "None"
        da['jamb_lga'] = dc['lga'] = lga
        da['app_email'] = dp['email'] = email
        da['app_mobile'] = dp['phone'] = phone_nr
        dp['firstname'] = student.Firstname
        dp['middlename'] = student.Middlename
        dp['lastname'] = student.Lastname
        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
        da['jamb_sex'] = student.Sex
        dp['sex'] = student.Sex == 'F'
        dp['perm_address'] = student.Permanent_Address
        application.getContent().edit(mapping=da)
        self.portal_workflow.doActionFor(application,'close',dest_container=application)
        personal.getContent().edit(mapping=dp)
        clearance.getContent().edit(mapping=dc)
        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
        #
        # Study Course
        #
        student_obj.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_obj.study_course
        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = certcode
        dsc['current_level'] = level
        dsc['current_verdict'] = verdict
        dsc['current_mode'] = em
        dsc['current_session'] = '05'
        studycourse.getContent().edit(mapping=dsc)
        #
        # Level
        #
##        l = getattr(studycourse,level,None)
##        if l is None:
##            studycourse.invokeFactory('StudentStudyLevel', level)
##            l = getattr(studycourse, level)
##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
##            l.getContent().edit(mapping={'Title': "Level %s" % level})
###)

    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
    def admitOneStudent(self,brain,entry_session,pin_password):
        "create Datastructure for an admitted Student"
        #import pdb;pdb.set_trace()
        if not hasattr(self,"_v_certificates"):
            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
        students_folder = self.portal_url.getPortalObject().campus.students
        logger = logging.getLogger('WAeUPTool.admitOneStudent')
        res = self.students_catalog(jamb_reg_no = brain.reg_no)
        if res:
            logger.info('student with this reg_no already exists %s with id %s' % (brain.reg_no,res[0].id))
            return
        if brain.status != "admitted":
            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
            return
        pin_parts = brain.pin.split('-')
        if pin_parts and len(pin_parts) != 3:
            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
            return
        if brain.course_admitted not in self._v_certificates:
            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
            return
        if brain.sex not in (True,False):
            logger.info('sex of %s not available' % (brain.reg_no))
            return
        student_id = self.generateStudentId('?')
        students_folder.invokeFactory('Student', student_id)
        student_object = getattr(students_folder,student_id)
        if pin_password:
            password = pin_parts[2]
            self.makeStudentMember(student_id,password = password)
        student_object.manage_setLocalRoles(student_id, ['Owner',])
        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
        #
        # application
        #
        student_object.invokeFactory('StudentApplication','application')
        application = student_object.application
        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
        da = {'Title': 'Application Data'}
        da['jamb_reg_no'] = brain.reg_no

        sex = 'M'
        if brain.sex:
            sex = 'F'
        da['jamb_sex'] = sex
        da['jamb_age'] = brain.jamb_age
        #da['app_ac_pin'] = brain.pin
        da['jamb_lga'] = brain.jamb_lga
        da['jamb_state'] = brain.jamb_state
        da['jamb_score'] = brain.aggregate
        da['app_email'] = brain.email
        da['app_mobile'] = brain.phone
        if brain.entry_mode:
            da['entry_mode'] = brain.entry_mode
        elif brain.screening_type == 'pume':
            da['entry_mode'] = 'ume_ft'
        elif brain.screening_type == 'pde':
            da['entry_mode'] = 'de_ft'
        elif brain.screening_type == 'pce':
            da['entry_mode'] = 'pce'
        elif brain.screening_type == 'prence':
            da['entry_mode'] = 'prence'
        else:
            da['entry_mode'] = ''
        da['entry_session'] = entry_session
        da['jamb_lastname'] = brain.lastname
        da['jamb_middlename'] = brain.middlenames   # different field names!
        da['jamb_firstname'] = brain.firstname
        da['screening_application_date'] = brain.application_date
        da['date_of_birth'] = brain.date_of_birth
        da['jamb_first_cos'] = brain.course1
        da['jamb_second_cos'] = brain.course2
        da['course3'] = brain.course3
        da['screening_type'] = brain.screening_type
        da['screening_score'] = brain.screening_score
        da['screening_date'] = brain.screening_date
        da['hq_type'] = brain.hq_type
        da['hq_grade'] = brain.hq_grade
        da['aos'] = brain.aos

        application.getContent().edit(mapping=da)
        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
        #
        # personal
        #
        student_object.invokeFactory('StudentPersonal','personal')
        personal = student_object.personal
        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
        dp = {'Title': 'Personal Data'}
        dp['sex'] = brain.sex
        dp['email'] = brain.email
        dp['phone'] = brain.phone
        dp['lastname'] = brain.lastname
        dp['middlename'] = brain.middlenames   # different field names!
        dp['firstname'] = brain.firstname
        personal.getContent().edit(mapping=dp)
        #
        # clearance
        #
        student_object.invokeFactory('StudentClearance','clearance')
        clearance = student_object.clearance
        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
        dc = {'Title': 'Clearance/Eligibility Record'}
        dc['lga'] = brain.lga
        dc['birthday'] = brain.date_of_birth
        clearance.getContent().edit(mapping=dc)
        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
        #
        # study Course
        #
        student_object.invokeFactory('StudentStudyCourse','study_course')
        studycourse = student_object.study_course
        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
        dsc = {}
        dsc['study_course'] = brain.course_admitted
        dsc['current_verdict'] = ''
        dsc['current_mode'] = da['entry_mode']
        if da['entry_mode'].startswith('de'):
            dsc['current_level'] = '200'
        elif da['entry_mode'].startswith('pre'):
            dsc['current_level'] = '000'
        else:
            dsc['current_level'] = '100'
        dsc['current_session'] = entry_session
        studycourse.getContent().edit(mapping=dsc)
        #
        # payments folder
        student_object.invokeFactory('PaymentsFolder','payments')
        payments = getattr(student_object,'payments')
        dpay = {}
        dpay['Title'] = 'Payments'
        payments.getContent().edit(mapping=dpay)
        self.portal_workflow.doActionFor(payments,'open')
        #
        # passport foto
        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
                                                              brain.screening_type,
                                                              brain.reg_no)
        images_dir = getImagesDir(student_id)
        #images_dir = os.path.join("%s" % images_base,student_id)
        letter_dir,student_dir = os.path.split(images_dir)
        if not os.path.exists(letter_dir):
            os.mkdir(letter_dir)
        if not os.path.exists(images_dir):
            os.mkdir(images_dir)
        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
        if os.path.exists(app_picture):
            copy2(app_picture,image_name)
        else:
            logger.info('passport of %s/%s not found: %s' % (student_id,
                                                             brain.reg_no,
                                                             app_picture))

        return student_id
    ###)

    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
    def makeStudentLevel(self,student_id):
        "create the StudyLevel for a returning Student"
        #import pdb;pdb.set_trace()
        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
        students_folder = self.portal_url.getPortalObject().campus.students
        res = self.students_catalog(id=student_id)
        if res:
            st = res[0]
        course = st.course
        matric_no = st.matric_no
        level = st.level
        res = self.results_import(matric_no = matric_no)
        if res:
            results = res
        logger.info('%s creating Level %s' % (student_id,level))
        #
        # Level
        #
        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        studycourse = getattr(student_obj,"study_course",None)
        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
        l = getattr(studycourse,level,None)
        if l is None:
            studycourse.invokeFactory('StudentStudyLevel', level)
            l = getattr(studycourse, level)
            self.portal_workflow.doActionFor(l,'open',dest_container=l)
            l.getContent().edit(mapping={'Title': "Level %s" % level})
        ###)

    security.declarePublic('getHallInfo') ###(
    def getHallInfo(self,bed):
        """return Hall Info"""
        info = {}
        hall,block,room,letter = bed.split('_')
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
        if res and len(res) == 1:
            hall_brain = res[0]
            hall_doc = hall_brain.getObject().getContent()
        else:
            return info
        info['hall_title'] = hall_brain.Title
        info['maintenance_code'] = hall_doc.maintenance_code
        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
        batch_doc = None
        for brain in res:
            if brain.id.startswith(info['maintenance_code']):
                batch_doc = brain.getObject().getContent()
                break
        if batch_doc is None:
            info['maintenance_fee'] = ''
        else:
            info['maintenance_fee'] = batch_doc.cost
        return info
    ###)

    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
    def removePictureFolder(self,student_id):
        """remove picture_folder by renaming it"""
        path = 'images'
        picture_path = os.path.join(i_home,path,student_id)
        if not os.path.exists(picture_path):
            return False
        os.rename(picture_path,picture_path + "_removed")
        return True
    ###)

    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
    def restorePictureFolder(self,student_id):
        """restore picture_folder by renaming it"""
        path = 'images'
        picture_path = os.path.join(i_home,path,student_id)
        if not os.path.exists(picture_path + "_removed"):
            return False
        os.rename(picture_path + "_removed",picture_path)
        return True
    ###)

    security.declarePublic('picturesExist') ###(
    def picturesExist(self, ids,student_id=None):
        """check if pictures exist in the filesystem"""
        if student_id is None:
            student_id = self.getStudentId()
        if student_id is None:
            return False
        picture_path = getImagesDir(student_id)
        #picture_path = os.path.join(images_base,student_id)
        if not os.path.exists(picture_path):
            return False
        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
        return set(ids).issubset(set(pictures))
    ###)

    security.declarePublic('picturesList') ###(
    def picturesList(self):
        """check if pictures exist in the filesystem"""
        path = 'images'
        student_id = self.getStudentId()
        #picture_path = os.path.join(i_home,path,student_id)
        picture_path = getImagesDir(student_id)
        if not os.path.exists(picture_path):
            return []
        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
    ###)

    security.declarePublic('showFsPicture') ###(
    def showFsPicture(self,path):
        """return a picture from the filesystem"""
        #picture_path = os.path.join(i_home,path)
        picture_path = os.path.join(images_base,path)
        response = self.REQUEST.RESPONSE
        #import pdb;pdb.set_trace()
        registry = getToolByName(self, 'mimetypes_registry')
        mimetype = str(registry.lookupExtension(path.lower()) or
                    registry.lookupExtension('file.bin'))
        if os.path.exists(picture_path):
            response.setHeader('Content-type',mimetype)
            return open(picture_path).read()
        picture_path = os.path.join(i_home,'import',path)
        if os.path.exists(picture_path):
            return open(picture_path).read()
    ###)

    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
    def deleteAllCourses(self,department="All"):
        ''' delete the courses'''
        pm = self.portal_membership
        member = pm.getAuthenticatedMember()

        if str(member) not in ("henrik","joachim"):
            return "not possible"
        if department == "All":
            res = self.portal_catalog({'meta_type': 'Department'})
        if len(res) < 1:
            return "No Departments found"

        deleted = []
        for dep in res:
            cf = dep.getObject().courses
            if cf:
                cf.manage_delObjects(ids=cf.objectIds())
                deleted.append("deleted Courses in %s" % dep.getId)
        return "\r".join(deleted)
    ###)

    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
    def getLogfileLines(self,filename="event.log",numlines=20):
        """Get last NUMLINES lines of logfile FILENAME.

        Return last lines' of a file in the instances logfile directory as
        a list. The number of returned lines equals `numlines' or less. If
        less than `numlines' lines are available, the whole file ist
        returned. If the file can not be opened or some other error
        occurs, empty list is returend.
        """
        result = []
        lines_hit = 0

        # We only handle files in instances' log directory...
        logpath = os.path.join(i_home, "log")
        filename = str(os.path.abspath( os.path.join( logpath, filename )))
        if not filename.startswith( logpath ):
            # Attempt to access file outside log-dir...
            return []

        try:
            fd = file( filename, "rb" )
        except IOError:
            return []
        if not fd:
            return []

        if os.linesep == None:
            linesep = '\n'
        else:
            linesep = os.linesep

        # Try to find 'numlines' times a lineseparator, searching from end
        # and moving to the beginning of file...
        fd.seek( 0, 2) # Move to end of file...
        while lines_hit < numlines:
            if fd.read(1) == linesep[-1]: # This moves filedescriptor
                                          # one step forward...
                lines_hit += 1
            try:
                fd.seek( -2, 1) # Go two bytes back from current pos...
            except IOError:
                # We cannot go back two bytes. Maybe the file is too small...
                break
        fd.seek(2,1)

        # Read all lines from current position...
        result = fd.readlines()
        # Remove line endings...
        result = [x.strip() for x in result]
        fd.close()
        return result
    ###)

    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
    def getCallbacksFromLog(self,filename):
        """fix Online Payment Transactions from Z2.log entries"""
        import transaction
        import random
        from cgi import parse_qs
        from urlparse import urlparse
        #from pdb import set_trace
        wftool = self.portal_workflow
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        students_folder = self.portal_url.getPortalObject().campus.students
        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
        data = re.compile(s)
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        #name = 'epaymentsuccessful_z2log2'
        name = filename
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
        try:
            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
        except:
            logger.error('Error reading %s' % name)
            return
        tas = []
        for line in transactions:
            dict = {}
            items = data.search(line)
            dict['idict'] = idict = items.groupdict()
            #print idict
            #from pdb import set_trace;set_trace()
            urlparsed = urlparse(idict['get'][4:])
            #print urlparsed
            path = urlparsed[2].split('/')
            dict['student_id'] = student_id = path[8]
            dict['payment_id'] = payment_id = path[10]
            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
            tas.append(dict)
            tr_count += 1
        return tas
    ###)

    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
    def importOnlinePaymentTransactions(self):
        """load Online Payment Transactions from CSV values"""
        import transaction
        import random
        #from pdb import set_trace
        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
        opt = self.online_payments_import
        students_folder = self.portal_url.getPortalObject().campus.students
        start = True
        tr_count = 1
        total = 0
        #name = 'pume_results'
        name = 'OnlineTransactions'
        no_import = []
        imported = []
        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
        try:
            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
        except:
            logger.error('Error reading %s.csv' % name)
            return
        for pay_transaction in transactions:
            if start:
                start = False
                logger.info('Start loading from %s.csv' % name)
                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
                no_import.append('%s,"Error"' % s)
                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
                format_error = format + ',"%(Error)s"'
            data = {}

            # format of the first file sent by Tayo
            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = pay_transaction['Resp Code']
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the second file sent by Tayo
            #data['datetime'] = date = 0
            #data['student_id'] = student_id = pay_transaction['Payer ID']
            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
            #data['response_code'] = response_code = '00'
            #data['amount'] = amount = pay_transaction['Amount']

            # format of the third file sent by Kehinde
            data['datetime'] = date = 0
            data['student_id'] = student_id = pay_transaction['customer_id']
            data['order_id'] = order_id = pay_transaction['merchant_reference']
            data['response_code'] = response_code = '00'
            data['amount'] = amount = pay_transaction['Amount']

            dup = False
            if response_code == "12":
                continue
            try:
                opt.addRecord(**data)
            except ValueError:
                dup = True
            #from pdb import set_trace;set_trace()
            if dup:
                if response_code == "00":
                    try:
                        opt.modifyRecord(**data)
                    except:
                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
                        continue
                else:
                    pay_transaction['Error'] = "Duplicate order_id"
                    no_import.append( format_error % pay_transaction)
                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
                    continue
            tr_count += 1
            if tr_count > 1000:
                if len(no_import) > 0:
                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                             '\n'.join(no_import) + '\n')
                    no_import = []
                em = '%d transactions committed\n' % (tr_count)
                transaction.commit()
                regs = []
                logger.info(em)
                total += tr_count
                tr_count = 0
        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
                                                '\n'.join(no_import))
        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
    ###)

    security.declareProtected(ModifyPortalContent,"importData")###(
    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
        """load data from CSV values"""
        import transaction
        import random
        students_folder = self.portal_url.getPortalObject().campus.students
        pending_only = False
        pend_str = '--'
        elapse = time.time()
        #
        # preparations
        #
        if filename == pend_str:
            pending_only = True
        # mode = "create"
        # if edit:
        #     if filename.endswith('_toDelete'):
        #         mode = "delete"
        #     else:
        #         mode = "edit"
        importer_name = ''.join([part.capitalize() for part in name.split('_')])
        importer = eval("%sImport" % importer_name)(self)
        logger = importer.logger
        if importer.init_errors:
            logger.info(importer.init_errors)
            return importer.init_errors
        member = importer.member
        current = importer.current
        import_date = importer.import_date
        #
        # not_imported
        #
        info = importer.info
        data_keys = importer.data_keys
        csv_keys = importer.csv_keys
        #csv_keys.extend(info.keys())
        headline_mapping = dict((k,k) for k in csv_keys)
        #
        # pending
        #
        pending_path = importer.pending_path
        pending_tmp = importer.pending_tmp
        pending_backup = importer.pending_backup
        pending_fn = importer.pending_fn
        imported_path = importer.imported_path
        imported_fn = importer.imported_fn
        commit_after = importer.commit_after
        pending = []
        pending_digests = []
        #total_added_to_pending = 0
        if not pending_only:
            pending,pending_digests = importer.makeIdLists()
            pending_at_start = len(pending)
        datafile = open(pending_tmp,"w")
        pending_csv_writer = csv.DictWriter(datafile,
                                                    csv_keys,
                                                    extrasaction='ignore')
        pending_csv_writer.writerow(headline_mapping)
        datafile.close()
        #
        # imported
        #
        if not os.path.exists(imported_path):
            datafile = open(imported_path,"w")
            imported_csv_writer = csv.DictWriter(datafile,
                                                 csv_keys,
                                                 extrasaction='ignore')
            imported_csv_writer.writerow(headline_mapping)
            datafile.close()
        start = True
        tr_count = 0
        total = 0
        total_added_to_imported = 0
        total_pending = 0
        if pending_only:
            import_source_path = pending_path
        else:
            import_source_path = "%s/import/%s.csv" % (i_home,filename)
        if not os.path.exists(import_source_path):
            fn = os.path.split(import_source_path)[1]
            em = 'No  %(fn)s' % vars()
            return em
        import_source_fn = os.path.split(import_source_path)[1]
        info['imported_from'] = import_source_fn
        attrs = csv.reader(open(import_source_path,"rb")).next()
        if "import_mode" not in attrs:
            msg = 'import_mode must be in heading'
            return msg

        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
                                                        or k.strip() in info.keys())]
        diff2schema = set(import_keys).difference(set(importer.schema.keys()))
        diff2layout = set(import_keys).difference(set(importer.layout.keys()))
        if diff2schema and diff2schema != set(['id',]):
            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
            return msg
        #
        # start importing
        #
        try:
            reader = csv.DictReader(open(import_source_path,"rb"))
        except:
            msg = 'Error reading %s.csv' % filename
            logger.error(msg)
            return msg
        items = [item for item in reader]
        total_to_import = len(items)
        tti_float = float(total_to_import)
        if pending_only:
            pending_at_start = total_to_import
        count = 0
        imported = []
        old_commit_count = 0
        error_count = imported_count = 0
        already_in = 0
        for record in items:
            item = {}
            empty_value_keys = []
            for k,v in record.items():
                if v:
                    if v == EMPTY:
                        empty_value_keys += k,
                        v = ''
                    item[k.strip()] = v.strip()
            count += 1
            if start:
                start = False
                adapters = [MappingStorageAdapter(importer.schema, item)]
                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
            dm = DataModel(item, adapters,context=self)
            ds = DataStructure(data=item,datamodel=dm)
            error_string = ""
            total += 1
            import_mode = item.get('import_mode','edit')
            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
            if import_method is None:
                error_string += "no %(import_mode)s " % vars()
            elif (import_mode in importer.required_modes and
                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
            else:
                for k in import_keys:
                    if k not in item.keys() or k not in importer.validators.keys():
                        continue
                    if not importer.validators[k](ds,mode=import_mode):
                        error_string += ' ++ '
                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
                                                                           ds.getErrorMapping(k)))
            if error_string:
                error = error_string
                id = ''
                mapping = item
            else:
                temp_item = item.copy()
                temp_item.update(dm)
                results = import_method(temp_item)
                id = results[0]
                error = results[1]
                mapping = results[2]
            for k in empty_value_keys:
                mapping[k] = EMPTY
            if mapping.has_key('sex'):
                #import pdb;pdb.set_trace()
                if mapping['sex']:
                    mapping['sex'] = 'F'
                else:
                    mapping['sex'] = 'M'
            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
            info['error'] = error
            mapping.update(info)
            log_list = []
            if error:
                error_count += 1
                digest = makeDigest(mapping,data_keys)
                if digest not in pending_digests:
                    pending_digests += digest,
                    pending.append(mapping)
                    if not pending_only:
                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
                else:
                    already_in += 1
                    pass
            else:
                imported_count += 1
                imported += mapping,
                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
            if log_list:
                time_till_now = time.time() - elapse
                percent_finished = (error_count + imported_count)/tti_float*100
                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
                logger.info(' '.join(log_list))
            finished = count > total_to_import - 1
            must_commit = False
            if len(imported) != old_commit_count:
                if not len(imported) % commit_after:
                    must_commit = True
                    old_commit_count = len(imported)

            if must_commit or finished:
                if len(imported):
                    transaction.commit()
                    datafile = open(imported_path,"a")
                    writer = csv.DictWriter(datafile,
                                            csv_keys,
                                            extrasaction='ignore')
                    writer.writerows(imported)
                    datafile.close()
                    total_added_to_imported += len(imported)
                    imported = []
                if len(pending) > 0:
                    datafile = open(pending_tmp,"a")
                    writer = csv.DictWriter(datafile,
                                            csv_keys,
                                            extrasaction='ignore')
                    writer.writerows(pending)
                    datafile.close()
                    total_pending += len(pending)
                    #total_added_to_pending += len(pending)
                    pending = []
                if not finished:
                    msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
                    logger.info(msg)
        elapse = time.time() - elapse
        copy2(pending_tmp,pending_path)
        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
        if pending_only:
            removed_pending = pending_at_start - total_pending
            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
        else:
            added_pending = total_pending - pending_at_start
            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
        logger.info(msg)
        os.remove(pending_tmp)
        return msg
    ###)



    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
    def moveImagesToFS(self,student_id="O738726"):
        "move the images to the filesystem"
        images_dir = getImagesDir(student_id)
        #images_dir = os.path.join("%s" % images_base,student_id)
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key,None)
                if not image or not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))
    ###)

    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
    def movePassportToFS(self,student_id="O738726"):
        "move the passports to the filesystem"
        images_dir = os.path.join("%s" % i_home,'passports')
        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
        stool = getToolByName(self, 'portal_schemas')
        schemas = ['student_application',
                   #'student_clearance',
                   ]
        created = False
        for schema_id in schemas:
            schema = stool._getOb(schema_id)
            object = getattr(student_folder,schema_id[len('student_'):],None)
            if object is None:
                continue
            doc = object.getContent()
            for key in schema.keys():
                if schema[key].meta_type != "CPS Image Field":
                    continue
                #import pdb;pdb.set_trace()
                image = getattr(doc,key)
                if not hasattr(image,"data"):
                    continue
                if not created:
                    if not os.path.exists(images_dir):
                        os.mkdir(images_dir)
                    created = True
                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
                open(filename,"wb").write(str(image.data))
    ###)

InitializeClass(WAeUPTool)