Index: WAeUP_SRP/base/WAeUPImport.py
===================================================================
--- WAeUP_SRP/base/WAeUPImport.py	(revision 3172)
+++ WAeUP_SRP/base/WAeUPImport.py	(revision 3172)
@@ -0,0 +1,920 @@
+#-*- mode: python; mode: fold -*-
+# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
+# Author: Joachim Schmitz (js@aixtraware.de)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+# 02111-1307, USA.
+#
+# $Id$
+"""The WAeUP Tool Box.
+"""
+
+#from AccessControl import ClassSecurityInfo
+#from Acquisition import aq_inner
+#from Acquisition import aq_parent
+#from Globals import DTMLFile
+#from Globals import InitializeClass
+#from OFS.SimpleItem import SimpleItem
+from zExceptions import BadRequest
+
+#from Products.CMFCore.utils import getToolByName
+#from Products.CPSSchemas.DataStructure import DataStructure
+#from Products.CPSSchemas.DataModel import DataModel
+#from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
+#from Products.CMFCore.ActionProviderBase import ActionProviderBase
+#from Products.CMFCore.permissions import View
+#from Products.ZCatalog.ZCatalog import ZCatalog
+#from Products.CMFCore.permissions import ModifyPortalContent
+#from Products.CMFCore.permissions import ManagePortal
+#from Products.CMFCore.utils import UniqueObject
+#from Products.CMFCore.URLTool import URLTool
+from Products.CMFCore.utils import getToolByName
+from Globals import package_home,INSTANCE_HOME
+from Products.AdvancedQuery import Eq, Between, Le,In
+import csv,re,os,sys
+from shutil import copy2
+import DateTime,time
+import logging
+p_home = package_home(globals())
+i_home = INSTANCE_HOME
+from utils import makeDigest
+
+class WAeUPImport: ###( 
+    """ WAeUPImport """
+    required_modes = ('create',)
+
+    def __init__(self,mode,waeup_tool):
+        self.member = member = waeup_tool.portal_membership.getAuthenticatedMember()
+        self.import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S")
+        self.current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        self.waeup_tool = waeup_tool
+        self.schema_tool = getToolByName(waeup_tool, 'portal_schemas')
+        self.layout_tool = getToolByName(waeup_tool, 'portal_layouts')
+        self.portal_workflow = getToolByName(waeup_tool, 'portal_workflow')
+        self.portal_url = getToolByName(waeup_tool, 'portal_url')
+        self.students_catalog = waeup_tool.students_catalog
+        self.courses_catalog = waeup_tool.courses_catalog
+        self.course_results = waeup_tool.course_results
+        self.mode = mode
+        self.import_method = getattr(self, '%s' % mode,None)
+        errors = []
+        if self.import_method is None:
+            errors.append('No importer method %s' % mode)
+        self.pending_path = "%s/import/%s_pending.csv" % (i_home,self.plural_name)
+        self.pending_tmp = "%s/import/%s_pending.csv.tmp" % (i_home,self.plural_name)
+        self.pending_backup = "%s/import/%s_pending.csv.backup" % (i_home,self.plural_name)
+        self.pending_fn = os.path.split(self.pending_path)[1]
+        self.imported_path = "%s/import/%s_imported.csv" % (i_home,self.plural_name)
+        self.imported_fn = os.path.split(self.imported_path)[1]
+        iname = "import_%s" % self.name
+        self.logger = logging.getLogger('WAeUPTool.import%s' % self.plural_name.capitalize())
+        self.schema = self.schema_tool._getOb(iname,None)
+        self.pending_schema = self.schema_tool._getOb("%s_pending" % iname,None)
+        self.layout = self.layout_tool._getOb(iname,None)
+        if self.schema is None:
+            errors.append('No schema %s' % iname)
+        if self.pending_schema is None:
+            self.pending_schema = self.schema
+        if self.layout is None:
+            errors.append('No such layout %s' % iname)
+        self.data_keys = self.pending_schema.keys()
+        self.csv_keys = self.pending_schema.keys()
+        info = {}
+        info['imported_from'] = ''
+        info['imported_by'] = str(member)
+        info['import_date'] = self.import_date
+        info['import_mode'] = mode
+        info['error'] = ''
+        self.info = info
+        self.csv_keys.extend(self.info)
+        self.validators = {}
+        for widget in self.layout.keys():
+            self.validators[widget] = self.layout[widget].validate
+        self.init_errors = ','.join(errors)
+
+    def makeIdLists(self):
+        pending_digests = []
+        pending = []
+        # pending_student_ids = []
+        # pending_matric_nos = []
+        # pending_reg_ns = []
+        if os.path.exists(self.pending_path):
+            datafile = open(self.pending_path,"r")
+            pending_csv_reader = csv.DictReader(datafile,self.csv_keys,)
+            pending_csv_reader.next() # skip headline
+            for item in pending_csv_reader:
+                digest = makeDigest(item,self.data_keys)
+                if digest not in pending_digests:
+                    pending_digests += digest,
+                    pending += item,
+            copy2(self.pending_path,self.pending_backup)
+            pending_at_start = len(pending)                
+            datafile.close()
+        return pending, pending_digests
+###)
+
+class ApplicationImport(WAeUPImport):###(
+    name = "applications"
+    plural_name = "%ss" % name
+    commit_after = 1000
+
+    def create(self,mapping):###(
+        reg_no = mapping.get('reg_no')
+        msg = ''
+        while True:
+            try:
+                self.applicants_catalog.addRecord(**mapping)
+            except ValueError:
+                msg =  "applicant record with reg_no %s already exists" % reg_no
+            break
+        return reg_no,msg,mapping
+    ###)
+
+    def edit(self,mapping):###(
+        reg_no = mapping.get('reg_no')
+        status = mapping.get('status')
+        msg = ''
+        while True:
+            res = self.applicants_catalog(reg_no = reg_no)
+            if len(res):
+                if res[0].status == 'created' and status != 'created':
+                    msg =  "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
+                elif status == 'created' and res[0].status != 'created':
+                    msg =  "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
+                else:
+                    self.applicants_catalog.modifyRecord(**mapping)
+            else:
+                msg =  "applicant record with reg_no %s does not exist" % reg_no
+            break
+        return reg_no,msg,mapping
+    ###)
+
+###)
+
+class CertificateImport(WAeUPImport):###(
+    name = "certificate"
+    plural_name = "%ss" % name
+    commit_after = 100000
+
+    def create(self,mapping):###(
+        if getattr(self,'_v_certificate_list',None) is None:
+            self._v_certificate_list = []
+        if getattr(self,'_v_department_certificates',None) is None:
+            res = self.portal_catalog(portal_type = "Department")
+            self._v_department_certificates = {}
+            for d in res:
+                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
+        did = mapping['department_code']
+        msg = ''
+        while True:
+            d = self._v_department_certificates.get(did,None)
+            if d is None:
+                msg =  "No Department with ID: %s" % did
+                break
+            certificate_id = mapping.get('code')
+            if certificate_id in self._v_certificate_list:
+                msg =  "Duplicate Certificate ID: %s" % did
+                break
+            c = getattr(d,certificate_id,None)
+            if c is not None:
+                msg =  "Duplicate Certificate ID: %s" % did
+                break
+            try:
+                d.invokeFactory('Certificate', certificate_id)
+            except BadRequest,E:
+                msg =  "%s" % E
+                break
+            self._v_certificate_list.append(certificate_id)
+            c = getattr(d,certificate_id)
+            c.getContent().edit(mapping=mapping)
+            break
+        return certificate_id,msg,mapping
+    ###)
+
+    def edit(self,mapping):###(
+        certificate_id = mapping.get('code')
+        res = self.portal_catalog(id=certificate_id)
+        msg = ''
+        while True:
+            if not res:
+                msg =  "No Certificate with ID: %s" % certificate_id
+                break
+            c = res[0].getObject()
+            c.getContent().edit(mapping=mapping)
+            break
+        return certificate_id,msg,mapping
+    ###)
+###)
+
+class CourseImport(WAeUPImport):###(
+    name = "course"
+    plural_name = "%ss" % name
+    commit_after = 1000
+
+    def create(self,mapping):###(
+        if getattr(self,'_v_course_list',None) is None:
+            self._v_course_list = []
+        if getattr(self,'_v_departments',None) is None:
+            res = self.portal_catalog(portal_type = "Department")
+            self._v_department_courses = {}
+            for d in res:
+                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
+        did = mapping['department_code']
+        msg = ''
+        while True:
+            d = self._v_department_courses.get(did,None)
+            if d is None:
+                msg =  "No Department with ID: %s" % did
+                break
+            course_id = mapping.get('code')
+            if course_id in self._v_course_list:
+                msg =  "Duplicate Course ID: %s" % did
+                break
+            c = getattr(d,course_id,None)
+            if c is not None:
+                msg =  "Duplicate Course ID: %s" % did
+                break
+            try:
+                d.invokeFactory('Course', course_id)
+            except BadRequest,E:
+                msg =  "%s" % E
+                break
+            self._v_course_list.append(course_id)
+            c = getattr(d,course_id)
+            c.getContent().edit(mapping=mapping)
+            break
+        return course_id,msg,mapping
+    ###)
+
+    def edit(self,mapping): ###(
+        course_id = mapping.get('code')
+        res = self.portal_catalog(id=course_id)
+        while True:
+            if not res:
+                msg =  "No Course with ID: %s" % course_id
+                break
+            c = res[0].getObject()
+            c.getContent().edit(mapping=mapping)
+            break
+        return course_id,msg,mapping
+    ###)
+###)
+
+class CourseResultImport(WAeUPImport):###(
+    """ CourseresultImport """
+    name = "course_result"
+    plural_name = "%ss" % name
+    commit_after = 1000000
+
+    def getStudentRecord(self,mapping):
+        for id_key in ('student_id','matric_no'):
+            id_field = mapping.get(id_key,'')
+            if id_field:
+                search_key = id_key
+                search_field = id_field
+                break
+        if search_key == "student_id":
+            search_key = 'id'
+        query = Eq(search_key,search_field)
+        res = self.students_catalog.evalAdvancedQuery(query)
+        student_record = None
+        msg = ''
+        if res:
+            student_record = res[0]
+            if search_key == "matric_no":
+                mapping['student_id'] = student_record.id
+            elif search_key == "student_id":
+                mapping['matric_no'] = student_record.matric_no
+        else:
+            msg = "No student with %(search_key)s %(search_field)s" % vars()
+        return student_record,msg
+            
+    def create(self,mapping):###(
+        students_folder = self.portal_url.getPortalObject().campus.students
+        if getattr(self,'_v_courses',None) is None:
+            res = self.courses_catalog()
+            self._v_courses = {}
+            for brain in res:
+                self._v_courses[brain.code] = brain
+        if getattr(self,'_v_level_created',None) is None:
+            self._v_level_created = []
+        msg = ''
+        key = ''
+        while True:
+            course_id = mapping.get('code')
+            if course_id not in self._v_courses.keys():
+                msg = "No course with ID: %s" % course_id
+                break
+            student_record,msg = self.getStudentRecord(mapping)
+            if msg:
+                break
+            student_id = student_record.id
+            level_id = mapping['level_id']
+            code = mapping['code']
+            if student_id not in self._v_level_created:
+                try:
+                    context = getattr(getattr(students_folder,
+                                            "%(student_id)s" % vars()),
+                                    'study_course')
+                except:
+                    msg = "could not create level %(level_id)s for %(student_id)s" % vars()
+                    break
+                if level_id not in context.objectIds():
+                    context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
+                    level = getattr(context,"%(level_id)s" % vars())
+                    self.portal_workflow.doActionFor(level,'open')
+                    # the session string must not be copied into the level object
+                    current_verdict = getattr(student_record,'current_verdict','')
+                    current_session = getattr(student_record,'current_session','')
+                    if current_verdict and student_record.current_level == level_id:
+                        level.getContent().edit(mapping={'verdict': "%s" %
+                                                        current_verdict,
+                                                        'session': "%s" %
+                                                        current_session,
+                                                        })
+                        self.portal_workflow.doActionFor(level,'close')
+                self._v_level_created += student_id,
+            mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
+            for k in ('semester','credits',):
+                mapping[k] = getattr(self._v_courses[course_id],k)
+            try:
+                self.course_results.addRecord(**mapping)
+            except ValueError:
+                msg = "course result already exists: %s" % key
+            break
+        return key,msg,mapping
+    ###)
+
+    def edit(self,mapping): ###(
+        #import pdb;pdb.set_trace()
+        msg = ''
+        key = ''
+        while True:
+            student_record,msg = self.getStudentRecord(mapping)
+            if msg:
+                break
+            student_id = student_record.id
+            level_id = mapping['level_id']
+            code = mapping['code']
+            mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
+            break
+        try:
+            self.course_results.modifyRecord(**mapping)
+        except KeyError:
+            msg = "No course result to edit: %s" % key
+        return key,msg,mapping
+    ###)
+
+    def delete(self,mapping):###(
+        id_key = ''
+        msg = ''
+        while True:
+            student_record,msg = self.getStudentRecord(mapping)
+            if msg:
+                break
+            student_id = student_record.id
+            level_id = mapping['level_id']
+            code = mapping['code']
+            key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
+            if self.course_results.getRecordByKey(key) is None:
+                msg =  "no course-result with %(key)s" % vars()
+                break
+            self.course_results.deleteRecord(key)
+            break
+        return key,msg,mapping
+    ###)
+
+###)
+
+class CertificateCourseImport(WAeUPImport):###(
+    name = "certificate_course"
+    plural_name = "%ss" % name
+    commit_after = 100000
+
+    def create(self,mapping):
+        if getattr(self,'_v_courses',None) is None:
+            res = self.courses_catalog()
+            self._v_courses= [course.code for course in res]
+        if getattr(self,'_v_ceritficates',None) is None:
+            res = self.portal_catalog(portal_type = "Certificate")
+            self._v_certificates = {}
+            for cert in res:
+                self._v_certificates[cert.getId] = cert.getObject()
+        msg = ''
+        while True:
+            certificate_course_id = mapping.get('code')
+            if certificate_course_id not in self._v_courses:
+                msg =  "No Course with ID: %s" % certificate_course_id
+                break
+            cert_id = mapping['certificate_code']
+            cert = self._v_certificates.get(cert_id,None)
+            if cert is None:
+                msg =  "No Certificate with ID: %s" % cert_id
+                break
+            level_id = mapping.get('level')
+            level = getattr(cert,level_id,None)
+            if level is None:
+                cert.invokeFactory('StudyLevel', level_id)
+                level = getattr(cert,level_id,None)
+            elif hasattr(level,certificate_course_id):
+                msg =  "Duplicate CertificateCourse ID: %(certificate_course_id)s " % vars()
+                msg += "in %(cert_id)s/ %(level_id)s" % vars()
+                break
+            level.invokeFactory('CertificateCourse', certificate_course_id)
+            c = getattr(level,certificate_course_id)
+            c.getContent().edit(mapping=mapping)
+            break
+        return certificate_course_id,msg,mapping
+    ###)
+
+class DepartmentImport(WAeUPImport):###(
+    name = "department"
+    plural_name = "%ss" % name
+    commit_after = 100000
+    
+
+    def create(self,mapping):###(
+        "create a department in the correct faculty"
+        fid = mapping['faculty_code']
+        msg = ''
+        if getattr(self,'_v_faculties',None) is None:
+            res = self.portal_catalog(portal_type = "Faculty")
+            self._v_faculties = {}
+            for f in res:
+                self._v_faculties[f.getId] = f.getObject()
+        while True:
+            f = self._v_faculties.get(fid,None)
+            if f is None:
+                msg =  "No Faculty with ID: %s" % fid
+                break
+            else:
+                did = mapping.get('code')
+                d = getattr(f,did,None)
+                if d is None or d.portal_type == "Faculty":
+                    #logger.info('Creating Department %(code)s, %(title)s' % mapping)
+                    try:
+                        f.invokeFactory('Department', did)
+                    except BadRequest,E:
+                        msg =  "%s" % E
+                        break
+                    d = getattr(f,did)
+                    d.invokeFactory('CoursesFolder','courses')
+                    courses = getattr(d,'courses')
+                    dict = {'Title': 'Courses'}
+                    courses.getContent().edit(mapping=dict)
+                    d.invokeFactory('CertificatesFolder','certificates')
+                    certificates = getattr(d,'certificates')
+                    dict = {'Title': 'Certificates'}
+                    certificates.getContent().edit(mapping=dict)
+                    d.getContent().edit(mapping=mapping)
+            break
+        return did,msg,mapping
+    ###)
+
+    def edit(self,mapping): ###(
+        "create a department in the correct faculty"
+        academics_folder = self.portal_url.getPortalObject().campus.academics
+        fid = mapping['faculty_code']
+        did = mapping.get('code')
+        msg = ''
+        while True:
+            try:
+                d = getattr(getattr(academics_folder,fid),did,None)
+            except KeyError:
+                msg =  "Department %s or Faculty %s wrong" % (did,fid)
+                break
+            else:
+                if d is None or d.portal_type == "Faculty":
+                    #logger.info('Editing Department %(code)s, %(title)s' % mapping)
+                    d.getContent().edit(mapping=mapping)
+            break
+        return did,msg,mapping
+    ###)
+###)
+
+class FacultyImport(WAeUPImport):###(
+    name = "faculty"
+    plural_name = "%ss" % name
+    commit_after = 100
+
+    def create(self,mapping): ###(
+        "create a faculty"
+        academics_folder = self.portal_url.getPortalObject().campus.academics
+        fid = mapping['code']
+        msg = ''
+        while True:
+            if getattr(academics_folder,fid,None) is not None:
+                msg =  "Faculty with ID: %s exists" % fid
+                break
+            logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
+            try:
+                academics_folder.invokeFactory('Faculty', fid)
+            except BadRequest,E:
+                msg =  "%s" % E
+                break
+            f = getattr(academics_folder,fid,None)
+            f.getContent().edit(mapping=mapping)
+            return fid,msg,mapping
+        ###)
+
+    def edit(self,mapping): ###(
+        "edit a faculty"
+        academics_folder = self.portal_url.getPortalObject().campus.academics
+        fid = mapping['code']
+        msg = ''
+        while True:
+            f = getattr(academics_folder,fid,None)
+            if f is None:
+                msg =  "Faculty with ID: %s does not exist" % fid
+                break
+            f.getContent().edit(mapping=mapping)
+        return fid,msg,mapping
+    ###)
+###)
+
+class StudentImport(WAeUPImport):###(
+    name = "student"
+    plural_name = "%ss" % name
+    commit_after = 100
+    
+    field2types_student = {   ###(
+                      'StudentApplication':
+                          {'id': 'application',
+                           'title': 'Application Data',
+                           'wf_transition_return': 'close',
+                           'wf_transition_admit': 'remain',
+                           'fields':
+                             ('jamb_reg_no',
+                              'entry_mode',
+                              'entry_session',
+                              'jamb_score',
+                              'app_email',
+                              'jamb_age',
+                              'jamb_state',
+                              'jamb_lga',
+                              'jamb_sex',
+                              )
+                              },
+                      #'StudentPume':
+                      #    {'id': 'pume',
+                      #     'title': 'Pume Data',
+                      #     'wf_transition_return': 'close',
+                      #     'wf_transition_admit': 'close',
+                      #     'fields':
+                      #       ('pume_score',
+                      #        )
+                      #        },
+                      'StudentClearance':
+                          {'id': 'clearance',
+                           'title': 'Clearance/Eligibility Record',
+                           'wf_transition_return': 'close',
+                           'wf_transition_admit': 'remain',
+                           'fields':
+                             ('matric_no',
+                              'nationality',
+                              'lga',
+                              'birthday',
+                              )
+                              },
+                         'StudentPersonal':
+                          {'id': 'personal',
+                           'title': 'Personal Data',
+                           'wf_transition_return': 'open',
+                           'wf_transition_admit': 'remain',
+                           'fields':
+                             ('firstname',
+                              'middlename',
+                              'lastname',
+                              'sex',
+                              'email',
+                              'phone',
+                              'perm_address',
+                              )
+                              },
+                         'StudentStudyCourse':
+                          {'id': 'study_course',
+                           'title': 'Study Course',
+                           'wf_transition_return': 'open',
+                           'wf_transition_admit': 'remain',
+                           'fields':
+                             ('study_course',
+                              'current_level',
+                              'current_session',
+                              'current_mode',
+                              'current_verdict',
+                              'previous_verdict',
+                              )
+                              },
+                         # 'StudentStudyLevel':
+                         #  {'id': 'current_level',
+                         #   'title': '',
+                         #   'wf_transition_return': 'open',
+                         #   'wf_transition_admit': 'remain',
+                         #   'fields':
+                         #     ('verdict',
+                         #      'session',
+                         #      )
+                         #      },
+                         'PaymentsFolder':
+                          {'id': 'payments',
+                           'title': 'Payments',
+                           'wf_transition_return': 'open',
+                           'wf_transition_admit': 'open',
+                           'fields':
+                             ()
+                              },
+                         }
+    ###)
+
+    def create(self,mapping): ###(
+        "create student records due import"
+        logger = logging.getLogger('WAeUPTool.mass_create_student')
+        students_folder = self.portal_url.getPortalObject().campus.students
+        jamb_reg_no = mapping.get('jamb_reg_no',None)
+        msg = ''
+        while True:
+            if jamb_reg_no:
+                res = self.students_catalog(jamb_reg_no = jamb_reg_no)
+                if res:
+                    msg = "jamb_reg_no exists"
+                    break
+            matric_no = mapping.get('matric_no',None)
+            if matric_no:
+                res = self.students_catalog(matric_no = matric_no)
+                if res:
+                    msg = "matric_no exists"
+                    break
+            student_id = self.waeup_tool.generateStudentId('?')
+            students_folder.invokeFactory('Student', student_id)
+            student_obj = getattr(students_folder,student_id)
+            f2t = self.field2types_student
+            d = {}
+            transition = mapping.get('reg_transition','admit')
+            if transition not in ('admit','return'):
+                msg = "no valid transition provided"
+                break
+            for pt in f2t.keys():
+                student_obj.invokeFactory(pt,f2t[pt]['id'])
+                sub_obj = getattr(student_obj,f2t[pt]['id'])
+                sub_doc = sub_obj.getContent()
+                d['Title'] = f2t[pt]['title']
+                for field in f2t[pt]['fields']:
+                    d[field] = mapping.get(field,'')
+    
+                if pt == "StudentApplication":
+                    #d['jamb_sex']  = 'M'
+                    #if mapping.get('sex'):
+                    #    d['jamb_sex']  = 'F'
+                    d['jamb_firstname'] = mapping.get('firstname',None)
+                    d['jamb_middlename'] = mapping.get('middlename',None)
+                    d['jamb_lastname'] = mapping.get('lastname',None)
+    
+                # if pt == "StudyCourse":
+                #     for von,zu in (('entry_mode','current_mode'),
+                #                    ('entry_session','current_session')):
+                #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
+                #             d[zu] = mapping[von]
+                sub_doc.edit(mapping = d)
+    
+                #import pdb;pdb.set_trace()
+                new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
+                if new_state != "remain":
+                    self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
+            self.portal_workflow.doActionFor(student_obj,transition)
+            student_obj.manage_setLocalRoles(student_id, ['Owner',])
+            break
+        return student_id,msg,mapping
+    ###)
+
+    def edit(self,mapping):
+        wftool = self.portal_workflow
+        "edit student records due import"
+        logger = logging.getLogger('WAeUPTool.mass_edit_student')
+        students_folder = self.portal_url.getPortalObject().campus.students
+        student_id = mapping.get('id',None)
+        jamb_reg_no = mapping.get('jamb_reg_no',None)
+        matric_no = mapping.get('matric_no',None)
+        editable_keys = mapping.keys()
+        msg = ''
+        while True:
+            if student_id:
+                res = self.students_catalog(id = student_id)
+                if not res:
+                    msg = "no student with id %s" % student_id
+                    break
+                student_record = res[0]
+                if matric_no and student_record.matric_no:
+                    if  matric_no != student_record.matric_no:
+                        msg = "old matric_no %s overwritten with %s" % (student_record.matric_no,matric_no)
+                        #logger.info("%s, old matric_no %s overwritten with %s" % (student_record.id,student_record.matric_no,matric_no))
+                if jamb_reg_no and student_record.jamb_reg_no:
+                    if jamb_reg_no != student_record.jamb_reg_no:
+                        msg = "old reg_no %s overwritten with %s" % (student_record.jamb_reg_no,jamb_reg_no)
+                        #logger.info("%s, old reg_no %s overwritten with %s" % (student_record.id,student_record.jamb_reg_no,jamb_reg_no))
+            elif jamb_reg_no:
+                res = self.students_catalog(jamb_reg_no = jamb_reg_no)
+                if not res:
+                    msg = "no student with jamb_reg_no %s" % jamb_reg_no
+                    break
+                student_record = res[0]
+                editable_keys.remove('jamb_reg_no')
+            elif matric_no:
+                res = self.students_catalog(matric_no = matric_no)
+                if not res:
+                    msg = "no student with matric_no %s" % matric_no
+                    break
+                student_record = res[0]
+                editable_keys.remove('matric_no')
+            ## included only to change wf state from admitted to returning
+            #if student_record.review_state not in ('admitted','objection_raised'):
+            #    return '%s' % student_record.id ,"student is not in state admitted or objection_raised"
+            ## end inclusion
+            student_id = student_record.id
+            student_obj = getattr(students_folder,student_id)
+            f2t = self.field2types_student
+            d = {}
+            any_change = False
+            #special treatment for StudentStudyLevel
+            d['verdict']  = mapping.get('current_verdict','')
+            d['session']  = mapping.get('current_session','')
+            current_level = mapping.get('current_level','')
+            while d['session'] and d['verdict'] and current_level:
+                sub_obj = getattr(student_obj,'study_course',None)
+                if sub_obj is None:
+                    break
+                level_obj = getattr(sub_obj,current_level,None)
+                if  level_obj is None:
+                    break
+                any_change = True
+                level_obj.getContent().edit(mapping = d)
+                try:
+                    wftool.doActionFor(level_obj,'close')
+                except:
+                    pass
+                break
+            for pt in f2t.keys():
+                #if pt == "StudentApplication":
+                #    d['jamb_sex']  = 'M'
+                #    if mapping.get('sex'):
+                #        d['jamb_sex']  = 'F'
+                intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
+                if intersect and pt not in ('StudentStudyLevel',):
+                    object_id = f2t[pt]['id']
+                    sub_obj = getattr(student_obj,object_id,None)
+                    if sub_obj is None:
+                        try:
+                            student_obj.invokeFactory(pt,object_id)
+                        except:
+                            continue
+                        sub_obj = getattr(student_obj,object_id)
+                        if f2t[pt]['title'] != '':
+                            d['Title'] = f2t[pt]['title']
+                    sub_doc = sub_obj.getContent()
+                    for field in intersect:
+                        changed = False
+                        if getattr(sub_doc,field,None) != mapping.get(field,''):
+                            any_change = True
+                            changed = True
+                            d[field] = mapping.get(field,'')
+                        if changed:
+                            sub_doc.edit(mapping = d)
+            ## included only to change wf state from admitted to returning
+            #    if student_record.review_state in ('admitted','objection_raised'):
+            #        new_state = f2t[pt]['wf_transition_return']
+            #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
+            #        if sub_obj and new_state != "remain":
+            #            try:
+            #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
+            #            except:
+            #                #logger.info('%s, wf transition %s of %s failed' % (student_id,new_state,sub_obj.id))
+            #                pass
+            #if student_record.review_state in ('admitted','objection_raised'):
+            #    wfaction = 'return'
+            #    try:
+            #        self.portal_workflow.doActionFor(student_obj,wfaction)
+            #        logger.info('%s, wf state changed' % student_id)
+            #        any_change = True
+            #    except:
+            #        logger.info('%s, wf transition failed, old state = %s' % (student_id,student_record.review_state))
+            #        pass
+            ## end inclusion
+            break
+        # if not any_change:
+        #     msg = 'not modified'
+        return student_id,msg,mapping
+    ###)
+
+class VerdictImport(WAeUPImport):###(
+    """ VerdictImport """
+    name = "verdict"
+    plural_name = "%ss" % name
+    commit_after = 100000
+    required_modes = ('create','edit')
+    
+    def edit(self,mapping):
+        "edit student verdicts"
+        wftool = self.portal_workflow
+        logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
+        students_folder = self.portal_url.getPortalObject().campus.students
+        student_id = mapping.get('id',None)
+        matric_no = mapping.get('matric_no',None)
+        editable_keys = mapping.keys()
+        while True:
+            key = ''
+            msg = ''
+            if student_id:
+                student_record = self.students_catalog.getRecordByKey(student_id)
+                if student_record is None:
+                    #return '',"no student with id %s" % student_id
+                    msg = "no student with id %s" % student_id
+                    break
+                if matric_no and student_record.matric_no and matric_no != student_record.matric_no:
+                    msg = 'student %s: matric_no %s does not match %s' % (student_record.id,
+                                                                          student_record.matric_no,
+                                                                          matric_no)
+                    break
+                mapping['matric_no'] = student_record.matric_no
+            elif matric_no:
+                res = self.students_catalog(matric_no = matric_no)
+                if not res:
+                    msg = "no student with matric_no %s" % matric_no
+                    break
+                student_record = res[0]
+                editable_keys.remove('matric_no')
+            else:
+                msg = "no id or matric_no specified"
+                break
+            student_id = student_record.id
+            mapping['id'] = student_id
+            d = {}
+            #import pdb;pdb.set_trace()
+            any_change = False
+            #special treatment for StudentStudyLevel
+            current_session = d['session'] = mapping.get('current_session','')
+            if current_session and student_record.session != current_session:
+                msg = 'student %s: imported session %s does not match current_session %s' % (student_id,
+                                                                                            current_session,
+                                                                                            student_record.session)
+                break
+            current_level = mapping.get('current_level','')
+            if not current_level.isdigit():
+                msg = 'student %s: imported level is empty' % (student_id,)
+                break
+            if current_level and student_record.level != current_level:
+                msg = 'student %s: imported level %s does not match current_level %s' % (student_id,
+                                                                                        current_level,
+                                                                                        student_record.level)
+                break
+            student_review_state =  student_record.review_state
+            if student_review_state == 'deactivated':
+                msg = "student %s in review_state %s" % (student_id, student_review_state)
+                break
+            if student_review_state not in ('courses_validated','returning'):
+                msg = "student %s in wrong review_state %s" % (student_id, student_review_state)
+                break
+            student_obj = getattr(students_folder,student_id)
+            # f2t = self.field2types_student
+            study_course_obj = getattr(student_obj,'study_course',None)
+            if study_course_obj is None:
+                msg = 'student %s: no study_course object' % student_id
+                break
+            level_obj = getattr(study_course_obj,current_level,None)
+            if  level_obj is None:
+                msg = 'student %s: no study_level object for level %s' % (student_id,
+                                                                                current_level)
+                break
+            verdict = d['verdict'] = d['current_verdict']  = mapping.get('current_verdict','')
+    
+            #if verdict == student_record.verdict:
+            #    msg = 'student %s: verdict already set to %s' % (student_id,
+            #                                                            verdict)
+    
+            level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
+            if level_review_state != "closed":
+                wftool.doActionFor(level_obj,'close')
+                # msg = 'student %s: level %s is not closed' % (student_id,
+                #                                                      current_level)
+    
+            study_course_obj.getContent().edit(mapping = d)
+            level_obj.getContent().edit(mapping = d)
+            if student_review_state != "returning":
+                wftool.doActionFor(student_obj,'return')
+            # try:
+            #     wftool.doActionFor(level_obj,'close')
+            # except:
+            #     pass
+            break
+        return student_id,msg,mapping
+    ###)
+
Index: WAeUP_SRP/base/WAeUPTool.py
===================================================================
--- WAeUP_SRP/base/WAeUPTool.py	(revision 3171)
+++ WAeUP_SRP/base/WAeUPTool.py	(revision 3172)
@@ -43,8 +43,13 @@
 from Students import makeCertificateCode
 from Globals import package_home,INSTANCE_HOME
+from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
+from WAeUPImport import CourseImport,CourseResultImport
+from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
+from utils import makeDigest
 import DateTime,time
 import logging
 import transaction
 import csv,re,os,sys
+import md5
 from shutil import copy2
 from Products.AdvancedQuery import Eq, Between, Le,In
@@ -61,5 +66,4 @@
         return getattr(object,name)
     return None
-
 
 class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
@@ -1335,706 +1339,241 @@
     ###)
 
-    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
-    def mass_create_faculty(self,mapping):
-        "create a faculty"
-        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
-        academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['code']
-        if getattr(academics_folder,fid,None) is not None:
-            return '', "Faculty with ID: %s exists" % fid
-        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
-        try:
-            academics_folder.invokeFactory('Faculty', fid)
-        except BadRequest,E:
-            return '', "%s" % E
-        f = getattr(academics_folder,fid,None)
-        f.getContent().edit(mapping=mapping)
-        return fid,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
-    def mass_edit_faculty(self,mapping):
-        "edit a faculty"
-        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
-        academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['code']
-        f = getattr(academics_folder,fid,None)
-        if f is None:
-            return '', "Faculty with ID: %s does not exist" % fid
-        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
-        f.getContent().edit(mapping=mapping)
-        return fid,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
-    def mass_create_department(self,mapping):
-        "create a department in the correct faculty"
-        logger = logging.getLogger('WAeUPTool.mass_create_department')
-        fid = mapping['faculty_code']
-        if getattr(self,'_v_faculties',None) is None:
-            res = self.portal_catalog(portal_type = "Faculty")
-            self._v_faculties = {}
-            for f in res:
-                self._v_faculties[f.getId] = f.getObject()
-        f = self._v_faculties.get(fid,None)
-        if f is None:
-            return '', "No Faculty with ID: %s" % fid
-        else:
-            did = mapping.get('code')
-            d = getattr(f,did,None)
-            if d is None or d.portal_type == "Faculty":
-                logger.info('Creating Department %(code)s, %(title)s' % mapping)
-                try:
-                    f.invokeFactory('Department', did)
-                except BadRequest,E:
-                    return '', "%s" % E
-                d = getattr(f,did)
-                d.invokeFactory('CoursesFolder','courses')
-                courses = getattr(d,'courses')
-                dict = {'Title': 'Courses'}
-                courses.getContent().edit(mapping=dict)
-                d.invokeFactory('CertificatesFolder','certificates')
-                certificates = getattr(d,'certificates')
-                dict = {'Title': 'Certificates'}
-                certificates.getContent().edit(mapping=dict)
-            d.getContent().edit(mapping=mapping)
-        return did,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
-    def mass_edit_department(self,mapping):
-        "create a department in the correct faculty"
-        logger = logging.getLogger('WAeUPTool.mass_create_department')
-        academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['faculty_code']
-        did = mapping.get('code')
-        try:
-            d = getattr(getattr(academics_folder,fid),did,None)
-        except KeyError:
-            return '', "Department %s or Faculty %s wrong" % (did,fid)
-        else:
-            if d is None or d.portal_type == "Faculty":
-                logger.info('Editing Department %(code)s, %(title)s' % mapping)
-            d.getContent().edit(mapping=mapping)
-        return did,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
-    def mass_create_course(self,mapping):
-        #import pdb;pdb.set_trace()
-        if getattr(self,'_v_course_list',None) is None:
-            self._v_course_list = []
-        if getattr(self,'_v_departments',None) is None:
-            res = self.portal_catalog(portal_type = "Department")
-            self._v_department_courses = {}
-            for d in res:
-                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
-        did = mapping['department_code']
-        d = self._v_department_courses.get(did,None)
-        if d is None:
-            return '', "No Department with ID: %s" % did
-        course_id = mapping.get('code')
-        if course_id in self._v_course_list:
-            return '', "Duplicate Course ID: %s" % did
-        c = getattr(d,course_id,None)
-        if c is not None:
-            return '', "Duplicate Course ID: %s" % did
-        try:
-            d.invokeFactory('Course', course_id)
-        except BadRequest,E:
-            return '', "%s" % E
-        self._v_course_list.append(course_id)
-        c = getattr(d,course_id)
-        c.getContent().edit(mapping=mapping)
-        return course_id,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
-    def mass_edit_course(self,mapping):
-        #import pdb;pdb.set_trace()
-        course_id = mapping.get('code')
-        res = self.portal_catalog(id=course_id)
-        if not res:
-            return '', "No Course with ID: %s" % course_id
-        c = res[0].getObject()
-        c.getContent().edit(mapping=mapping)
-        return course_id,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
-    def mass_create_certificate(self,mapping):
-        if getattr(self,'_v_certificate_list',None) is None:
-            self._v_certificate_list = []
-        if getattr(self,'_v_department_certificates',None) is None:
-            res = self.portal_catalog(portal_type = "Department")
-            self._v_department_certificates = {}
-            for d in res:
-                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
-        did = mapping['department_code']
-        d = self._v_department_certificates.get(did,None)
-        if d is None:
-            return '', "No Department with ID: %s" % did
-        certificate_id = mapping.get('code')
-        if certificate_id in self._v_certificate_list:
-            return '', "Duplicate Certificate ID: %s" % did
-        c = getattr(d,certificate_id,None)
-        if c is not None:
-            return '', "Duplicate Certificate ID: %s" % did
-        try:
-            d.invokeFactory('Certificate', certificate_id)
-        except BadRequest,E:
-            return '', "%s" % E
-        self._v_certificate_list.append(certificate_id)
-        c = getattr(d,certificate_id)
-        c.getContent().edit(mapping=mapping)
-        return certificate_id,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
-    def mass_edit_certificate(self,mapping):
-        #import pdb;pdb.set_trace()
-        certificate_id = mapping.get('code')
-        res = self.portal_catalog(id=certificate_id)
-        if not res:
-            return '', "No Certificate with ID: %s" % certificate_id
-        c = res[0].getObject()
-        c.getContent().edit(mapping=mapping)
-        return certificate_id,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
-    def mass_create_application(self,mapping):
-        #import pdb;pdb.set_trace()
-        reg_no = mapping.get('reg_no')
-        try:
-            self.applicants_catalog.addRecord(**mapping)
-        except ValueError:
-            return '', "applicant record with reg_no %s already exists" % reg_no
-        return reg_no,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
-    def mass_edit_application(self,mapping):
-        #import pdb;pdb.set_trace()
-        reg_no = mapping.get('reg_no')
-        status = mapping.get('status')
-        res = self.applicants_catalog(reg_no = reg_no)
-        if len(res):
-            if res[0].status == 'created' and status != 'created':
-                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
-            elif status == 'created' and res[0].status != 'created':
-                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
-            else:
-                self.applicants_catalog.modifyRecord(**mapping)
-                return reg_no,''
-        else:
-            return '', "applicant record with reg_no %s does not exist" % reg_no
-
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
-    def mass_create_course_result(self,mapping):
-        #import pdb;pdb.set_trace()
-        students_folder = self.portal_url.getPortalObject().campus.students
-        if getattr(self,'_v_courses',None) is None:
-            res = self.courses_catalog()
-            self._v_courses = {}
-            for brain in res:
-                self._v_courses[brain.code] = brain
-        if getattr(self,'_v_level_created',None) is None:
-            self._v_level_created = []
-        course_id = mapping.get('code')
-        if course_id not in self._v_courses.keys():
-            return '', "No course with ID: %s" % course_id
-        id_key = ''
-        for id_key in ('student_id','matric_no'):
-            id_field = mapping.get(id_key,None)
-            if id_field is not None:
-                student_id = id_field
-                break
-        query = Eq(id_key,id_field)
-        res = self.students_catalog.evalAdvancedQuery(query)
-        if not res:
-            return '', "no student with %(id_key)s %(id_field)s" % vars()
-        student_rec = res[0]
-        if id_field != "student_id":
-            mapping['student_id'] = student_rec.id
-        student_id = student_rec.id
-        level_id = mapping['level_id']
-        code = mapping['code']
-        if student_id not in self._v_level_created:
-            try:
-                context = getattr(getattr(students_folder,
-                                          "%(student_id)s" % vars()),
-                                  'study_course')
-            except:
-                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
-            if level_id not in context.objectIds():
-                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
-                level = getattr(context,"%(level_id)s" % vars())
-                self.portal_workflow.doActionFor(level,'open')
-                # the session string must not be copied into the level object
-                current_verdict = getattr(student_rec,'current_verdict','')
-                current_session = getattr(student_rec,'current_session','')
-                if current_verdict and student_rec.current_level == level_id:
-                    level.getContent().edit(mapping={'verdict': "%s" %
-                                                     current_verdict,
-                                                     'session': "%s" %
-                                                     current_session,
-                                                    })
-                    self.portal_workflow.doActionFor(level,'close')
-            self._v_level_created += student_id,
-        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
-        for k in ('semester','credits',):
-            mapping[k] = getattr(self._v_courses[course_id],k)
-        try:
-            self.course_results.addRecord(**mapping)
-        except ValueError:
-            return '', "course result already exists: %s" % key
-        return key,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
-    def mass_edit_course_result(self,mapping):
-        #import pdb;pdb.set_trace()
-        while True:
-            result_id = mapping.get('id',None)
-            if result_id is not None:
-                mapping['key'] = key = result_id
-                break
-            for id_key in ('student_id','matric_no'):
-                id_field = mapping.get(id_key,None)
-                if id_field is not None:
-                    student_id = id_field
-                    break
-            query = Eq(id_key,id_field)
-            res = self.students_catalog.evalAdvancedQuery(query)
-            if not res:
-                return '', "No student with %(id_key)s %(id_field)s" % vars()
-            if id_field == "matric_no":
-                mapping['student_id'] = res[0].id
-            elif id_field == "student_id":
-                mapping['matric_no'] = res[0].matric_no
-            if not res:
-                return '', "No student with %(id_key)s %(id_field)s" % vars()
-            student_rec = res[0]
-            student_id = student_rec.id
-            level_id = mapping['level_id']
-            code = mapping['code']
-            mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
-            break
-        try:
-            self.course_results.modifyRecord(**mapping)
-        except KeyError:
-            return key, "No course result to edit: %s" % key,mapping
-        return key,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_delete_course_result') ###(
-    def mass_delete_course_result(self,mapping):
-        id_key = ''
-        for id_key in ('student_id','matric_no'):
-            id_field = mapping.get(id_key,None)
-            if id_field is not None:
-                student_id = id_field
-                break
-        query = Eq(id_key,id_field)
-        res = self.students_catalog.evalAdvancedQuery(query)
-        if not res:
-            return '', "No student with %(id_field)s: %(id_key)s" % vars()
-        if id_field != "student_id":
-            mapping['student_id'] = res[0].id
-        if not res:
-            return '', "no student with %(id_key)s %(id_field)s" % vars()
-        student_rec = res[0]
-        student_id = student_rec.id
-        level_id = mapping['level_id']
-        code = mapping['code']
-        key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
-        # for k in ('semester',):
-        #     mapping[k] = getattr(self._v_courses[course_id],k)
-        if self.course_results.getRecordByKey(key) is None:
-            return '', "no course-result with %(key)s" % vars()
-        self.course_results.deleteRecord(key)
-        return key,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
-    def mass_create_certificate_course(self,mapping):
-        if getattr(self,'_v_courses',None) is None:
-            res = self.courses_catalog()
-            self._v_courses= [course.code for course in res]
-        if getattr(self,'_v_ceritficates',None) is None:
-            res = self.portal_catalog(portal_type = "Certificate")
-            self._v_certificates = {}
-            for cert in res:
-                self._v_certificates[cert.getId] = cert.getObject()
-        certificate_course_id = mapping.get('code')
-        if certificate_course_id not in self._v_courses:
-            return '', "No Course with ID: %s" % certificate_course_id
-        cert_id = mapping['certificate_code']
-        cert = self._v_certificates.get(cert_id,None)
-        if cert is None:
-            return '', "No Certificate with ID: %s" % cert_id
-        level_id = mapping.get('level')
-        level = getattr(cert,level_id,None)
-        if level is None:
-            cert.invokeFactory('StudyLevel', level_id)
-            level = getattr(cert,level_id,None)
-        elif hasattr(level,certificate_course_id):
-            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
-            (certificate_course_id,cert_id,level_id)
-        level.invokeFactory('CertificateCourse', certificate_course_id)
-        c = getattr(level,certificate_course_id)
-        c.getContent().edit(mapping=mapping)
-        return certificate_course_id,''
-    ###)
-
-    field2types_student = {   ###(
-                      'StudentApplication':
-                          {'id': 'application',
-                           'title': 'Application Data',
-                           'wf_transition_return': 'close',
-                           'wf_transition_admit': 'remain',
-                           'fields':
-                             ('jamb_reg_no',
-                              'entry_mode',
-                              'entry_session',
-                              'jamb_score',
-                              'app_email',
-                              'jamb_age',
-                              'jamb_state',
-                              'jamb_lga',
-                              'jamb_sex',
-                              )
-                              },
-                      #'StudentPume':
-                      #    {'id': 'pume',
-                      #     'title': 'Pume Data',
-                      #     'wf_transition_return': 'close',
-                      #     'wf_transition_admit': 'close',
-                      #     'fields':
-                      #       ('pume_score',
-                      #        )
-                      #        },
-                      'StudentClearance':
-                          {'id': 'clearance',
-                           'title': 'Clearance/Eligibility Record',
-                           'wf_transition_return': 'close',
-                           'wf_transition_admit': 'remain',
-                           'fields':
-                             ('matric_no',
-                              'nationality',
-                              'lga',
-                              'birthday',
-                              )
-                              },
-                         'StudentPersonal':
-                          {'id': 'personal',
-                           'title': 'Personal Data',
-                           'wf_transition_return': 'open',
-                           'wf_transition_admit': 'remain',
-                           'fields':
-                             ('firstname',
-                              'middlename',
-                              'lastname',
-                              'sex',
-                              'email',
-                              'phone',
-                              'perm_address',
-                              )
-                              },
-                         'StudentStudyCourse':
-                          {'id': 'study_course',
-                           'title': 'Study Course',
-                           'wf_transition_return': 'open',
-                           'wf_transition_admit': 'remain',
-                           'fields':
-                             ('study_course',
-                              'current_level',
-                              'current_session',
-                              'current_mode',
-                              'current_verdict',
-                              'previous_verdict',
-                              )
-                              },
-                         # 'StudentStudyLevel':
-                         #  {'id': 'current_level',
-                         #   'title': '',
-                         #   'wf_transition_return': 'open',
-                         #   'wf_transition_admit': 'remain',
-                         #   'fields':
-                         #     ('verdict',
-                         #      'session',
-                         #      )
-                         #      },
-                         'PaymentsFolder':
-                          {'id': 'payments',
-                           'title': 'Payments',
-                           'wf_transition_return': 'open',
-                           'wf_transition_admit': 'open',
-                           'fields':
-                             ()
-                              },
-                         }
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
-    def mass_create_student(self,mapping):
-        "create student records due import"
-        logger = logging.getLogger('WAeUPTool.mass_create_student')
-        students_folder = self.portal_url.getPortalObject().campus.students
-        jamb_reg_no = mapping.get('jamb_reg_no',None)
-        if jamb_reg_no:
-            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
-            if res:
-                return '',"jamb_reg_no exists"
-        matric_no = mapping.get('matric_no',None)
-        if matric_no:
-            res = self.students_catalog(matric_no = matric_no)
-            if res:
-                return '',"matric_no exists"
-        sid = self.waeup_tool.generateStudentId('?')
-        students_folder.invokeFactory('Student', sid)
-        student_obj = getattr(students_folder,sid)
-        f2t = self.field2types_student
-        d = {}
-        transition = mapping.get('reg_transition','admit')
-        if transition not in ('admit','return'):
-            return '',"no valid transition provided"
-        for pt in f2t.keys():
-            student_obj.invokeFactory(pt,f2t[pt]['id'])
-            sub_obj = getattr(student_obj,f2t[pt]['id'])
-            sub_doc = sub_obj.getContent()
-            d['Title'] = f2t[pt]['title']
-            for field in f2t[pt]['fields']:
-                d[field] = mapping.get(field,'')
-
-            if pt == "StudentApplication":
-                #d['jamb_sex']  = 'M'
-                #if mapping.get('sex'):
-                #    d['jamb_sex']  = 'F'
-                d['jamb_firstname'] = mapping.get('firstname',None)
-                d['jamb_middlename'] = mapping.get('middlename',None)
-                d['jamb_lastname'] = mapping.get('lastname',None)
-
-            # if pt == "StudyCourse":
-            #     for von,zu in (('entry_mode','current_mode'),
-            #                    ('entry_session','current_session')):
-            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
-            #             d[zu] = mapping[von]
-            sub_doc.edit(mapping = d)
-
-            #import pdb;pdb.set_trace()
-            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
-            if new_state != "remain":
-                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
-        self.portal_workflow.doActionFor(student_obj,transition)
-        student_obj.manage_setLocalRoles(sid, ['Owner',])
-        return sid,''
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
-    def mass_edit_student(self,mapping):
-        wftool = self.portal_workflow
-        "edit student records due import"
-        logger = logging.getLogger('WAeUPTool.mass_edit_student')
-        students_folder = self.portal_url.getPortalObject().campus.students
-        sid = mapping.get('id',None)
-        jamb_reg_no = mapping.get('jamb_reg_no',None)
-        matric_no = mapping.get('matric_no',None)
-        editable_keys = mapping.keys()
-        if sid:
-            res = self.students_catalog(id = sid)
-            if not res:
-                return '',"no student with id %s" % sid
-            if matric_no and res[0].matric_no and\
-              matric_no != res[0].matric_no:
-                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
-            if jamb_reg_no and res[0].jamb_reg_no and\
-              jamb_reg_no != res[0].jamb_reg_no:
-                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
-        elif jamb_reg_no:
-            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
-            if not res:
-                return '',"no student with jamb_reg_no %s" % jamb_reg_no
-            editable_keys.remove('jamb_reg_no')
-        elif matric_no:
-            res = self.students_catalog(matric_no = matric_no)
-            if not res:
-                return '',"no student with matric_no %s" % matric_no
-            editable_keys.remove('matric_no')
-
-        ## included only to change wf state from admitted to returning
-        #if res[0].review_state not in ('admitted','objection_raised'):
-        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
-        ## end inclusion
-
-        sid = res[0].id
-        student_obj = getattr(students_folder,sid)
-        f2t = self.field2types_student
-        d = {}
-        #import pdb;pdb.set_trace()
-        any_change = False
-        #special treatment for StudentStudyLevel
-        d['verdict']  = mapping.get('current_verdict','')
-        d['session']  = mapping.get('current_session','')
-        current_level = mapping.get('current_level','')
-        while d['session'] and d['verdict'] and current_level:
-            sub_obj = getattr(student_obj,'study_course',None)
-            if sub_obj is None:
-                break
-            level_obj = getattr(sub_obj,current_level,None)
-            if  level_obj is None:
-                break
-            any_change = True
-            level_obj.getContent().edit(mapping = d)
-            try:
-                wftool.doActionFor(level_obj,'close')
-            except:
-                pass
-            break
-        for pt in f2t.keys():
-            #if pt == "StudentApplication":
-            #    d['jamb_sex']  = 'M'
-            #    if mapping.get('sex'):
-            #        d['jamb_sex']  = 'F'
-            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
-            if intersect and pt not in ('StudentStudyLevel',):
-                object_id = f2t[pt]['id']
-                sub_obj = getattr(student_obj,object_id,None)
-                if sub_obj is None:
-                    try:
-                        student_obj.invokeFactory(pt,object_id)
-                    except:
-                        continue
-                    sub_obj = getattr(student_obj,object_id)
-                    if f2t[pt]['title'] != '':
-                        d['Title'] = f2t[pt]['title']
-                sub_doc = sub_obj.getContent()
-                for field in intersect:
-                    changed = False
-                    if getattr(sub_doc,field,None) != mapping.get(field,''):
-                        any_change = True
-                        changed = True
-                        d[field] = mapping.get(field,'')
-                    if changed:
-                        sub_doc.edit(mapping = d)
-
-
-        ## included only to change wf state from admitted to returning
-        #    if res[0].review_state in ('admitted','objection_raised'):
-        #        new_state = f2t[pt]['wf_transition_return']
-        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
-        #        if sub_obj and new_state != "remain":
-        #            try:
-        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
-        #            except:
-        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
-        #                pass
-        #if res[0].review_state in ('admitted','objection_raised'):
-        #    wfaction = 'return'
-        #    try:
-        #        self.portal_workflow.doActionFor(student_obj,wfaction)
-        #        logger.info('%s, wf state changed' % sid)
-        #        any_change = True
-        #    except:
-        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
-        #        pass
-        ## end inclusion
-
-
-        if any_change:
-            return sid,''
-        else:
-            return sid,'not modified'
-    ###)
-
-    security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
-    def mass_edit_verdict(self,mapping):
-        wftool = self.portal_workflow
-        "edit student verdicts"
-        logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
-        students_folder = self.portal_url.getPortalObject().campus.students
-        student_id = mapping.get('id',None)
-        matric_no = mapping.get('matric_no',None)
-        editable_keys = mapping.keys()
-        if student_id:
-            student_record = self.students_catalog.getRecordByKey(student_id)
-            if student_record is None:
-                return '',"no student with id %s" % student_id
-            if matric_no and student_record.matric_no and\
-              matric_no != student_record.matric_no:
-                return '', 'student %s: matric_no %s does not match %s' % (student_record.id,
-                                                                          student_record.matric_no,
-                                                                          matric_no)
-            mapping['matric_no'] = student_record.matric_no
-        elif matric_no:
-            res = self.students_catalog(matric_no = matric_no)
-            if not res:
-                return '',"no student with matric_no %s" % matric_no
-            student_record = res[0]
-            editable_keys.remove('matric_no')
-        else:
-            return '',"no id or matric_no specified"
-        student_id = student_record.id
-        d = {}
-        #import pdb;pdb.set_trace()
-        any_change = False
-        #special treatment for StudentStudyLevel
-        current_session = d['session'] = mapping.get('current_session','')
-        if current_session and student_record.session != current_session:
-            msg = 'student %s: imported session %s does not match current_session %s' % (student_id,
-                                                                                        current_session,
-                                                                                        student_record.session)
-            return student_id,msg,mapping
-        current_level = mapping.get('current_level','')
-        if not current_level.isdigit():
-            return '','student %s: imported level is empty' % (student_id,)
-        if current_level and student_record.level != current_level:
-            msg = 'student %s: imported level %s does not match current_level %s' % (student_id,
-                                                                                    current_level,
-                                                                                    student_record.level)
-            return student_id,msg,mapping
-        student_review_state =  student_record.review_state
-        if student_review_state == 'deactivated':
-            return '',"student %s in review_state %s" % (student_id, student_review_state)
-        if student_review_state not in ('courses_validated','returning'):
-            msg = "student %s in wrong review_state %s" % (student_id, student_review_state)
-            return student_id,msg,mapping
-        student_obj = getattr(students_folder,student_id)
-        f2t = self.field2types_student
-        study_course_obj = getattr(student_obj,'study_course',None)
-        if study_course_obj is None:
-            return '','student %s: no study_course object' % student_id
-        level_obj = getattr(study_course_obj,current_level,None)
-        if  level_obj is None:
-            return '','student %s: no study_level object for level %s' % (student_id,
-                                                                             current_level)
-        verdict = d['verdict'] = d['current_verdict']  = mapping.get('current_verdict','')
-
-        #if verdict == student_record.verdict:
-        #    return '','student %s: verdict already set to %s' % (student_id,
-        #                                                            verdict)
-
-        level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
-        if level_review_state != "closed":
-            wftool.doActionFor(level_obj,'close')
-            # return '','student %s: level %s is not closed' % (student_id,
-            #                                                      current_level)
-
-        study_course_obj.getContent().edit(mapping = d)
-        level_obj.getContent().edit(mapping = d)
-        if student_review_state != "returning":
-            wftool.doActionFor(student_obj,'return')
-        # try:
-        #     wftool.doActionFor(level_obj,'close')
-        # except:
-        #     pass
-        return student_id,''
-    ###)
-
     security.declareProtected(ModifyPortalContent,"importData")###(
     def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
+        """load data from CSV values"""
+        import transaction
+        import random
+        students_folder = self.portal_url.getPortalObject().campus.students
+        pending_only = False
+        pend_str = '_pending'
+        elapse = time.time()
+        #
+        # preparations
+        #
+        if name.endswith(pend_str):
+            pending_only = True
+            name = name[:-len(pend_str)]
+        mode = "create"
+        if edit:
+            if filename.endswith('_toDelete'):
+                mode = "delete"
+            else:
+                mode = "edit"
+        # schema = stool._getOb(iname)
+        # if schema is None:
+        #     em = 'No such schema %s' % iname
+        #     logger.error('No such schema %s' % iname)
+        #     return em
+        # pending_schema = stool._getOb("%s_pending" % iname)
+        # if pending_schema is None:
+        #     em = 'No such schema %s_pending' % iname
+        #     logger.error('No such schema %s_pending' % iname)
+        #     return em
+        # layout = ltool._getOb(iname)
+        # if layout is None:
+        #     em = 'No such layout %s' % iname
+        #     logger.error(em)
+        #     return em
+        importer_name = ''.join([part.capitalize() for part in name.split('_')])
+        importer = eval("%sImport" % importer_name)(mode,self)
+        logger = importer.logger
+        if importer.init_errors:
+            logger.info(self.init_errors)
+            return self.init_errors
+        member = importer.member
+        current = importer.current
+        import_date = importer.import_date
+        #
+        # not_imported
+        #
+        info = importer.info
+        data_keys = importer.data_keys
+        csv_keys = importer.csv_keys 
+        #csv_keys.extend(info.keys())
+        headline_mapping = dict((k,k) for k in csv_keys)
+        #
+        # pending
+        #
+        pending_path = importer.pending_path
+        pending_tmp = importer.pending_tmp
+        pending_backup = importer.pending_backup
+        pending_fn = importer.pending_fn
+        imported_path = importer.imported_path
+        imported_fn = importer.imported_fn
+        pending = []
+        pending_digests = []
+        #total_added_to_pending = 0
+        if not pending_only:
+            pending,pending_digests = importer.makeIdLists()
+            pending_at_start = len(pending)
+        datafile = open(pending_tmp,"w")
+        pending_csv_writer = csv.DictWriter(datafile,
+                                                    csv_keys,
+                                                    extrasaction='ignore')
+        pending_csv_writer.writerow(headline_mapping)
+        datafile.close()
+        #
+        # imported
+        #
+        if not os.path.exists(imported_path):
+            datafile = open(imported_path,"w")
+            imported_csv_writer = csv.DictWriter(datafile,
+                                                 csv_keys,
+                                                 extrasaction='ignore')
+            imported_csv_writer.writerow(headline_mapping)
+            datafile.close()
+        start = True
+        tr_count = 0
+        total = 0
+        total_added_to_imported = 0
+        total_pending = 0
+        if pending_only:
+            import_source_path = pending_path
+        else:
+            import_source_path = "%s/import/%s.csv" % (i_home,filename)
+        if not os.path.exists(import_source_path):
+            em = 'No  %(import_source_fn)s' % vars()
+            return em
+        import_source_fn = os.path.split(import_source_path)[1]
+        info['imported_from'] = import_source_fn
+        attrs = csv.reader(open(import_source_path,"rb")).next()
+        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
+                                                        or k.strip() in info.keys())]
+        diff2schema = set(import_keys).difference(set(importer.schema.keys()))
+        diff2layout = set(import_keys).difference(set(importer.layout.keys()))
+        if diff2schema and diff2schema != set(['id',]):
+            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
+            return msg
+        if mode in importer.required_modes:
+            required_keys = [importer.layout.getIdUnprefixed(id)
+                                for id,widget in importer.layout.objectItems()
+                                if widget.is_required]
+            if not set(required_keys).issubset(set(import_keys)):
+                diff2import = set(required_keys).difference(set(import_keys))
+                msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
+                return msg
+        #
+        # start importing
+        #
+        try:
+            reader = csv.DictReader(open(import_source_path,"rb"))
+        except:
+            msg = 'Error reading %s.csv' % filename
+            logger.error(msg)
+            return msg
+        items = [item for item in reader]
+        total_to_import = len(items)
+        tti_float = float(total_to_import) 
+        if pending_only:
+            pending_at_start = total_to_import
+        count = 0
+        imported = []
+        old_commit_count = 0
+        with_error_count = imported_count = 0
+        already_in = 0
+        for record in items:
+            item = {}
+            for k,v in record.items():
+                if v:
+                    item[k.strip()] = v.strip()
+            count += 1
+            #item = dict((k.strip(),l.strip()) for (k,l) in item.items())
+            if start:
+                start = False
+                adapters = [MappingStorageAdapter(importer.schema, item)]
+                logger.info('%(member)s starts import from %(import_source_fn)s in %(mode)s mode' % vars())
+                #logger.info('%s starts import from %s in %s mode with schema and layout %s' % (member,filename,mode,iname))
+            dm = DataModel(item, adapters,context=self)
+            ds = DataStructure(data=item,datamodel=dm)
+            error_string = ""
+            total += 1
+            for k in import_keys:
+                if k not in item.keys() or k not in importer.validators.keys():
+                    continue
+                if not importer.validators[k](ds,mode=mode):
+                    if error_count:
+                        error_string += ' ++ '
+                    error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
+                                                                           ds.getErrorMapping(k)))
+                    error_count += 1
+            if error_string:
+                error = error_string
+                id = ''
+                mapping = item
+            else:
+                temp_item = item.copy()
+                temp_item.update(dm)
+                results = importer.import_method(temp_item)
+                id = results[0]
+                error = results[1]
+                mapping = results[2]
+            data_string = ",".join("%s : %s" % (k,v) for k,v in mapping.items())
+            info['error'] = error
+            mapping.update(info)
+            if error:
+                digest = makeDigest(mapping,data_keys)
+                with_error_count += 1
+                percent_finished = (with_error_count + imported_count)/tti_float*100
+                if digest not in pending_digests:
+                    pending_digests += digest,
+                    pending.append(mapping)
+                    if not pending_only:
+                        logger.info("%(percent_finished)6.3f %% done added to %(pending_fn)s %(data_string)s" % vars())
+                else:
+                    already_in += 1
+                    #logger.info("already in %(pending_fn)s %(data_string)s" % vars())
+                    pass
+            else:
+                imported_count += 1
+                imported += mapping,
+                percent_finished = (with_error_count + imported_count)/tti_float*100
+                logger.info("%(percent_finished)6.3f %% done  imported and added to  %(imported_fn)s %(data_string)s" % vars())
+            finished = count > total_to_import - 1
+            must_commit = False
+            if len(imported) != old_commit_count:
+                if not len(imported) % importer.commit_after:
+                    must_commit = True
+                    old_commit_count = len(imported)
+                    
+            if must_commit or finished:
+                if len(imported):
+                    transaction.commit()
+                    datafile = open(imported_path,"a")
+                    writer = csv.DictWriter(datafile,
+                                            csv_keys,
+                                            extrasaction='ignore')
+                    writer.writerows(imported)
+                    datafile.close()
+                    total_added_to_imported += len(imported)
+                    imported = []
+                if len(pending) > 0:
+                    datafile = open(pending_tmp,"a")
+                    writer = csv.DictWriter(datafile,
+                                            csv_keys,
+                                            extrasaction='ignore')
+                    writer.writerows(pending)
+                    datafile.close()
+                    total_pending += len(pending)
+                    #total_added_to_pending += len(pending)
+                    pending = []
+                if not finished:
+                    msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
+                    logger.info(msg)
+        elapse = time.time() - elapse
+        copy2(pending_tmp,pending_path)
+        msg = "Finished importing in %(mode)s mode from %(import_source_fn)s in %(elapse)6.2f seconds: " % vars()
+        msg += "read = %(count)d; %(imported_fn)s: added = %(total_added_to_imported)d; " % vars()
+        if pending_only:
+            removed_pending = pending_at_start - total_pending
+            msg += "%(pending_fn)s: removed = %(removed_pending)d, " % vars()
+        else:
+            added_pending = total_pending - pending_at_start 
+            msg += "%(pending_fn)s: added = %(added_pending)d, already in = %(already_in)s " % vars()
+        msg += ", total %(total_pending)d; " % vars()
+        logger.info(msg)
+        return msg
+    ###)
+
+    def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###(
         """load data from CSV values"""
         import transaction
@@ -2322,151 +1861,4 @@
     ###)
 
-    def old_importData(self,filename,name,edit=False,bypass_queue_catalog=False):###(
-        """load data from CSV values"""
-        import transaction
-        import random
-
-        pm = self.portal_membership
-        member = pm.getAuthenticatedMember()
-
-        logger = logging.getLogger('WAeUPTool.importData')
-        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
-        students_folder = self.portal_url.getPortalObject().campus.students
-        start = True
-        tr_count = 0
-        total_imported = 0
-        total_not_imported = 0
-        total = 0
-        iname = "import_%s" % name
-        if iname in ('import_application','import_course_result',):
-            commit_after = 2000
-        else:
-            commit_after = 100
-        stool = getToolByName(self, 'portal_schemas')
-        ltool = getToolByName(self, 'portal_layouts')
-        schema = stool._getOb(iname)
-        if schema is None:
-            em = 'No such schema %s' % iname
-            logger.error('No such schema %s' % iname)
-            return em
-        layout = ltool._getOb(iname)
-        if layout is None:
-            em = 'No such layout %s' % iname
-            logger.error(em)
-            return em
-        validators = {}
-        for widget in layout.keys():
-            validators[widget] = layout[widget].validate
-        mode = "create"
-        if edit:
-            if filename.endswith('_toDelete'):
-                mode = "delete"
-            else:
-                mode = "edit"
-        importer_name = "mass_%(mode)s_%(name)s" % vars()
-        importer = getattr(self, '%s' % importer_name,None)
-        if importer is None:
-            em = 'No importer function %s' % importer_name
-            logger.error(em)
-            return em
-        not_imported = []
-        imported = []
-        try:
-            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
-        except:
-            em = 'Error reading %s.csv' % filename
-            logger.error(em)
-            return em
-        for item in items:
-            #import pdb;pdb.set_trace()
-            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
-            if start:
-                start = False
-                adapters = [MappingStorageAdapter(schema, item)]
-                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
-                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
-                import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
-                diff2schema = set(import_keys).difference(set(schema.keys()))
-                diff2layout = set(import_keys).difference(set(layout.keys()))
-                if diff2schema:
-                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
-                    return em
-                if mode == "create":
-                    required_keys = [layout.getIdUnprefixed(id)
-                                     for id,widget in layout.objectItems()
-                                     if widget.is_required]
-                    if not set(required_keys).issubset(set(import_keys)):
-                        diff2import = set(required_keys).difference(set(import_keys))
-                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
-                        return em
-                s = ','.join(['"%s"' % fn for fn in import_keys])
-                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
-                s = '"id",' + s
-                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
-                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
-                format_error = format + ',"%(Error)s"'
-                format = '"%(id)s",'+ format
-
-            dm = DataModel(item, adapters,context=self)
-            ds = DataStructure(data=item,datamodel=dm)
-            error_string = ""
-            total += 1
-            error_count = 0
-            for k in import_keys:
-                if not validators[k](ds,mode=mode):
-                    if error_count:
-                        error_string += ' ++ '
-                    error_string += "%s: %s" % (k,
-                                                  self.translation_service(ds.getError(k),
-                                                                           ds.getErrorMapping(k)))
-                    error_count += 1
-            if not error_string:
-                temp_item = item.copy()
-                temp_item.update(dm)
-                temp_item['id'],error = importer(temp_item)
-                if error:
-                    error_string += error
-                else:
-                    item = temp_item
-            if error_string:
-                item['Error'] = error_string
-                not_imported.append(format_error % item)
-                total_not_imported += 1
-            else:
-                em = format % item
-                imported.append(em)
-                tr_count += 1
-                total_imported += 1
-                logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(em)s" % vars())
-
-            if total and not total % commit_after:
-                transaction.commit()
-                if len(not_imported) > 0:
-                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
-                             '\n'.join(not_imported) + '\n')
-                    not_imported = []
-                if len(imported) > 0:
-                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
-                             '\n'.join(imported) + '\n')
-                    imported = []
-                em = '%d transactions committed\n' % (tr_count)
-                regs = []
-                logger.info(em)
-                tr_count = 0
-                #if total > 100:
-                #    return
-        if len(imported) > 0:
-            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
-                                                '\n'.join(imported))
-        if len(not_imported) > 0:
-            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
-                                                '\n'.join(not_imported))
-        #if mode.endswith('e'):
-        #    mode = mode[:-1]
-        em = "Finished importing in %(mode)s mode from %(filename)s.csv: %(total_imported)d imported, %(total_not_imported)d not imported (of total %(total)d)" % vars()
-        logger.info(em)
-        return em
-    ###)
-
     security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
     def moveImagesToFS(self,student_id="O738726"):
Index: WAeUP_SRP/base/doc/srp_documentation.mm
===================================================================
--- WAeUP_SRP/base/doc/srp_documentation.mm	(revision 3171)
+++ WAeUP_SRP/base/doc/srp_documentation.mm	(revision 3172)
@@ -2,17 +2,4 @@
 <!-- To view this file, download free mind mapping software FreeMind from http://freemind.sourceforge.net -->
 <node CREATED="1170079518946" ID="Freemind_Link_650524897" MODIFIED="1172489583516" TEXT="WAeUP Documentation">
-<node CREATED="1172489553062" ID="Freemind_Link_1276207160" MODIFIED="1172489556316" POSITION="right" TEXT="Glossary">
-<node CREATED="1172588742298" ID="Freemind_Link_495359346" LINK="../profiles/default/schemas.xml" MODIFIED="1173007237566" TEXT="Field Names (Attributes)"/>
-<node CREATED="1172588782286" ID="Freemind_Link_1234262632" LINK="../profiles/default/types.xml" MODIFIED="1172588916308" TEXT="Types"/>
-<node CREATED="1172588791669" ID="Freemind_Link_832896495" LINK="../profiles/default/layouts.xml" MODIFIED="1173007318993" TEXT="Labels (Titles)"/>
-<node CREATED="1196238383254" ID="Freemind_Link_105251244" MODIFIED="1196238410482" TEXT="Variables Naming">
-<node CREATED="1196238414175" ID="Freemind_Link_1389091182" MODIFIED="1196238448729" TEXT="Student">
-<node CREATED="1196238450567" ID="Freemind_Link_412992846" MODIFIED="1196239946904" TEXT="students_folder : &#xa;  the folder with students "/>
-<node CREATED="1196240327175" ID="Freemind_Link_1324355897" MODIFIED="1196240334961" TEXT="student_id"/>
-<node CREATED="1196240336208" ID="Freemind_Link_392767031" MODIFIED="1196240345004" TEXT="student_obj"/>
-<node CREATED="1196239950564" ID="Freemind_Link_1283570746" MODIFIED="1196240283454" TEXT="student_record:&#xa;record from students_catalog&#xa;students_catalog.getRecordByKey(student_id)&#xa;"/>
-</node>
-</node>
-</node>
 <node CREATED="1172489529057" FOLDED="true" ID="Freemind_Link_43083883" MODIFIED="1172489545651" POSITION="right" TEXT="SRP Data Structure">
 <node CREATED="1170079621592" ID="Freemind_Link_1297017754" MODIFIED="1170079640872" TEXT="Campus">
@@ -131,5 +118,5 @@
 <node CREATED="1172156481009" ID="Freemind_Link_1817393683" LINK="waeup_student_subobject_wf.gif" MODIFIED="1172903933018" TEXT="Diagram"/>
 </node>
-<node CREATED="1170883343230" FOLDED="true" ID="Freemind_Link_998392827" MODIFIED="1172162754390" TEXT="User Interface">
+<node CREATED="1170883343230" ID="Freemind_Link_998392827" MODIFIED="1172162754390" TEXT="User Interface">
 <node CREATED="1170883352226" FOLDED="true" ID="Freemind_Link_556396570" MODIFIED="1172163796143" TEXT="Student">
 <node CREATED="1171124982312" ID="Freemind_Link_683557361" LINK="../profiles/default/layouts/student_clearance_fe.xml" MODIFIED="1172590208847" TEXT="Layout: student_clearance_fe" VSHIFT="2"/>
@@ -205,5 +192,5 @@
 </node>
 </node>
-<node CREATED="1170082200667" ID="Freemind_Link_140160478" MODIFIED="1170082364784" TEXT="Payments">
+<node CREATED="1170082200667" FOLDED="true" ID="Freemind_Link_140160478" MODIFIED="1170082364784" TEXT="Payments">
 <node CREATED="1171711430276" ID="Freemind_Link_86727434" MODIFIED="1171711435834" TEXT="Id: payments"/>
 <node CREATED="1171711367726" ID="Freemind_Link_1873791240" MODIFIED="1171711381666" TEXT="Type: PaymentsFolder"/>
@@ -330,12 +317,37 @@
 </node>
 </node>
-<node CREATED="1200500239581" ID="Freemind_Link_318351857" MODIFIED="1200500251927" POSITION="right" TEXT="Old Documentation">
-<node CREATED="1200500254038" ID="Freemind_Link_1556830810" LINK="../../WAeUP_Doc/InstructionsPart1.html" MODIFIED="1200500331010" TEXT="InstructionsPart1"/>
-<node CREATED="1200500254038" ID="Freemind_Link_1914051391" LINK="../../WAeUP_Doc/InstructionsPart2.html" MODIFIED="1200500386200" TEXT="InstructionsPart2"/>
-<node CREATED="1200500254038" ID="Freemind_Link_49864370" LINK="../../WAeUP_Doc/InstructionsPart3.html" MODIFIED="1200500412566" TEXT="InstructionsPart3"/>
-<node CREATED="1200500254038" ID="Freemind_Link_1242839170" LINK="../../WAeUP_Doc/InstructionsPart4.html" MODIFIED="1200500436427" TEXT="InstructionsPart4"/>
-<node CREATED="1200500254038" ID="Freemind_Link_629032224" LINK="../../WAeUP_Doc/InstructionsPart5.html" MODIFIED="1200500454751" TEXT="InstructionsPart5"/>
-<node CREATED="1200500254038" ID="Freemind_Link_1794158955" LINK="../../WAeUP_Doc/InstructionsPart6.html" MODIFIED="1200500470854" TEXT="InstructionsPart6"/>
-<node CREATED="1200500254038" ID="Freemind_Link_350494723" LINK="../../WAeUP_Doc/InstructionsPart7.html" MODIFIED="1200500488122" TEXT="InstructionsPart7"/>
+<node CREATED="1200500239581" FOLDED="true" ID="Freemind_Link_318351857" MODIFIED="1200500251927" POSITION="left" TEXT="Old Documentation">
+<node CREATED="1200500254038" ID="Freemind_Link_1556830810" LINK="file:///home/perry/Zope/Products/WAeUP_Doc/InstructionsPart1.html" MODIFIED="1200561147628" TEXT="Setup of an SRP instance"/>
+<node CREATED="1200500254038" ID="Freemind_Link_1914051391" LINK="../../WAeUP_Doc/InstructionsPart2.html" MODIFIED="1200502851884" TEXT="SRP Backend Usage"/>
+<node CREATED="1200500254038" ID="Freemind_Link_1794158955" LINK="../../WAeUP_Doc/InstructionsPart6.html" MODIFIED="1200503022233" TEXT="Contact forms"/>
+<node CREATED="1200500254038" ID="Freemind_Link_1242839170" LINK="../../WAeUP_Doc/InstructionsPart4.html" MODIFIED="1200502952099" TEXT="Testing Accommodation"/>
+<node CREATED="1200500254038" ID="Freemind_Link_49864370" LINK="../../WAeUP_Doc/InstructionsPart3.html" MODIFIED="1200503076521" TEXT="Testing Application"/>
+<node CREATED="1200500254038" ID="Freemind_Link_350494723" LINK="../../WAeUP_Doc/InstructionsPart7.html" MODIFIED="1200503054207" TEXT="Testing Clearance"/>
+<node CREATED="1200500254038" ID="Freemind_Link_629032224" LINK="../../WAeUP_Doc/InstructionsPart5.html" MODIFIED="1200502991059" TEXT="Testing PUME"/>
+</node>
+<node CREATED="1200931065202" ID="Freemind_Link_1866715831" MODIFIED="1200931080820" POSITION="left" TEXT="New Documentation">
+<node CREATED="1200500254038" ID="Freemind_Link_1876930383" LINK="../../WAeUP_Doc/ImportVerdicts.html" MODIFIED="1200931201900" TEXT="Import Verdicts HOWTO"/>
+<node CREATED="1200500254038" ID="Freemind_Link_1970450916" LINK="../../WAeUP_Doc/ImportNew.html" MODIFIED="1203323590449" TEXT="New Import procedure "/>
+</node>
+<node CREATED="1172489553062" FOLDED="true" ID="Freemind_Link_1276207160" MODIFIED="1172489556316" POSITION="left" TEXT="Glossary">
+<node CREATED="1172588742298" ID="Freemind_Link_495359346" LINK="../profiles/default/schemas.xml" MODIFIED="1173007237566" TEXT="Field Names (Attributes)"/>
+<node CREATED="1172588782286" ID="Freemind_Link_1234262632" LINK="../profiles/default/types.xml" MODIFIED="1172588916308" TEXT="Types"/>
+<node CREATED="1172588791669" ID="Freemind_Link_832896495" LINK="../profiles/default/layouts.xml" MODIFIED="1173007318993" TEXT="Labels (Titles)"/>
+<node CREATED="1196238383254" ID="Freemind_Link_105251244" MODIFIED="1196238410482" TEXT="Variables Naming">
+<node CREATED="1196238414175" ID="Freemind_Link_1389091182" MODIFIED="1196238448729" TEXT="Student">
+<node CREATED="1196238450567" ID="Freemind_Link_412992846" MODIFIED="1196239946904" TEXT="students_folder : &#xa;  the folder with students "/>
+<node CREATED="1196240327175" ID="Freemind_Link_1324355897" MODIFIED="1196240334961" TEXT="student_id"/>
+<node CREATED="1196240336208" ID="Freemind_Link_392767031" MODIFIED="1196240345004" TEXT="student_obj"/>
+<node CREATED="1196239950564" ID="Freemind_Link_1283570746" MODIFIED="1196240283454" TEXT="student_record:&#xa;record from students_catalog&#xa;students_catalog.getRecordByKey(student_id)&#xa;"/>
+</node>
+</node>
+</node>
+<node CREATED="1201877638475" ID="Freemind_Link_312955008" MODIFIED="1201877709483" POSITION="right" TEXT="Quantity structure">
+<node CREATED="1201877714895" ID="Freemind_Link_1862106138" MODIFIED="1201877724946" TEXT="Students">
+<node CREATED="1201877848576" ID="Freemind_Link_552788283" MODIFIED="1201877974303" TEXT="Zope Database: "/>
+<node CREATED="1201877861169" ID="Freemind_Link_759610055" MODIFIED="1201877937556" TEXT="Filesystem: Images aprox 600 kb per student"/>
+</node>
+<node CREATED="1201877727911" ID="Freemind_Link_380010877" MODIFIED="1201877735965" TEXT="Academics"/>
+<node CREATED="1201877737347" ID="Freemind_Link_764317466" MODIFIED="1201877737347" TEXT=""/>
 </node>
 </node>
Index: WAeUP_SRP/base/utils.py
===================================================================
--- WAeUP_SRP/base/utils.py	(revision 3172)
+++ WAeUP_SRP/base/utils.py	(revision 3172)
@@ -0,0 +1,29 @@
+#-*- mode: python; mode: fold -*-
+# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
+# Author: Joachim Schmitz (js@aixtraware.de)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+# 02111-1307, USA.
+#
+# $Id$
+"""
+Standalone utils.
+"""
+import md5
+def makeDigest(item,data_keys):
+    d = {}
+    for key in data_keys:
+        d[key] = str(item.get(key,''))
+    return md5.new(str(d)).digest()
+
