#-*- mode: python; mode: fold -*- # (C) Copyright 2005 The WAeUP group # Author: Joachim Schmitz (js@aixtraware.de) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA # 02111-1307, USA. # # $Id: WAeUPImport.py 9233 2012-09-26 04:11:16Z henrik $ """The WAeUP Tool Box. """ #from AccessControl import ClassSecurityInfo #from Acquisition import aq_inner #from Acquisition import aq_parent #from Globals import DTMLFile #from Globals import InitializeClass from OFS.SimpleItem import SimpleItem from zExceptions import BadRequest #from Products.CMFCore.utils import getToolByName #from Products.CPSSchemas.DataStructure import DataStructure #from Products.CPSSchemas.DataModel import DataModel #from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter from Products.CMFCore.ActionProviderBase import ActionProviderBase #from Products.CMFCore.permissions import View #from Products.ZCatalog.ZCatalog import ZCatalog #from Products.CMFCore.permissions import ModifyPortalContent #from Products.CMFCore.permissions import ManagePortal from Products.CMFCore.utils import UniqueObject #from Products.CMFCore.URLTool import URLTool from Products.CMFCore.utils import getToolByName from Globals import package_home,INSTANCE_HOME from Products.AdvancedQuery import Eq, Between, Le,In import csv,re,os,sys from shutil import copy2 import DateTime,time import logging p_home = package_home(globals()) i_home = INSTANCE_HOME from utils import makeDigest NO_KEY = '----' IGNORE = 'ignore' class WAeUPImport(UniqueObject, SimpleItem, ActionProviderBase): ###( """ WAeUPImport """ required_modes = ('create',) def __init__(self,waeup_tool): ###( self.students_folder = waeup_tool.portal_url.getPortalObject().campus.students self.member = member = waeup_tool.portal_membership.getAuthenticatedMember() self.import_date = DateTime.DateTime() self.imported_by = str(member) #self.current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") self.waeup_tool = waeup_tool self.session = waeup_tool.getSessionId()[0] self.academics_folder = waeup_tool.portal_url.getPortalObject().campus.academics self.schema_tool = getToolByName(waeup_tool, 'portal_schemas') self.layout_tool = getToolByName(waeup_tool, 'portal_layouts') self.types_tool = getToolByName(waeup_tool, 'portal_types') self.portal_workflow = getToolByName(waeup_tool, 'portal_workflow') self.portal_url = getToolByName(waeup_tool, 'portal_url') self.portal_catalog = waeup_tool.portal_catalog self.portal_directories = waeup_tool.portal_directories self.students_catalog = waeup_tool.students_catalog self.courses_catalog = waeup_tool.courses_catalog self.course_results = waeup_tool.course_results self.payments_catalog = waeup_tool.payments_catalog self.applicants_catalog = waeup_tool.applicants_catalog self.removed_student_ids = waeup_tool.removed_student_ids #self.mode = mode # self.import_method = getattr(self, '%s' % mode,None) errors = [] # if self.import_method is None: # errors.append('No importer method %s' % mode) self.pending_path = "%s/import/%s.pending" % (i_home,self.plural_name) self.pending_tmp = "%s/import/%s.pending.tmp" % (i_home,self.plural_name) self.pending_backup = "%s/import/%s.pending.old" % (i_home,self.plural_name) self.pending_fn = os.path.split(self.pending_path)[1] self.imported_path = "%s/import/%s.imported" % (i_home,self.plural_name) self.imported_fn = os.path.split(self.imported_path)[1] iname = "import_%s" % self.name self.logger = logging.getLogger('WAeUPImport.%sImport' % self.plural_name.capitalize()) self.schema = self.schema_tool._getOb(iname,None) #self.pending_schema = self.schema_tool._getOb("%s_pending" % iname,None) self.layout = self.layout_tool._getOb(iname,None) while True: if self.schema is None: errors.append('no schema %s' % iname) # if self.pending_schema is None: # self.pending_schema = self.schema if self.layout is None: errors.append('no such layout %s' % iname) if errors: break self.data_keys = self.schema.keys() self.csv_keys = self.schema.keys() info = {} info['imported_from'] = '' info['import_record_no'] = 0 info['imported_by'] = self.imported_by info['import_date'] = self.import_date.strftime("%d/%m/%y %H:%M:%S") info['error'] = '' self.info = info self.csv_keys.extend(self.info) self.validators = {} for widget in self.layout.keys(): self.validators[widget] = self.layout[widget].validate self.required_keys = {} for mode in self.required_modes: self.required_keys[mode] = [self.layout.getIdUnprefixed(id) for id,widget in self.layout.objectItems() if widget.is_required] break self.init_errors = ','.join(errors) ###) def findStudent(self,mode,student_id=None, matric_no=None, jamb_reg_no=None): ###( #logger = logging.getLogger('WAeUPImport.findStudent') student_record = None msg = '' key_used = '' while True: if student_id: key_used = 'student_id' res = self.students_catalog(id = student_id) if not res: msg = "no student with id %s" % student_id break student_record = res[0] #if matric_no and student_record.matric_no: # if matric_no != student_record.matric_no: # #msg = "old matric_no %s overwritten with %s" % (student_record.matric_no,matric_no) # logger.info("%s, old matric_no %s overwritten with %s" % (student_record.id,student_record.matric_no,matric_no)) #if jamb_reg_no and student_record.jamb_reg_no: # if jamb_reg_no != student_record.jamb_reg_no: # #msg = "old reg_no %s overwritten with %s" % (student_record.jamb_reg_no,jamb_reg_no) # logger.info("%s, old reg_no %s overwritten with %s" % (student_record.id,student_record.jamb_reg_no,jamb_reg_no)) elif jamb_reg_no: key_used = 'jamb_reg_no' res = self.students_catalog(jamb_reg_no = jamb_reg_no) if res: if mode == 'create': msg = "jamb_reg_no exists" break else: if mode == 'edit': msg = "no student with jamb_reg_no %s" % jamb_reg_no break student_record = res[0] elif matric_no: key_used = 'matric_no' res = self.students_catalog(matric_no = matric_no) if not res: msg = "no student with matric_no %s" % matric_no break student_record = res[0] else: msg = "neither id, matric_no nor reg_no specified" break d = {} d['student_record'] = student_record d['key_used'] = key_used d['msg'] = msg return d ###) def makeIdLists(self): ###( pending_digests = [] pending = [] # pending_student_ids = [] # pending_matric_nos = [] # pending_reg_ns = [] if os.path.exists(self.pending_path): datafile = open(self.pending_path,"r") reader = csv.reader(datafile) old_headline = reader.next() #datafile.seek(0) pending_csv_reader = csv.DictReader(datafile,old_headline,) #pending_csv_reader.next() # skip headline for item in pending_csv_reader: digest = makeDigest(item,self.data_keys) if digest not in pending_digests: pending_digests += digest, pending += item, datafile.close() #copy2(self.pending_path,self.pending_backup) return pending, pending_digests ###) def checkHeadline(self,headline): ###( """ check the headline of a csv file """ import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore') or k.strip() in self.info.keys())] # import_keys = [k.strip() for k in headline if not (k.strip() in self.info.keys())] diff2schema = set(import_keys).difference(set(self.schema.keys())) diff2layout = set(import_keys).difference(set(self.layout.keys())) #if diff2schema and diff2schema != set(['id',]): if diff2schema: return list(diff2schema) return [] ###) def getHeadlineFields(self,headline,values): ###( """ check the headline of a csv file """ # import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore') # or k.strip() in self.info.keys())] import_keys = [k.strip() for k in headline if not (k.strip() in self.info.keys())] info_keys = [k.strip() for k in headline if k.strip() in self.info.keys()] si = set(import_keys) ss = set(self.schema.keys()) invalid_keys = si - ss keys = [] i = 0 duplicates = False singels = [] msg = '' while True: if len(values) != len(import_keys): if len(values) == len(import_keys) + len(info_keys): msg += "fields from pending data file in headline" else: msg += "%d fields in headline but %d values" % (len(import_keys),len(values)) break for k in import_keys: if k in singels: keys += (k,'%s' % k,values[i],'(duplicate)'), msg += ("duplicate %s," % k) keys[singels.index(k)] = (k,'%s' % k,values[singels.index(k)],'(duplicate)') elif k in invalid_keys and not k.startswith(IGNORE): keys += (k,NO_KEY,values[i],'(invalid)'), else: keys += (k,k,values[i],''), i += 1 singels += k, break return msg,keys ###) ###) class ApplicationImport(WAeUPImport):###( name = "application" plural_name = "%ss" % name commit_after = 1000 def create(self,mapping):###( reg_no = mapping.get('reg_no') msg = '' while True: try: self.applicants_catalog.addRecord(**mapping) except ValueError: msg = "applicant record with reg_no %s already exists" % reg_no break return reg_no,msg,mapping ###) def remove(self,mapping): reg_no = mapping.get('reg_no') msg = '' while True: if self.applicants_catalog.getRecordByKey(reg_no) is None: msg = "no application record with reg_no %(reg_no)s" % vars() break self.applicants_catalog.deleteRecord(reg_no) break return reg_no,msg,mapping def edit(self,mapping):###( reg_no = mapping.get('reg_no') status = mapping.get('status') jamb_reg_no = mapping.get('jamb_reg_no') msg = '' while True: if jamb_reg_no: res = self.applicants_catalog(jamb_reg_no = jamb_reg_no.upper()) if not len(res): res = self.applicants_catalog(jamb_reg_no = jamb_reg_no.lower()) if not len(res): msg = "applicant record with jamb_reg_no %s does not exist" % jamb_reg_no break else: res = self.applicants_catalog(reg_no = reg_no) if len(res): if res[0].status == 'created' and status != 'created': msg = "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no) elif status == 'created' and res[0].status != 'created': msg = "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no) else: self.applicants_catalog.modifyRecord(**mapping) else: msg = "applicant record with reg_no %s does not exist" % reg_no break return reg_no,msg,mapping ###) ###) class CertificateImport(WAeUPImport):###( name = "certificate" plural_name = "%ss" % name commit_after = 100000 def create(self,mapping):###( if getattr(self,'_v_certificate_list',None) is None: self._v_certificate_list = [] if getattr(self,'_v_department_certificates',None) is None: departments = self.portal_catalog(portal_type = "Department") self._v_department_certificates = {} for department in departments: certificates_container = getattr(department.getObject(),"certificates",None) self._v_department_certificates[department.getId] = {'container': certificates_container, 'certificates': certificates_container.objectIds(), } department_id = mapping['department_code'] msg = '' certificate_id = mapping.get('code') while True: department_certificates = self._v_department_certificates.get(department_id,None) if department_certificates is None: msg = "No Department with ID: %s" % department_id break certificates_container = department_certificates['container'] certificates = department_certificates['certificates'] if certificate_id in self._v_certificate_list: msg = "Duplicate Certificate ID: %s" % department_id break if certificate_id in certificates: msg = "Duplicate Certificate ID: %s" % department_id break try: certificates_container.invokeFactory('Certificate', certificate_id) except BadRequest,E: msg = "%s" % E break self._v_certificate_list.append(certificate_id) certificate = getattr(certificates_container,certificate_id) certificate.getContent().edit(mapping=mapping) break return certificate_id,msg,mapping ###) def edit(self,mapping):###( certificate_id = mapping.get('code') res = self.portal_catalog(id=certificate_id) msg = '' while True: if not res: msg = "no certificate with id: %s" % certificate_id break c = res[0].getObject() c.getContent().edit(mapping=mapping) break return certificate_id,msg,mapping ###) ###) class CourseImport(WAeUPImport):###( name = "course" plural_name = "%ss" % name commit_after = 1000 def create(self,mapping):###( if getattr(self,'_v_course_list',None) is None: self._v_course_list = [] if getattr(self,'_v_department_courses',None) is None: departments = self.portal_catalog(portal_type = "Department") self._v_department_courses = {} for department in departments: courses_container = getattr(department.getObject(),"courses",None) if courses_container is not None: self._v_department_courses[department.getId] = {'container': courses_container, 'courses': courses_container.objectIds(), } department_id = mapping['department_code'] course_id = mapping.get('code','') msg = '' while True: department_courses = self._v_department_courses.get(department_id,None) if department_courses is None: msg = "no department with id: %(department_id)s" % vars() break courses_container = department_courses['container'] courses = department_courses['courses'] if course_id in self._v_course_list: msg = "duplicate course id: %(course_id)s" % vars() break if course_id in courses: msg = "course %(course_id)s already exists in department %(department_id)s" % vars() break try: courses_container.invokeFactory('Course', course_id) except BadRequest,E: msg = "%s" % E break self._v_course_list.append(course_id) course = getattr(courses_container,course_id) course.getContent().edit(mapping=mapping) break return course_id,msg,mapping ###) def edit(self,mapping): ###( course_id = mapping.get('code','') course = self.courses_catalog.getRecordByKey(course_id) msg = '' while True: if course is None: msg = "no course with id: %s" % course_id break course_object = getattr(getattr(getattr(getattr(self.academics_folder,course.faculty), course.department), 'courses'), course_id) course_object.getContent().edit(mapping=mapping) break return course_id,msg,mapping ###) def remove(self,mapping): ###( course_id = mapping.get('code','') course = self.courses_catalog.getRecordByKey(course_id) msg = '' while True: if course is None: msg = "no course with id: %s" % course_id break courses = getattr(getattr(getattr(self.academics_folder,course.faculty), course.department), 'courses') courses.manage_delObjects((course_id),) break return course_id,msg,mapping ###) ###) class CourseResultImport(WAeUPImport):###( """ CourseresultImport """ name = "course_result" plural_name = "%ss" % name commit_after = 1000000 required_modes = ('create','edit','remove') def create(self,mapping):###( students_folder = self.portal_url.getPortalObject().campus.students if getattr(self,'_v_courses',None) is None: res = self.courses_catalog() self._v_courses = {} for brain in res: self._v_courses[brain.code] = brain if getattr(self,'_v_level_created',None) is None: self._v_level_created = [] if getattr(self,'_v_student_study_course',None) is None: self._v_student_study_course = {} msg = '' key = '' matric_no = mapping.get('matric_no','') id = mapping.get('id','') while True: course_id = mapping.get('code') if course_id not in self._v_courses.keys(): msg = "no course with id: %s" % course_id break result = self.findStudent('create',student_id=id,matric_no=matric_no) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = mapping['student_id'] = student_record.id level_id = mapping.get('level_id','') code = mapping.get('code','') level_ident = "%(student_id)s_%(level_id)s" % vars() if level_ident not in self._v_level_created: context = self._v_student_study_course.get(student_id,None) if context is None: try: context = getattr(getattr(students_folder, "%(student_id)s" % vars()), 'study_course') self._v_student_study_course[student_id] = context except: msg = "could not create level %(level_id)s for %(student_id)s" % vars() break if level_id not in context.objectIds(): context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars()) level = getattr(context,"%(level_id)s" % vars()) self.portal_workflow.doActionFor(level,'open') session_id = mapping.get('session_id','') if session_id: level.getContent().edit(mapping={'session': "%s" % session_id,}) self.portal_workflow.doActionFor(level,'close') self._v_level_created += level_ident, mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars() #overwrite semester and credits in create mode for k in ('semester','credits',): mapping[k] = getattr(self._v_courses[course_id],k) # edit course result in create mode if it exists if self.course_results.getRecordByKey(key) is None: try: self.course_results.addRecord(**mapping) except ValueError: msg = "unknown error: %s" % key else: self.course_results.modifyRecord(**mapping) break return key,msg,mapping ###) def edit(self,mapping): ###( msg = '' key = '' matric_no = mapping.get('matric_no','') id = mapping.get('id','') while True: result = self.findStudent('edit',student_id=id,matric_no=matric_no) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = student_record.id level_id = mapping.get('level_id','') code = mapping.get('code','') #code = mapping['code'] mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars() if self.course_results.getRecordByKey(key) is None: msg = "no course result with key %(key)s" % vars() break # don't overwrite session id but check if session ids match # (helps to avoid overwriting of course results) session_in_course_result = self.course_results.getRecordByKey(key)['session_id'] if mapping.get('session_id','') and not session_in_course_result == mapping.get('session_id',''): msg = "import session_id %s does not match session_id %s in course result" % (mapping['session_id'],session_in_course_result) break self.course_results.modifyRecord(**mapping) break return key,msg,mapping ###) def remove(self,mapping):###( key = '' msg = '' matric_no = mapping.get('matric_no','') id = mapping.get('id','') while True: result = self.findStudent('edit',student_id=id,matric_no=matric_no) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = student_record.id level_id = mapping.get('level_id','') code = mapping.get('code','') key = "%(student_id)s|%(level_id)s|%(code)s" % vars() if self.course_results.getRecordByKey(key) is None: msg = "no course result with key %(key)s" % vars() break self.course_results.deleteRecord(key) break return key,msg,mapping ###) ###) class CertificateCourseImport(WAeUPImport):###( name = "certificate_course" plural_name = "%ss" % name commit_after = 1000 required_modes = ('create','edit') def create(self,mapping): if getattr(self,'_v_courses',None) is None: res = self.courses_catalog() self._v_courses= [course.code for course in res] if getattr(self,'_v_ceritficates',None) is None: res = self.portal_catalog(portal_type = "Certificate") self._v_certificates = {} for cert in res: self._v_certificates[cert.getId] = cert.getObject() msg = '' while True: certificate_course_id = mapping.get('code') if certificate_course_id not in self._v_courses: msg = "no course with id: %s" % certificate_course_id break cert_id = mapping['certificate_code'] cert = self._v_certificates.get(cert_id,None) if cert is None: msg = "no certificate with id: %s" % cert_id break level_id = mapping.get('level') level = getattr(cert,level_id,None) if level is None: cert.invokeFactory('StudyLevel', level_id) level = getattr(cert,level_id,None) elif hasattr(level,certificate_course_id): msg = "duplicate certificate course id: %(certificate_course_id)s " % vars() msg += "in %(cert_id)s/ %(level_id)s" % vars() break level.invokeFactory('CertificateCourse', certificate_course_id) c = getattr(level,certificate_course_id) c.getContent().edit(mapping=mapping) break return certificate_course_id,msg,mapping ###) class DepartmentImport(WAeUPImport):###( name = "department" plural_name = "%ss" % name commit_after = 1000 def create(self,mapping):###( "create a department in the correct faculty" faculty_id = mapping['faculty_code'] msg = '' if getattr(self,'_v_faculties',None) is None: res = self.portal_catalog(portal_type = "Faculty") self._v_faculties = {} for f in res: self._v_faculties[f.getId] = f.getObject() department_id = mapping.get('code','') while True: faculty = self._v_faculties.get(faculty_id,None) if faculty is None: msg = "no faculty with id: %s" % faculty_id break else: d = getattr(faculty,department_id,None) if d is None or d.portal_type == "Faculty": try: faculty.invokeFactory('Department', department_id) except BadRequest,E: msg = "%s" % E break d = getattr(faculty,department_id) d.invokeFactory('CoursesFolder','courses') courses = getattr(d,'courses') dict = {'Title': 'Courses'} courses.getContent().edit(mapping=dict) d.invokeFactory('CertificatesFolder','certificates') certificates = getattr(d,'certificates') dict = {'Title': 'Certificates'} certificates.getContent().edit(mapping=dict) d.getContent().edit(mapping=mapping) break return department_id,msg,mapping ###) def edit(self,mapping): ###( "edit a department in the correct faculty" academics_folder = self.portal_url.getPortalObject().campus.academics faculty_id = mapping['faculty_code'] department_id = mapping.get('code','') msg = '' while True: try: d = getattr(getattr(academics_folder,faculty_id),department_id,None) except KeyError: msg = "department %s or faculty %s wrong" % (department_id,faculty_id) break if d is None or d.portal_type == "Faculty": msg = "department %s not found" % (department_id) break d.getContent().edit(mapping=mapping) break return department_id,msg,mapping ###) ###) class FacultyImport(WAeUPImport):###( name = "faculty" plural_name = "faculties" commit_after = 100 def create(self,mapping): ###( "create a faculty" academics_folder = self.portal_url.getPortalObject().campus.academics faculty_id = mapping.get('code','') msg = '' while True: if faculty_id in academics_folder.objectIds(): msg = "faculty with id: %s exists" % faculty_id break try: academics_folder.invokeFactory('Faculty', faculty_id) except BadRequest,E: msg = "%s" % E break f = getattr(academics_folder,faculty_id,None) f.getContent().edit(mapping=mapping) break return faculty_id,msg,mapping ###) def edit(self,mapping): ###( "edit a faculty" academics_folder = self.portal_url.getPortalObject().campus.academics faculty_id = mapping['code'] msg = '' while True: f = getattr(academics_folder,faculty_id,None) if f is None: msg = "faculty with id: %s does not exist" % faculty_id f.getContent().edit(mapping=mapping) break return faculty_id,msg,mapping ###) ###) class StudentImport(WAeUPImport):###( name = "student" plural_name = "%ss" % name commit_after = 500 field2types_student = { ###( 'StudentApplication': {'id': 'application', #'title': 'Application Data', 'wf_transition_return': 'close', 'wf_transition_admit': 'remain', 'wf_transition_graduate': 'close', 'wf_transition_pay_school_fee': 'close', 'wf_transition_validate_courses': 'close', 'wf_transition_reject_clearance': 'remain', 'fields': ('jamb_reg_no', 'entry_mode', 'entry_session', 'jamb_score', 'app_email', 'app_mobile', 'jamb_age', 'jamb_state', 'jamb_lga', 'jamb_sex', 'app_ac_pin', 'app_reg_pin', 'app_ac_date', ) }, 'StudentClearance': {'id': 'clearance', #'title': 'Clearance/Eligibility Record', 'wf_transition_return': 'close', 'wf_transition_admit': 'remain', 'wf_transition_graduate': 'close', 'wf_transition_pay_school_fee': 'close', 'wf_transition_validate_courses': 'close', 'wf_transition_reject_clearance': 'open', 'fields': ('matric_no', 'nationality', 'lga', 'birthday', 'clr_ac_pin', 'request_date', 'cleared_date', 'clearance_officer', ) }, 'StudentPersonal': {'id': 'personal', #'title': 'Personal Data', 'wf_transition_return': 'open', 'wf_transition_admit': 'remain', 'wf_transition_graduate': 'close', 'wf_transition_pay_school_fee': 'open', 'wf_transition_validate_courses': 'open', 'wf_transition_reject_clearance': 'remain', 'fields': ('firstname', 'middlename', 'lastname', 'sex', 'email', 'phone', 'perm_address', 'marit_stat', 'disabled', ) }, 'StudentStudyCourse': {'id': 'study_course', #'title': 'Study Course', 'wf_transition_return': 'open', 'wf_transition_admit': 'remain', 'wf_transition_graduate': 'close', 'wf_transition_pay_school_fee': 'open', 'wf_transition_validate_courses': 'open', 'wf_transition_reject_clearance': 'remain', 'fields': ('study_course', 'current_level', 'current_session', 'current_mode', #is no longer used and visible but can still be imported 'current_verdict', 'previous_verdict', ) }, 'StudentAccommodation': {'id': 'accommodation_final', 'title': '', 'wf_transition_return': 'remain', 'wf_transition_admit': 'remain', 'wf_transition_graduate': 'remain', 'wf_transition_pay_school_fee': 'remain', 'wf_transition_validate_courses': 'remain', 'wf_transition_reject_clearance': 'remain', 'fields': ('acco_maint_date', 'acco_maint_code', 'acco_maint_fee', 'acco_maint_sc_pin', 'acco_maint_pay_id', 'acco_res_date', 'acco_res_sc_pin', 'bed', 'session', 'student_status', ) }, 'PaymentsFolder': {'id': 'payments', #'title': 'Payments', 'wf_transition_return': 'open', 'wf_transition_admit': 'open', 'wf_transition_graduate': 'close', 'wf_transition_pay_school_fee': 'open', 'wf_transition_validate_courses': 'open', 'wf_transition_reject_clearance': 'remain', 'fields': () }, } ###) def create(self,mapping): ###( "create student records due import" logger = logging.getLogger('WAeUPImport.StudentImport.create') students_folder = self.portal_url.getPortalObject().campus.students jamb_reg_no = mapping.get('jamb_reg_no',None) matric_no = mapping.get('matric_no',None) entry_mode = mapping.get('entry_mode',None) password = mapping.get('password',None) msg = '' student_id = mapping.get('id',None) while True: if student_id: #msg = "student_id must not be specified in create mode" #break res = self.students_catalog(id = student_id) if res: msg = "student_id %s already assigned" % res[0].id break self.waeup_tool.restorePictureFolder(student_id) else: student_id = self.waeup_tool.generateStudentId('?') if jamb_reg_no and not entry_mode == 'transfer': res = self.students_catalog(jamb_reg_no = jamb_reg_no) if res: msg = "jamb_reg_no already assigned to student %s" % res[0].id break if matric_no: res = self.students_catalog(matric_no = matric_no) if res: msg = "matric_no already assigned to student %s" % res[0].id break if not matric_no and not jamb_reg_no: msg = "jamb_reg_no or matric_no must be specified" break students_folder.invokeFactory('Student', student_id) student_obj = getattr(students_folder,student_id) f2t = self.field2types_student d = {} transition = mapping.get('reg_transition','admit') if transition == 'admitted': transition = 'admit' elif transition == 'returning': transition = 'return' if transition not in ('admit','return','pay_school_fee','validate_courses','graduate','reject_clearance'): msg = "no valid transition provided" break for pt in f2t.keys(): if pt == 'StudentAccommodation': if mapping.get('bed',None): student_obj.invokeFactory(pt,f2t[pt]['id']) else: continue else: student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() #d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = mapping.get(field,'') if pt == "StudentApplication": #d['jamb_sex'] = 'M' #if mapping.get('sex'): # d['jamb_sex'] = 'F' d['jamb_firstname'] = mapping.get('firstname',None) d['jamb_middlename'] = mapping.get('middlename',None) d['jamb_lastname'] = mapping.get('lastname',None) # if pt == "StudyCourse": # for von,zu in (('entry_mode','current_mode'), # ('entry_session','current_session')): # if mapping.get(zu,None) is None and mapping.get(von,None) is not None: # d[zu] = mapping[von] sub_doc.edit(mapping = d) #import pdb;pdb.set_trace() new_state = f2t[pt]['wf_transition_%(transition)s' % vars()] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) self.portal_workflow.doActionFor(student_obj,transition) student_obj.manage_setLocalRoles(student_id, ['Owner',]) mapping['id'] = student_id if password: self.waeup_tool.makeStudentMember(student_id,password) break return student_id,msg,mapping ###) def remove(self,mapping): ###( "remove student records due import" logger = logging.getLogger('WAeUPImport.StudentImport.remove') students_folder = self.portal_url.getPortalObject().campus.students stdir = self.portal_directories.students student_id = mapping.get('id',None) msg = '' export_file = "%s/export/students_removed.csv" % (i_home) reimport_file = "%s/export/students_for_reimport.csv" % (i_home) toexport_for_reimport = { 'application': ('jamb_reg_no', 'entry_mode', 'entry_session', 'jamb_score', 'app_email', 'app_mobile', 'jamb_age', 'jamb_state', 'jamb_lga', 'jamb_sex', 'app_ac_pin', 'app_reg_pin', 'app_ac_date', ), 'clearance': ('matric_no', 'nationality', 'lga', 'birthday', 'clr_ac_pin', 'request_date', 'cleared_date', 'clearance_officer', ), 'study_course': ('study_course', 'current_level', 'current_session', 'current_verdict', 'previous_verdict', ), 'personal': ('firstname', 'middlename', 'lastname', 'sex', 'email', 'phone', 'perm_address', 'marit_stat', 'disabled', ), 'accommodation': ('acco_maint_date', 'acco_maint_code', 'acco_maint_fee', 'acco_maint_sc_pin', 'acco_maint_pay_id', 'acco_res_date', 'acco_res_sc_pin', 'bed', 'session', 'student_status', ), } sub_types = ['StudentApplication','StudentClearance','StudentPersonal','StudentStudyCourse','StudentAccommodation'] non_image_keys = {} for i in sub_types: pt_ob = getattr(self.types_tool,i) sub_schemas = pt_ob.schemas for i in sub_schemas: if i.startswith('student_'): schema = getattr(self.schema_tool,i) field_id = i.replace('student_','') non_image_keys[field_id] = [key for key in schema.keys() if schema[key].meta_type != "CPS Image Field"] # export field names must be unique but, unfortunately, firstname, middlename and lastname are not # thus we have to omit them non_image_keys['clearance'].remove('firstname') non_image_keys['clearance'].remove('middlename') non_image_keys['clearance'].remove('lastname') toexport = non_image_keys #import pdb;pdb.set_trace() while True: if hasattr(students_folder,student_id): # begin export line = [] fields = ['student_id','reg_state','password'] line_for_reimport = [] fields_for_reimport = ['student_id','reg_state','password','import_mode'] #fields = [] #for f in self.students_catalog.schema(): # fields.append(f) for k in toexport.keys(): for f in toexport[k]: fields.append(f) for k in toexport_for_reimport.keys(): for f in toexport_for_reimport[k]: fields_for_reimport.append(f) if not os.path.exists(export_file): headline = ','.join(fields) open(export_file,"a").write(headline +'\n') if not os.path.exists(reimport_file): headline_for_reimport = ','.join(fields_for_reimport) open(reimport_file,"a").write(headline_for_reimport +'\n') format = '"%(' + ')s","%('.join(fields) + ')s"' format_for_reimport = '"%(' + ')s","%('.join(fields_for_reimport) + ')s"' res = self.students_catalog(id = student_id) student = res[0] student_obj = getattr(students_folder,student_id) #d = student.getFormattedStudentEntry(student) d = {'student_id':student_id,'reg_state':student.review_state,'import_mode':'create'} for k in toexport.keys()[0:]: if k.startswith('acco'): session = self.session subobject = 'accommodation_' + session else: subobject = k try: object = getattr(student_obj,subobject) object_doc = object.getContent() except: for f in toexport[k]: d[f] = '' continue for f in toexport[k]: d[f] = getattr(object_doc,f,'') if hasattr(stdir, student_id): d['password'] = self.waeup_tool.getCredential(student_id) stdir.deleteEntry(student_id) else: d['password'] = "not set" self.waeup_tool.removePictureFolder(student_id) line.append(format % d) open(export_file,"a").write('\n'.join(line) +'\n') line_for_reimport.append(format_for_reimport % d) open(reimport_file,"a").write('\n'.join(line_for_reimport) +'\n') self.course_results.exportRemoveAllCourses(student_id = student_id, export = True, remove = True) # remove should be false, otherwise an uncatalog error occurs if payment objects exist self.payments_catalog.exportRemoveAllPayments(student_id = student_id, export = True, remove = False) self.waeup_tool.exportAllStudyLevels(student_id = student_id) # end export students_folder.manage_delObjects((student_id),) try: self.removed_student_ids.addRecord(id = student_id) except: msg = "Student already in removed_student_ids" else: msg = "Student object not found" break break return student_id,msg,mapping def edit(self,mapping): ###( "edit student records due import" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.StudentImport.edit') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) jamb_reg_no = mapping.get('jamb_reg_no',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() password = mapping.get('password',None) msg = '' while True: if student_id: res = self.students_catalog(id = student_id) if not res: msg = "no student with id %s" % student_id break student_record = res[0] if matric_no and student_record.matric_no: if matric_no != student_record.matric_no and not matric_no == 'transferred': res = self.students_catalog(matric_no = matric_no) if res: msg = "matric_no already assigned to student %s" % res[0].id break msg = "old matric_no %s overwritten with %s" % (student_record.matric_no,matric_no) #logger.info("%s, old matric_no %s overwritten with %s" % (student_record.id,student_record.matric_no,matric_no)) if jamb_reg_no and student_record.jamb_reg_no: if jamb_reg_no != student_record.jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if res: msg = "jamb_reg_no already assigned to student %s" % res[0].id break msg = "old jamb_reg_no %s overwritten with %s" % (student_record.jamb_reg_no,jamb_reg_no) #logger.info("%s, old reg_no %s overwritten with %s" % (student_record.id,student_record.jamb_reg_no,jamb_reg_no)) elif jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if not res: msg = "no student with jamb_reg_no %s" % jamb_reg_no break student_record = res[0] editable_keys.remove('jamb_reg_no') elif matric_no: res = self.students_catalog(matric_no = matric_no) if not res: msg = "no student with matric_no %s" % matric_no break student_record = res[0] editable_keys.remove('matric_no') student_id = student_record.id student_obj = getattr(students_folder,student_id) f2t = self.field2types_student d = {} any_change = False #special treatment for StudentStudyLevel d['verdict'] = mapping.get('current_verdict','') d['session'] = mapping.get('current_session','') current_level = mapping.get('current_level','') transition = mapping.get('reg_transition',None) change_reg_state = mapping.get('change_reg_state',None) # for brute force method # the validate_courses import transition is not really useful because it does not execute validate_courses.py if transition and transition not in ('admit','return','graduate','pay_school_fee','validate_courses','reject_clearance'): msg = "no valid transition provided" break if transition: #import pdb;pdb.set_trace() try: self.portal_workflow.doActionFor(student_obj,transition) except: msg = "reg_transition not allowed" break elif change_reg_state: msg = self.waeup_tool.changeStudentState(student_obj, change_reg_state) if msg: break while d['session'] and d['verdict'] and current_level: sub_obj = getattr(student_obj,'study_course',None) if sub_obj is None: break level_obj = getattr(sub_obj,current_level,None) if level_obj is None: break any_change = True level_obj.getContent().edit(mapping = d) try: wftool.doActionFor(level_obj,'close') except: pass break for pt in f2t.keys(): #if pt == "StudentApplication": # d['jamb_sex'] = 'M' # if mapping.get('sex'): # d['jamb_sex'] = 'F' intersect = set(f2t[pt]['fields']).intersection(set(editable_keys)) #import pdb;pdb.set_trace() object_id = f2t[pt]['id'] sub_obj = getattr(student_obj,object_id,None) if intersect and pt not in ('StudentStudyLevel',): if sub_obj is None: try: student_obj.invokeFactory(pt,object_id) except: continue sub_obj = getattr(student_obj,object_id) #if f2t[pt]['title'] != '': # d['Title'] = f2t[pt]['title'] sub_doc = sub_obj.getContent() for field in intersect: changed = False if getattr(sub_doc,field,None) != mapping.get(field,''): any_change = True changed = True d[field] = mapping.get(field,'') if changed: sub_doc.edit(mapping = d) if transition: new_state = f2t[pt]['wf_transition_%(transition)s' % vars()] if new_state != "remain": try: self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) except: pass if password: self.waeup_tool.editPassword(student_id,password) break return student_id,msg,mapping ###) ###) class StudentStudyLevelImport(WAeUPImport):###( """ StudentStudyLevelImport """ name = "student_study_level" plural_name = "%ss" % name commit_after = 1000 required_modes = ('create',) def create(self,mapping): ###( "edit student levels and create StudentStudyLevel object if not existent" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.StudentStudyLevelImport.create') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() key = '' msg = '' while True: result = self.findStudent('create',student_id=student_id,matric_no=matric_no) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = student_record.id mapping['id'] = student_id session = mapping.get('session','') level = key = mapping.get('code','') if not level.isdigit(): msg = 'student %s: imported level is empty' % (student_id,) break study_course_obj = getattr(getattr(students_folder,student_id),'study_course',None) if study_course_obj is None: msg = 'student %s: no study_course object' % student_id break level_obj = getattr(study_course_obj,level,None) if level_obj is None: # The only difference to the edit method is that we create a StudentStudyLevel object try: study_course_obj.invokeFactory('StudentStudyLevel',"%s" % level) level_obj = getattr(context,"%s" % level) except: continue level_obj.getContent().edit(mapping = mapping) level_review_state = wftool.getInfoFor(level_obj,'review_state',None) if level_review_state != "closed": wftool.doActionFor(level_obj,'close') break return key,msg,mapping ###) def edit(self,mapping): ###( "edit student levels" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.StudentStudyLevelImport.edit') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) matric_no = mapping.get('matric_no',None) key = '' msg = '' while True: result = self.findStudent('create',student_id=student_id,matric_no=matric_no) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = student_record.id mapping['id'] = student_id session = mapping.get('session','') level = key = mapping.get('code','') #import pdb;pdb.set_trace() if not level.isdigit(): msg = 'student %s: imported level is empty' % (student_id,) break study_course_obj = getattr(getattr(students_folder,student_id),'study_course',None) if study_course_obj is None: msg = 'student %s: no study_course object' % student_id break level_obj = getattr(study_course_obj,level,None) if level_obj is None: msg = 'student %s: no study_level object for level %s' % (student_id,level) break level_obj.getContent().edit(mapping = mapping) break return key,msg,mapping ###) ###) class VerdictImport(WAeUPImport):###( """ VerdictImport """ name = "verdict" plural_name = "%ss" % name commit_after = 1000 required_modes = ('create','edit') def create(self,mapping): ###( "edit student verdicts and create StudentStudyLevel object if not existent" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.VerdictImport.create') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() key = '' while True: result = self.findStudent('create',student_id=student_id,matric_no=matric_no) #student_record,msg = self.getStudentRecord(mapping) msg = result['msg'] if msg: break student_record = result['student_record'] student_id = student_record.id mapping['id'] = student_id d = {} #import pdb;pdb.set_trace() any_change = False #special treatment for StudentStudyLevel current_session = d['session'] = mapping.get('current_session','') if current_session and student_record.session != current_session: msg = 'student %s: imported session %s does not match current_session %s' % (student_id, current_session, student_record.session) break current_level = mapping.get('current_level','') if not current_level.isdigit(): msg = 'student %s: imported level is empty' % (student_id,) break if current_level and student_record.level != current_level: msg = 'student %s: imported level %s does not match current_level %s' % (student_id, current_level, student_record.level) break student_review_state = student_record.review_state if student_review_state == 'deactivated': msg = "student %s in review_state %s" % (student_id, student_review_state) break if student_review_state not in ('courses_validated','returning'): msg = "student %s in wrong review_state %s" % (student_id, student_review_state) break student_obj = getattr(students_folder,student_id) # f2t = self.field2types_student study_course_obj = getattr(student_obj,'study_course',None) if study_course_obj is None: msg = 'student %s: no study_course object' % student_id break level_obj = getattr(study_course_obj,current_level,None) if level_obj is None: # The only difference to the edit method is that we create a StudentStudyLevel object try: study_course_obj.invokeFactory('StudentStudyLevel',"%s" % current_level) level_obj = getattr(context,"%s" % current_level) level_obj.portal_workflow.doActionFor(level,'open') except: continue #msg = 'student %s: no study_level object for level %s' % (student_id, # current_level) #break verdict = d['verdict'] = d['current_verdict'] = mapping.get('current_verdict','') #if verdict == student_record.verdict: # msg = 'student %s: verdict already set to %s' % (student_id, # verdict) level_review_state = wftool.getInfoFor(level_obj,'review_state',None) if level_review_state != "closed": wftool.doActionFor(level_obj,'close') # msg = 'student %s: level %s is not closed' % (student_id, # current_level) study_course_obj.getContent().edit(mapping = d) level_obj.getContent().edit(mapping = d) if student_review_state != "returning": wftool.doActionFor(student_obj,'return') # try: # wftool.doActionFor(level_obj,'close') # except: # pass break return student_id,msg,mapping ###) def edit(self,mapping): ###( "edit student verdicts" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.VerdictImport.edit') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() key = '' while True: result = self.findStudent('edit',student_id=student_id,matric_no=matric_no) #student_record,msg = self.getStudentRecord(mapping) msg = result['msg'] if msg: break student_record = result['student_record'] key_used = result['key_used'] if key_used in editable_keys: editable_keys.remove(key_used) student_id = student_record.id mapping['id'] = student_id d = {} #import pdb;pdb.set_trace() any_change = False #special treatment for StudentStudyLevel current_session = d['session'] = mapping.get('current_session','') if current_session and student_record.session != current_session: msg = 'student %s: imported session %s does not match current_session %s' % (student_id, current_session, student_record.session) break current_level = mapping.get('current_level','') if not current_level.isdigit(): msg = 'student %s: imported level is empty' % (student_id,) break if current_level and student_record.level != current_level: msg = 'student %s: imported level %s does not match current_level %s' % (student_id, current_level, student_record.level) break student_review_state = student_record.review_state if student_review_state == 'deactivated': msg = "student %s in review_state %s" % (student_id, student_review_state) break if student_review_state not in ('courses_validated','returning'): msg = "student %s in wrong review_state %s" % (student_id, student_review_state) break student_obj = getattr(students_folder,student_id) # f2t = self.field2types_student study_course_obj = getattr(student_obj,'study_course',None) if study_course_obj is None: msg = 'student %s: no study_course object' % student_id break verdict = d['verdict'] = d['current_verdict'] = mapping.get('current_verdict','') level_obj = getattr(study_course_obj,current_level,None) if level_obj is None: pass #msg = 'student %s: no study_level object for level %s' % (student_id,current_level) #break else: level_review_state = wftool.getInfoFor(level_obj,'review_state',None) if level_review_state != "closed": self.waeup_tool.changeWorkflowState(level_obj, 'closed') level_obj.getContent().edit(mapping = d) study_course_obj.getContent().edit(mapping = d) if student_review_state != "returning": self.waeup_tool.changeStudentState(student_obj, 'returning') break return student_id,msg,mapping ###) class PaymentImport(WAeUPImport):###( """ PaymentImport """ name = "payment" plural_name = "%ss" % name commit_after = 2000 required_modes = ('create',) def create(self,mapping): ###( "create payment object" wftool = self.portal_workflow logger = logging.getLogger('WAeUPImport.PaymentImport.create') students_folder = self.portal_url.getPortalObject().campus.students student_id = mapping.get('id',None) order_id = key= mapping.get('order_id',None) payment_type = mapping.get('type',None) editable_keys = mapping.keys() key = '' msg = '' while True: result = self.findStudent('create',student_id=student_id) msg = result['msg'] if msg: break if payment_type == 'sc': hyphen = order_id.rfind('-') if not hyphen > -1: msg = '%s: wrong order_id of sc payment' % student_id break payment_object_id = 'p' + order_id[hyphen+1:] elif payment_type == 'online': payment_object_id = 'p' + order_id[6:] else: msg = '%s: no payment type defined' % student_id break student_record = result['student_record'] student_id = student_record.id mapping['id'] = student_id payments_folder = getattr(getattr(students_folder,student_id),'payments',None) if payments_folder is None: msg = '%s: no payments folder' % student_id break if getattr(payments_folder,payment_object_id,False): msg = '%s: payment object with id %s exists' % (student_id,payment_object_id) break try: #import pdb;pdb.set_trace() payments_folder.invokeFactory('Payment',"%s" % payment_object_id) payment_obj = getattr(payments_folder,"%s" % payment_object_id) except: msg = '%s: payment object %s cannot be created' % (student_id,payment_object_id) break payment_obj.getContent().edit(mapping = mapping) wftool.doActionFor(payment_obj,'close') break return key,msg,mapping ###) def remove(self,mapping): order_id = mapping.get('order_id') student_id = mapping.get('id') msg = '' while True: if not student_id and order_id: msg = 'student_id or order_id wrong' break students_folder = self.portal_url.getPortalObject().campus.students payments_folder = getattr(getattr(students_folder,student_id),'payments',None) payment_object_id = 'p' + order_id[6:] if payments_folder is None: msg = '%s: no payments folder' % student_id break if not getattr(payments_folder,payment_object_id,False): msg = '%s: payment object with id %s does not exist' % (student_id,payment_object_id) break payments_folder.manage_delObjects((payment_object_id),) #if self.payments_catalog.getRecordByKey(order_id) is None: # msg = "no payment record with order_id %(order_id)s" % vars() # break #self.payments_catalog.deleteRecord(order_id) break return order_id,msg,mapping ###)