# -*- mode: python; mode: fold; -*- # (C) Copyright 2005 The WAeUP group # Author: Joachim Schmitz (js@aixtraware.de) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA # 02111-1307, USA. # # $Id: WAeUPTool.py 3764 2008-11-14 07:14:25Z henrik $ """The WAeUP Tool Box. """ from AccessControl import ClassSecurityInfo from Acquisition import aq_inner from Acquisition import aq_parent from Globals import DTMLFile from Globals import InitializeClass from OFS.SimpleItem import SimpleItem from zExceptions import BadRequest from Products.CMFCore.utils import getToolByName from Products.CPSSchemas.DataStructure import DataStructure from Products.CPSSchemas.DataModel import DataModel from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter from Products.CMFCore.ActionProviderBase import ActionProviderBase from Products.CMFCore.permissions import View from Products.ZCatalog.ZCatalog import ZCatalog from Products.CMFCore.permissions import ModifyPortalContent from Products.CMFCore.permissions import ManagePortal from Products.CMFCore.utils import UniqueObject from Products.CMFCore.URLTool import URLTool from Products.CMFCore.utils import getToolByName from Students import makeCertificateCode from Globals import package_home,INSTANCE_HOME from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport from utils import makeDigest import DateTime,time import logging import transaction import csv,re,os,sys import md5 from shutil import copy2,copy from Products.AdvancedQuery import Eq, Between, Le,In p_home = package_home(globals()) i_home = INSTANCE_HOME images_base = os.path.join(i_home,"images") EMPTY = 'XXX' def getImagesDir(student_id): return os.path.join("%s" % images_base,student_id[0],student_id) def getObject(object,name): if object.hasObject(name): return getattr(object,name) return None class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase): """WAeUP tool""" id = 'waeup_tool' meta_type = 'WAeUP Tool' _actions = () security = ClassSecurityInfo() security.declareObjectProtected(View) manage_options = ( ActionProviderBase.manage_options + SimpleItem.manage_options ) security.declareProtected(View,'re_split') ###( def re_split(self,split_string,string): return re.split(split_string,string) ###) security.declareProtected(View,'difference') ###( def difference(self,l1,l2): return set(l1).difference(set(l2)) ###) def rwrite(self,s): ###( response = self.REQUEST.RESPONSE response.setHeader('Content-type','text/html; charset=ISO-8859-15') response.write("%s
\r\n" % s) ###) def addtodict(self,d,key,item): ###( d[key].append(item) return d[key] ###) def sleep(self,secs): ###( "sleep" import time time.sleep(secs) return ###) security.declareProtected(View,'updateRoleMappingsFor') ###( def updateRoleMappingsFor(self,wf_definition,ob): "do so for public" wf_def = getattr(self.portal_workflow,wf_definition) wf_def.updateRoleMappingsFor(ob) ###) security.declareProtected(View,'getStatesLgas') ###( def getStatesLgas(self): """return lga info""" voc = getattr(self.portal_vocabularies,'local_gov_areas') states = [] lgas = [] d = {} wd = {} for k,v in voc.items(): parts = v.split(' / ') if len(parts) == 1: state = parts[0].lower() lga = "" elif len(parts) == 2: state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower())) lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower())) else: continue if state not in states: states.append(state) if lga not in lgas: lgas.append(lga) words = re.split('[^a-zA-Z0-9/]',k) words.sort() wd[k] = words d[k] = v mapping = {} mapping['word_dict'] = wd mapping['lga_dict'] = d mapping['states'] = states mapping['lgas'] = lgas return mapping ###) security.declareProtected(View,'findLga') ###( def findLga(self,words,words_dict): words = re.split('[^a-zA-Z0-9/]',words) lga_words = [] for word in words: if word: lga_words += word.strip().lower(), lga_words.sort() state_lga = '' while not state_lga: for k,l in words_dict.items(): if lga_words == l: state_lga = k break break return state_lga ###) security.declareProtected(View,'getAccessInfo') ###( def getAccessInfo(self,context): "return a dict with access_info" logger = logging.getLogger('WAeUPTool.getAccessInfo') mtool = self.portal_membership member = mtool.getAuthenticatedMember() member_id = str(member) info = {} is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser() is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57 is_staff = info['is_staff'] = not is_anonymous and not is_student roles = member.getRolesInContext(context) info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or "SectionManager" in roles or "Manager" in roles) info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles) is_allowed = info['is_allowed'] = not is_anonymous requested_id = context.getStudentId() student_id = None if is_allowed: if not is_student and requested_id: student_id = requested_id elif not is_allowed and (not is_staff or member_id != requested_id): logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id)) else: student_id = member_id info['student_id'] = student_id return info ###) security.declareProtected(ModifyPortalContent,'openLog') ###( def openLog(self,name): """open a log file""" version = 1 path = "%s/log/%s_%d.log" % (i_home,name,version) while os.path.exists(path): version += 1 path = "%s/log/%s_%d.log" % (i_home,name,version) log = open(path,"w") return log ###) security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###( def bypassQueueCatalog(self,enable=True): """bypass the QueueCatalog by setting all indexes to process imediate, if enable is True (default) the old settings are restored """ ###) security.declareProtected(ModifyPortalContent,'measureOaT') ###( def measureOaT(self,method="a",probe="1000",nr_pts="1"): """measure Object access Time""" import random if hasattr(self,'portal_catalog_real'): aq_portal = self.portal_catalog_real.evalAdvancedQuery else: aq_portal = self.portal_catalog.evalAdvancedQuery nr_pts = int(nr_pts) probe = int(probe) intervall = probe/10 objects = ("application","clearance","personal") portal_types = ("StudentApplication","StudentClearance","StudentPersonal") #i = random.randrange(num_objects) count = 0 found = 0 not_found = 0 t_found = 0 t_not_found = 0 time_found = time_not_found = 0.0 t_time_found = t_time_not_found = 0.0 accessed = [] t_min = 1000 t_max = 0 #import pdb;pdb.set_trace() students = self.portal_catalog(portal_type="Student") num_students = len(students) if method == "d": query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts]) res = aq_portal(query) brains = {} for r in res: sid = r.relative_path.split('/')[-2] if brains.has_key(sid): brains[sid][r.portal_type] = r else: brains[sid] = {r.portal_type : r} brains_list = brains.keys() num_objects = len(brains_list) else: num_objects = num_students print "="*40 print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method, probe, nr_pts, num_objects) print "nr found/not time found/not min/max" elapse = time.time() i_elapse = time.time() c_elapse = time.clock() for c in range(1,probe + 1): i = random.randrange(num_objects) if method in ('a','b','c'): student_brain = students[i] elif method == "d": #import pdb;pdb.set_trace() student_brain = brains[brains_list[i]] if method == "c": query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts]) res = aq_portal(query) this_portal_types = [r.portal_type for r in res] for i in range(nr_pts): oid = objects[i] if method == "a": try: student_path = student_brain.getPath() path = "%s/%s" % (student_path,oid) doc = self.unrestrictedTraverse(path).getContent() found += 1 i_time = time.time() - i_elapse time_found += i_time except: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time pass elif method == "b": try: student_object = student_brain.getObject() doc = getattr(student_object,oid).getContent() found += 1 i_time = time.time() - i_elapse time_found += i_time except: i_time = time.time() - i_elapse time_not_found += i_time not_found += 1 pass elif method == "c": if portal_types[i] in this_portal_types: found += 1 doc = res[this_portal_types.index(portal_types[i])].getObject().getContent() i_time = time.time() - i_elapse time_found += i_time else: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time elif method == "d": if student_brain.has_key(portal_types[i]): found += 1 doc = student_brain[portal_types[i]].getObject().getContent() i_time = time.time() - i_elapse time_found += i_time else: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time i_elapse = time.time() if c and (c % intervall == 0): #i_time = time.time() - i_elapse t_per = 0.0 if found: t_per = time_found/found if t_per > t_max: t_max = t_per if t_per > 0.0 and t_per < t_min: t_min = t_per itf = 0.0 if found: itf = time_found/found itnf = 0.0 if not_found : itnf = time_not_found / not_found interval_time = time_found + time_not_found s = "%(c)d: %(found)d/%(not_found)d " % vars() s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() s += "%(t_min)6.4f/%(t_max)6.4f" % vars() print s t_found += found t_not_found += not_found t_time_found += time_found t_time_not_found += time_not_found time_found = time_not_found = 0.0 found = not_found = 0 # t_found += found # t_not_found += not_found elapse = time.time() - elapse itf = 0.0 if t_found: itf = t_time_found/t_found itnf = 0.0 if t_not_found: itnf = t_time_not_found / t_not_found #c_elapse = time.clock() - c_elapse s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars() s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() s += "%(t_min)6.4f/%(t_max)6.4f" % vars() print "-"*40 print s rel_found = float(t_found)/probe rel_not_found = float(t_not_found)/probe estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf) print estimated_total_time ###) security.declareProtected(ModifyPortalContent,'writeLog') ###( def writeLog(self,logfile,s): """write to the log file""" logfile.write(s) ###) def generateStudentId(self,letter): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if letter == '?': letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') sid = "%c%d" % (letter,r.randint(99999,1000000)) students = self.portal_url.getPortalObject().campus.students # while hasattr(students, sid): # sid = "%c%d" % (letter,r.randint(99999,1000000)) while self.students_catalog(id = sid): sid = "%c%d" % (letter,r.randint(99999,1000000)) return sid ###) def generatePassword(self,s=None): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if s is None: s = 'abcdefghklmnpqrstuvwxy23456789' pw = '' while len(pw) < 6: pw += r.choice(s) return pw ###) security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###( def dumpSchoolfeePayments(self): "dump paid schoolfees" mtool = self.portal_membership member = mtool.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.dumpSchoolfees') aq_student = self.students_catalog.evalAdvancedQuery query = In('review_state',('schoolfee_paid', 'courses_registered', 'courses_validated', )) res = aq_student(query) #import pdb;pdb.set_trace() l = [] logger.info("start for %d" % len(res)) count = 1 log_after = 100 for student in res: if not count % log_after: logger.info("processed %d total %d" % (log_after,count)) count += 1 fee_dict =self.getSchoolFee(student) fulltime = student.mode.endswith('_ft') d = {} d['student_id'] = student.id d['name'] = student.name d['amount'] = fee_dict.get(new_returning) l += d, csv_name = self.dumpListToCSV(l,'payments') logger.info('%s dumped payments to %s' % (member,export_file)) ###) security.declarePublic('dumpListToCSV') ###( def dumpListToCSV(self,l,filename,fields=None): """dump a list of dicts to a CSV file""" current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,) if fields is None: fields = l[0].keys() headline = ','.join(fields) out = open(export_file,"wb") out.write(headline +'\n') out.close() out = open(export_file,"a") csv_writer = csv.DictWriter(out,fields,) csv_writer.writerows(l) return export_file ###) security.declareProtected(ManagePortal, 'listMembers') ###( def listMembers(self): "list all members" mtool = self.portal_membership member = mtool.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.listMembers') if str(member) not in ('admin','joachim'): logger.info('%s tried to list members' % (member)) return None members = self.portal_directories.members all = members.listEntryIdsAndTitles() l = [] for user_id,name in all: d = {} d['user_id'] = user_id d['name'] = name d['pw'] = getattr(getattr(members,user_id),'password') d['email'] = getattr(getattr(members,user_id),'email') d['groups'] = " ".join(getattr(getattr(members,user_id),'groups')) d['roles'] = " ".join(getattr(getattr(members,user_id),'roles')) l += d, current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") export_file = "%s/export/member_list_%s.csv" % (i_home,current,) logger.info('%s dumped member list to %s' % (member,export_file)) fields = l[0].keys() headline = ','.join(fields) out = open(export_file,"wb") out.write(headline +'\n') out.close() out = open(export_file,"a") csv_writer = csv.DictWriter(out,fields,) csv_writer.writerows(l) ###) security.declareProtected(ManagePortal, 'listStudents') ###( def listStudents(self): "list all students" mtool = self.portal_membership member = mtool.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.listStudents') if str(member) not in ('admin','joachim'): logger.info('%s tried to list students' % (member)) return None students = self.portal_directories.students all = students.listEntryIdsAndTitles() l = [] for user_id,name in all: d = {} d['user_id'] = user_id d['name'] = name d['pw'] = getattr(getattr(students,user_id),'password') d['email'] = getattr(getattr(students,user_id),'email') d['groups'] = " ".join(getattr(getattr(students,user_id),'groups')) d['roles'] = " ".join(getattr(getattr(students,user_id),'roles')) l += d, current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") export_file = "%s/export/student_list_%s.csv" % (i_home,current,) logger.info('%s dumped student list to %s' % (member,export_file)) fields = l[0].keys() headline = ','.join(fields) out = open(export_file,"wb") out.write(headline +'\n') out.close() out = open(export_file,"a") csv_writer = csv.DictWriter(out,fields,) csv_writer.writerows(l) ###) security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###( def removeDeletedDocIds(self, max=1000): """ remove deleted docids from repository commit after max """ logger = logging.getLogger('WAeUPTool.removeDeletedDocIds') repository = getToolByName(self, 'portal_repository') pxtool = getToolByName(self, 'portal_proxies') logger.info('start') pxtool_infos = pxtool.getRevisionsUsed() logger.info('found %d used revisions ' % (len(pxtool_infos))) nb_revs = 0 docids_d = {} # all docids unused_docids_d = {} # all docids that are unused ids_unused_revs_docids = [] # ids for revs of unused docids unused_ids = [] # ids for unused revs total = 0 idlist = repository.objectIds() to_delete = 0 found = False for id in idlist: docid, rev = repository._splitId(id) if docid is None: logger.info("invalid doc_id %s" % docid) continue nb_revs += 1 if not pxtool_infos.has_key(docid): found = True to_delete += 1 unused_ids.append(id) elif not pxtool_infos[docid].has_key(rev): found = True to_delete += 1 unused_ids.append(id) if found and not to_delete % max: found = False #import pdb;pdb.set_trace() repository.manage_delObjects(unused_ids) transaction.commit() logger.info('removed %d total %d unused docids ' % (max,to_delete)) else: if unused_ids: repository.manage_delObjects(unused_ids) transaction.commit() logger.info('finished removing %d unused docids ' % (to_delete)) ###) security.declareProtected(View,'getCredential') ###( def getCredential(self,student_id): student_entry = getattr(self.portal_directories.students,student_id,None) if not self.isStaff(): mtool = self.portal_membership member = mtool.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.getCredential') logger.info('%s tried to access password of %s' % (member,student_id)) return None if student_entry is None: return None return getattr(student_entry,"password","not set") ###) security.declarePublic('checkPassword') ###( def checkPassword(self,student_id,password): student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return False return getattr(student_entry,"password","not set") == password ###) security.declarePublic('checkGenericPassword') ###( def checkGenericPassword(self,member_id): member_entry = getattr(self.portal_directories.members,member_id,None) if member_entry is None: return False ltool = getToolByName(self, 'portal_layouts') unsecure_words = ltool._getOb('members')['w__password'].check_words password = getattr(member_entry,"password","not set") is_unsecure = password in unsecure_words if is_unsecure: logger = logging.getLogger('WAeUPTool.checkGenericPassword') logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password)) return is_unsecure ###) security.declareProtected(ModifyPortalContent,'editPassword') ###( def editPassword(self,student_id,password): "edit a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return setattr(student_entry,'password',password) ###) security.declareProtected(ModifyPortalContent,'doCommit') ###( def doCommit(self,logger=None): "commit some transactions" transaction.commit() ###) security.declarePublic('loadStudentFoto') ###( def loadStudentFoto(self,student,filename,folder): "return a student passport picture" #import pdb;pdb.set_trace() picture ="%s/import/%s/%s" % (i_home,folder,filename) student_id = student.getId() images_dir = getImagesDir(student_id) if not os.path.exists(images_dir): os.mkdir(images_dir) image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) for extension in ('.jpg','.JPG'): fullname = "%(picture)s%(extension)s" % vars() if os.path.exists(fullname): copy2(fullname,image_name) return "successfully copied passport picture" return "passport picture not found: %s.jpg or .JPG" % picture ###) def old____loadStudentFoto(self,student,filename,folder): ###( "return a student passport picture" app = student.application app_doc = app.getContent() #clear = student.clearance #clear_doc = clear.getContent() #matric_no = clear_doc.matric_no.upper() picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename) picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename) #import pdb;pdb.set_trace() if os.path.exists(picture1): file = open(picture1) elif os.path.exists(picture2): file = open(picture2) else: return "passport picture not found %s" % picture1 reopened = False if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened': self.portal_workflow.doActionFor(app,'open') reopened = True outfile = file.read() app_doc.manage_addFile('passport', file=outfile, title="%s.jpg" % filename) if reopened: self.portal_workflow.doActionFor(app,'close') return "successfully loaded passport picture" ###) security.declareProtected(ModifyPortalContent,'createOne') ###( def createOne(self,students_folder,student_brain,letter,commit=False): sid = self.waeup_tool.generateStudentId(letter) students_folder.invokeFactory('Student', sid) student = getattr(students_folder,sid) self.portal_workflow.doActionFor(student,'return') student.manage_setLocalRoles(sid, ['Owner',]) matric_no = student_brain.matric_no jamb_reg_no = student_brain.Entryregno self.students_catalog.addRecord(id = sid, matric_no = matric_no, jamb_reg_no = jamb_reg_no, sex = student_brain.Sex == "F", name = "%s %s %s" % (student_brain.Firstname, student_brain.Middlename, student_brain.Lastname) ) if commit: transaction.commit() return sid,jamb_reg_no ###) security.declareProtected(ModifyPortalContent,'addStudent') ###( def addStudent(self,dict): students_folder = self.portal_url.getPortalObject().campus.students sid = self.waeup_tool.generateStudentId('?') students_folder.invokeFactory('Student', sid) student_obj = getattr(students_folder,sid) f2t = StudentImport.field2types_student #from pdb import set_trace; set_trace() d = {} #d['jamb_sex'] = 'M' #if dict.get('sex'): # d['jamb_sex'] = 'F' entry_session = dict.get('entry_session') if entry_session == self.getSessionId()[0]: wfaction = 'admit' wft = 'wf_transition_admit' password = None else: wfaction = 'return' wft = 'wf_transition_return' password = self.generatePassword() self.makeStudentMember(sid,password) for pt in f2t.keys(): student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj) #d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = dict.get(field,'') sub_doc.edit(mapping = d) new_state = f2t[pt][wft] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) self.portal_workflow.doActionFor(student_obj,wfaction) student_obj.manage_setLocalRoles(sid, ['Owner',]) return sid,password ###) security.declarePublic('getCertificateBrain') ###( def getCertificateBrain(self,cert_id): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"Certificate", 'id': cert_id}) if res: return res[0] return None ###) security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###( def get_csv_filenames(self): "do it" files = [file for file in os.listdir("%s/import/" % (i_home)) if file.endswith('.csv') and (file.find('imported') == -1 and file.find('pending') == -1)] return files ###) security.declarePublic('findStudentByMatricelNo') ###( def findStudentByMatricelNo(self,matric_no): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"StudentClearance", 'SearchableText': matric_no}) if res: return res[0] return None ###) security.declarePublic('makeStudentMember') ###( def makeStudentMember(self,sid,password='uNsEt'): """make the student a member""" membership = self.portal_membership membership.addMember(sid, password , roles=('Member', 'Student', ), domains='', properties = {'memberareaCreationFlag': False, 'homeless': True},) member = membership.getMemberById(sid) self.portal_registration.afterAdd(member, sid, password, None) #self.manage_setLocalRoles(sid, ['Owner',]) ###) security.declareProtected(View,'makeStudentData') ###( def makeStudentData(self,student_id,email=None,phone_nr=None): "create Datastructure for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentData') students_folder = self.portal_url.getPortalObject().campus.students #res = self.students_catalog(id=student_id) #if res: # st = res[0] #res = self.returning_import(matric_no = st.matric_no) res = self.returning_import(id = student_id) if res: student = res[0] else: logger.info('Id %s not found in returning_import' % student_id) return logger.info('%s creates data structure' % student_id) s_results = self.results_import(matric_no = student.matric_no) if s_results: lnr = self.getLevelFromResultsCosCode(s_results) level = "%d00" % lnr verdict,eligible = self.getVerdict(s_results[0].Verdict) #if eligible: # level = "%d00" % (lnr + 1) else: logger.info('matric_no %s not found in results_import' % student.matric_no) level = '' verdict = '' #student should not be allowed to perform this transition #wftool = self.portal_workflow #wftool.doActionFor(student,'return') certcode_org = student.Coursemajorcode certcode = makeCertificateCode(certcode_org) certificate_brain = self.getCertificateBrain(certcode) if not certificate_brain: em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org) logger.info(em) matric_no = student.matric_no sid = student_id student_obj = getattr(students_folder,sid) if not getattr(student_obj,'application'): student_obj.invokeFactory('StudentApplication','application') application = student_obj.application self.portal_workflow.doActionFor(application,'open',dest_container=application) da = {'Title': 'Application Data'} student_obj.invokeFactory('StudentPersonal','personal') da['jamb_reg_no'] = student.Entryregno em = self.getEntryMode(student.Entryregno) da['entry_mode'] = em personal = student_obj.personal self.portal_workflow.doActionFor(personal,'open',dest_container=personal) dp = {'Title': 'Personal Data'} student_obj.invokeFactory('StudentClearance','clearance') clearance = student_obj.clearance self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) dc = {'Title': 'Clearance/Eligibility Record'} dc['matric_no'] = matric_no state = student.State lga = student.LGA if state and lga: lga = state + ' / ' + lga else: lga = "None" da['jamb_lga'] = dc['lga'] = lga da['app_email'] = dp['email'] = email da['app_mobile'] = dp['phone'] = phone_nr dp['firstname'] = student.Firstname dp['middlename'] = student.Middlename dp['lastname'] = student.Lastname da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname) da['jamb_sex'] = student.Sex dp['sex'] = student.Sex == 'F' dp['perm_address'] = student.Permanent_Address application.getContent().edit(mapping=da) self.portal_workflow.doActionFor(application,'close',dest_container=application) personal.getContent().edit(mapping=dp) clearance.getContent().edit(mapping=dc) self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) # # Study Course # student_obj.invokeFactory('StudentStudyCourse','study_course') studycourse = student_obj.study_course self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) dsc = {} dsc['study_course'] = certcode dsc['current_level'] = level dsc['current_verdict'] = verdict dsc['current_mode'] = em dsc['current_session'] = '05' studycourse.getContent().edit(mapping=dsc) # # Level # # l = getattr(studycourse,level,None) # if l is None: # studycourse.invokeFactory('StudentStudyLevel', level) # l = getattr(studycourse, level) # self.portal_workflow.doActionFor(l,'open',dest_container=l) # l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) def init_timing(self): ###( if self.with_timing: if not hasattr(self,'_v_step_times'): self._v_timer_count = 0 self._v_total = 0 self._v_step_times = {} current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,) self.timer_step = 0 self.total_time = 0 self.elapse = time.time() self.i_elapse = time.time() self.c_elapse = time.clock() ###) def do_timing(self): ###( if self.with_timing: try: raise 'dummy' except: frame = sys.exc_traceback.tb_frame.f_back locals = frame.f_locals globals = frame.f_globals functionname = frame.f_code.co_name filename = os.path.basename(frame.f_code.co_filename) lineno = frame.f_lineno mod_line = "%(functionname)s:%(lineno)s" % vars() i_time = time.time() - self.i_elapse td = {} if self._v_step_times.has_key(mod_line): a_time = self._v_step_times[mod_line]['a_time'] + i_time td['a_time'] = a_time else: td['a_time'] = i_time td['i_time'] = i_time self._v_step_times[mod_line] = td self.i_time = i_time self.total_time += i_time self.timer_step +=1 self.i_elapse = time.time() ###) security.declareProtected(ModifyPortalContent,'print_timing') ###( ###( def print_timing(self): if self.with_timing: l = [] timer_count = self._v_timer_count + 1 mod_lines = self._v_step_times.keys() mod_lines.sort(cmp,reverse=0) for mod_line in mod_lines: td = self._v_step_times[mod_line] i_time = td['i_time'] a_time = td['a_time']/(self._v_timer_count + 1) l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()), total_time = self.total_time total_avarage = self._v_total / timer_count l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()), print "\r\n".join(l) out = open(self._v_timer_file,'a') out.write("\r\n".join(l)) out.close() ###) security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###( def get_timing_data(self): if self.with_timing: timer_count = self._v_timer_count + 1 results = {} for k,d in self._v_step_times.items(): dd = {} dd['a_time'] = d['a_time']/timer_count dd['i_time'] = d['i_time'] dd['count'] = timer_count results[k] = dd dd = {} dd['a_time'] = self._v_total / timer_count dd['i_time'] = self.total_time dd['count'] = timer_count results["total"] = dd return results ###) security.declareProtected(ModifyPortalContent,'admitOneStudent') ###( def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False): "create Datastructure for an admitted Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.admitOneStudent') self.with_timing = with_timing if brain.screening_type in ('cest','sandwich',): reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session) else: reg_no = brain.reg_no #ignore argument entry_session if not brain.entry_session: logger.info('no entry_session for %s provided' % (reg_no)) return if not hasattr(self,"_v_certificates"): self._v_certificates = self.getCertificatesDict() students_folder = self.portal_url.getPortalObject().campus.students res = self.students_catalog(jamb_reg_no = reg_no) if res: logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id)) return if brain.status != "admitted": logger.info('status of %s is %s' % (reg_no,brain.status)) return pin_parts = brain.pin.split('-') if pin_parts and len(pin_parts) != 3: logger.info('invalid pin %s for %s' % (brain.pin,reg_no)) return if not brain.course_admitted: logger.info('no course_admitted provided for %s' % (reg_no)) return if brain.course_admitted not in self._v_certificates: logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no)) return if brain.sex not in (True,False): logger.info('sex of %s not available' % (reg_no)) return self.init_timing() student_id = self.generateStudentId('?') students_folder.invokeFactory('Student', student_id) student_object = getattr(students_folder,student_id) self.do_timing() if pin_password: password = pin_parts[2] self.makeStudentMember(student_id,password = password) student_object.manage_setLocalRoles(student_id, ['Owner',]) self.do_timing() #logger.info("creating %s reg_no %s" % (student_id,reg_no)) # # application # student_object.invokeFactory('StudentApplication','application') application = student_object.application #self.portal_workflow.doActionFor(application,'open',dest_container=application) #self.do_timing() da = {'Title': 'Application Data'} da['jamb_reg_no'] = reg_no sex = 'M' if brain.sex: sex = 'F' da['jamb_sex'] = sex da['jamb_age'] = brain.jamb_age da['app_reg_pin'] = brain.pin da['jamb_lga'] = brain.jamb_lga da['jamb_state'] = brain.jamb_state da['jamb_score'] = brain.aggregate da['app_email'] = brain.email da['app_mobile'] = brain.phone da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode'] #da['entry_session'] = entry_session da['entry_session'] = brain.entry_session da['jamb_lastname'] = brain.lastname da['jamb_middlename'] = brain.middlenames # different field names! da['jamb_firstname'] = brain.firstname da['screening_application_date'] = brain.application_date da['date_of_birth'] = brain.date_of_birth da['jamb_first_cos'] = brain.course1 da['jamb_second_cos'] = brain.course2 da['course3'] = brain.course3 da['screening_type'] = brain.screening_type da['screening_score'] = brain.screening_score da['screening_date'] = brain.screening_date da['hq_type'] = brain.hq_type da['hq_grade'] = brain.hq_grade da['aos'] = brain.aos application.getContent().edit(mapping=da) self.do_timing() #self.portal_workflow.doActionFor(application,'close',dest_container=application) # # personal # student_object.invokeFactory('StudentPersonal','personal') personal = student_object.personal #self.portal_workflow.doActionFor(personal,'open',dest_container=personal) #self.do_timing() dp = {'Title': 'Personal Data'} dp['sex'] = brain.sex dp['email'] = brain.email dp['phone'] = brain.phone dp['lastname'] = brain.lastname dp['middlename'] = brain.middlenames # different field names! dp['firstname'] = brain.firstname personal.getContent().edit(mapping=dp) self.do_timing() # # clearance # student_object.invokeFactory('StudentClearance','clearance') clearance = student_object.clearance #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) dc = {'Title': 'Clearance/Eligibility Record'} dc['lga'] = brain.lga dc['birthday'] = brain.date_of_birth clearance.getContent().edit(mapping=dc) self.do_timing() #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) # # study Course # student_object.invokeFactory('StudentStudyCourse','study_course') studycourse = student_object.study_course #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) #self.do_timing() dsc = {} dsc['study_course'] = brain.course_admitted dsc['current_verdict'] = '' dsc['current_mode'] = da['entry_mode'] if da['entry_mode'].startswith('de'): dsc['current_level'] = '200' elif da['entry_mode'].startswith('pre'): dsc['current_level'] = '000' else: dsc['current_level'] = '100' dsc['current_session'] = entry_session studycourse.getContent().edit(mapping=dsc) self.do_timing() # # payments folder student_object.invokeFactory('PaymentsFolder','payments') payments = getattr(student_object,'payments') #self.do_timing() dpay = {} dpay['Title'] = 'Payments' payments.getContent().edit(mapping=dpay) self.portal_workflow.doActionFor(payments,'open') self.do_timing() # # passport foto app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home, brain.screening_type, brain.reg_no) images_dir = getImagesDir(student_id) #images_dir = os.path.join("%s" % images_base,student_id) letter_dir,student_dir = os.path.split(images_dir) if not os.path.exists(letter_dir): os.mkdir(letter_dir) if not os.path.exists(images_dir): os.mkdir(images_dir) image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) if os.path.exists(app_picture): copy2(app_picture,image_name) else: logger.info('passport of %s/%s not found: %s' % (student_id, brain.reg_no, app_picture)) self.do_timing() self.print_timing() if with_timing: self.timer_step = 0 self._v_timer_count += 1 self._v_total += self.total_time return student_id ###) security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###( def makeStudentLevel(self,student_id): "create the StudyLevel for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentLevel') students_folder = self.portal_url.getPortalObject().campus.students res = self.students_catalog(id=student_id) if res: st = res[0] course = st.course matric_no = st.matric_no level = st.level res = self.results_import(matric_no = matric_no) if res: results = res logger.info('%s creating Level %s' % (student_id,level)) # # Level # student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) studycourse = getattr(student_obj,"study_course",None) self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse) l = getattr(studycourse,level,None) if l is None: studycourse.invokeFactory('StudentStudyLevel', level) l = getattr(studycourse, level) self.portal_workflow.doActionFor(l,'open',dest_container=l) l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) security.declarePublic('getHallInfo') ###( def getHallInfo(self,bed): """return Hall Info""" info = {} bedsplit = bed.split('_') if len(bedsplit) == 4: hall,block,room,letter = bed.split('_') else: info['maintenance_code'] = 'None' return info res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall) if res and len(res) == 1: hall_brain = res[0] hall_doc = hall_brain.getObject().getContent() else: return info info['hall_title'] = hall_brain.Title info['maintenance_code'] = hall_doc.maintenance_code res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch") batch_doc = None for brain in res: if brain.id.startswith(info['maintenance_code']): batch_doc = brain.getObject().getContent() break if batch_doc is None: info['maintenance_fee'] = '' else: info['maintenance_fee'] = batch_doc.cost return info ###) security.declareProtected(ModifyPortalContent,'removePictureFolder') ###( def removePictureFolder(self,student_id): """remove picture_folder by renaming it""" path = 'images' picture_path = os.path.join(i_home,path,student_id) if not os.path.exists(picture_path): return False os.rename(picture_path,picture_path + "_removed") return True ###) security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###( def restorePictureFolder(self,student_id): """restore picture_folder by renaming it""" path = 'images' picture_path = os.path.join(i_home,path,student_id) if not os.path.exists(picture_path + "_removed"): return False os.rename(picture_path + "_removed",picture_path) return True ###) security.declarePublic('picturesExist') ###( def picturesExist(self, ids,student_id=None): """check if pictures exist in the filesystem""" if student_id is None: student_id = self.getStudentId() if student_id is None: return False picture_path = getImagesDir(student_id) #picture_path = os.path.join(images_base,student_id) if not os.path.exists(picture_path): return False pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] return set(ids).issubset(set(pictures)) ###) security.declarePublic('picturesList') ###( def picturesList(self): """check if pictures exist in the filesystem""" path = 'images' student_id = self.getStudentId() #picture_path = os.path.join(i_home,path,student_id) picture_path = getImagesDir(student_id) if not os.path.exists(picture_path): return [] return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] ###) security.declarePublic('showFsPicture') ###( def showFsPicture(self,path): """return a picture from the filesystem""" #picture_path = os.path.join(i_home,path) picture_path = os.path.join(images_base,path) response = self.REQUEST.RESPONSE #import pdb;pdb.set_trace() registry = getToolByName(self, 'mimetypes_registry') mimetype = str(registry.lookupExtension(path.lower()) or registry.lookupExtension('file.bin')) if os.path.exists(picture_path): response.setHeader('Content-type',mimetype) return open(picture_path).read() picture_path = os.path.join(i_home,'import',path) if os.path.exists(picture_path): return open(picture_path).read() ###) security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###( def deleteAllCourses(self,department="All"): ''' delete the courses''' pm = self.portal_membership member = pm.getAuthenticatedMember() if str(member) not in ("henrik","joachim"): return "not possible" if department == "All": res = self.portal_catalog({'meta_type': 'Department'}) if len(res) < 1: return "No Departments found" deleted = [] for dep in res: cf = dep.getObject().courses if cf: cf.manage_delObjects(ids=cf.objectIds()) deleted.append("deleted Courses in %s" % dep.getId) return "\r".join(deleted) ###) security.declareProtected(ModifyPortalContent,'getLogfileLines') ###( def getLogfileLines(self,filename="event.log",numlines=20): """Get last NUMLINES lines of logfile FILENAME. Return last lines' of a file in the instances logfile directory as a list. The number of returned lines equals `numlines' or less. If less than `numlines' lines are available, the whole file ist returned. If the file can not be opened or some other error occurs, empty list is returend. """ result = [] lines_hit = 0 # We only handle files in instances' log directory... logpath = os.path.join(i_home, "log") filename = str(os.path.abspath( os.path.join( logpath, filename ))) if not filename.startswith( logpath ): # Attempt to access file outside log-dir... return [] try: fd = file( filename, "rb" ) except IOError: return [] if not fd: return [] if os.linesep == None: linesep = '\n' else: linesep = os.linesep # Try to find 'numlines' times a lineseparator, searching from end # and moving to the beginning of file... fd.seek( 0, 2) # Move to end of file... while lines_hit < numlines: if fd.read(1) == linesep[-1]: # This moves filedescriptor # one step forward... lines_hit += 1 try: fd.seek( -2, 1) # Go two bytes back from current pos... except IOError: # We cannot go back two bytes. Maybe the file is too small... break fd.seek(2,1) # Read all lines from current position... result = fd.readlines() # Remove line endings... result = [x.strip() for x in result] fd.close() return result ###) security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###( def getCallbacksFromLog(self,filename): """fix Online Payment Transactions from Z2.log entries""" import transaction import random from cgi import parse_qs from urlparse import urlparse #from pdb import set_trace wftool = self.portal_workflow current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") students_folder = self.portal_url.getPortalObject().campus.students s = r'(?P\S+) - (?P\S+) \[' s += r'(?P.*)\] "(?P.*)" (?P\d+ \d+) "' s += r'(?P.*)" "(?P.*)"' data = re.compile(s) start = True tr_count = 1 total = 0 #name = 'pume_results' #name = 'epaymentsuccessful_z2log2' name = filename no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.getFailedTransactions') try: transactions = open("%s/import/%s" % (i_home,name),"rb").readlines() except: logger.error('Error reading %s' % name) return tas = [] for line in transactions: dict = {} items = data.search(line) dict['idict'] = idict = items.groupdict() #print idict #from pdb import set_trace;set_trace() urlparsed = urlparse(idict['get'][4:]) #print urlparsed path = urlparsed[2].split('/') dict['student_id'] = student_id = path[8] dict['payment_id'] = payment_id = path[10] dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4]) tas.append(dict) tr_count += 1 return tas ###) security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###( def importOnlinePaymentTransactions(self): """load Online Payment Transactions from CSV values""" import transaction import random #from pdb import set_trace current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") opt = self.online_payments_import students_folder = self.portal_url.getPortalObject().campus.students start = True tr_count = 1 total = 0 #name = 'pume_results' name = 'OnlineTransactions' no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions') try: transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) except: logger.error('Error reading %s.csv' % name) return for pay_transaction in transactions: if start: start = False logger.info('Start loading from %s.csv' % name) s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()]) no_import.append('%s,"Error"' % s) format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()]) format_error = format + ',"%(Error)s"' data = {} # format of the first file sent by Tayo #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date']) #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = pay_transaction['Resp Code'] #data['amount'] = amount = pay_transaction['Amount'] # format of the second file sent by Tayo #data['datetime'] = date = 0 #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = '00' #data['amount'] = amount = pay_transaction['Amount'] # format of the third file sent by Kehinde data['datetime'] = date = 0 data['student_id'] = student_id = pay_transaction['customer_id'] data['order_id'] = order_id = pay_transaction['merchant_reference'] data['response_code'] = response_code = '00' data['amount'] = amount = pay_transaction['Amount'] dup = False if response_code == "12": continue try: opt.addRecord(**data) except ValueError: dup = True #from pdb import set_trace;set_trace() if dup: if response_code == "00": try: opt.modifyRecord(**data) except: logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data) continue else: pay_transaction['Error'] = "Duplicate order_id" no_import.append( format_error % pay_transaction) logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data) continue tr_count += 1 if tr_count > 1000: if len(no_import) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import) + '\n') no_import = [] em = '%d transactions committed\n' % (tr_count) transaction.commit() regs = [] logger.info(em) total += tr_count tr_count = 0 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import)) return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) ###) security.declareProtected(ModifyPortalContent,"importData")###( def importData(self,filename,name,edit=False,bypass_queue_catalog=False): """load data from CSV values""" import transaction import random students_folder = self.portal_url.getPortalObject().campus.students uploads_folder = self.portal_url.getPortalObject().campus.uploads pending_only = False pend_str = '--' elapse = time.time() # # preparations # if filename == pend_str: pending_only = True importer_name = ''.join([part.capitalize() for part in name.split('_')]) importer = eval("%sImport" % importer_name)(self) logger = importer.logger if importer.init_errors: logger.info(importer.init_errors) return importer.init_errors member = importer.member #current = importer.current import_date = importer.import_date # # not_imported # info = importer.info data_keys = importer.data_keys csv_keys = importer.csv_keys #csv_keys.extend(info.keys()) headline_mapping = dict((k,k) for k in csv_keys) # # pending # pending_path = importer.pending_path pending_tmp = importer.pending_tmp pending_backup = importer.pending_backup pending_fn = importer.pending_fn imported_path = importer.imported_path imported_fn = importer.imported_fn commit_after = importer.commit_after pending = [] pending_digests = [] #total_added_to_pending = 0 if not pending_only: pending,pending_digests = importer.makeIdLists() pending_at_start = len(pending) datafile = open(pending_tmp,"w") pending_csv_writer = csv.DictWriter(datafile, csv_keys, extrasaction='ignore') pending_csv_writer.writerow(headline_mapping) datafile.close() # # imported # if not os.path.exists(imported_path): datafile = open(imported_path,"w") imported_csv_writer = csv.DictWriter(datafile, csv_keys, extrasaction='ignore') imported_csv_writer.writerow(headline_mapping) datafile.close() start = True tr_count = 0 total = 0 total_added_to_imported = 0 total_pending = 0 import_source_done = "" if pending_only: import_source_path = pending_path else: import_source_path = "%s/import/%s.csv" % (i_home,filename) import_source_done = "%s/import/%s.done" % (i_home,filename) if not os.path.exists(import_source_path): fn = os.path.split(import_source_path)[1] em = 'no import file %(fn)s' % vars() return em import_source_fn = os.path.split(import_source_path)[1] if not pending_only: info['imported_from'] = import_source_fn headline = csv.reader(open(import_source_path,"rb")).next() if "import_mode" not in headline: msg = 'import_mode must be in heading' return msg invalid_keys = importer.checkHeadline(headline) if invalid_keys: return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys) import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore') or k.strip() in info.keys())] # diff2schema = set(import_keys).difference(set(importer.schema.keys())) # diff2layout = set(import_keys).difference(set(importer.layout.keys())) # if diff2schema and diff2schema != set(['id',]): # msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) # return msg # # start importing # try: reader = csv.DictReader(open(import_source_path,"rb")) except: msg = 'Error reading %s.csv' % filename logger.error(msg) return msg items = [item for item in reader] total_to_import = len(items) tti_float = float(total_to_import) if pending_only: pending_at_start = total_to_import count = 0 imported = [] old_commit_count = 0 error_count = imported_count = 0 already_in = 0 for record in items: item = {} empty_value_keys = [] for k,v in record.items(): if k is None: continue if v: if v == EMPTY: empty_value_keys += k, v = '' item[k.strip()] = v.strip() count += 1 if start: start = False adapters = [MappingStorageAdapter(importer.schema, item)] logger.info('%(member)s starts import from %(import_source_fn)s' % vars()) dm = DataModel(item, adapters,context=self) ds = DataStructure(data=item,datamodel=dm) error_string = "" total += 1 import_mode = item.get('import_mode','') import_method = getattr(importer, '%(import_mode)s' % vars(),None ) if import_method is None: error_string += "import_mode '%(import_mode)s' is invalid" % vars() elif (import_mode in importer.required_modes and not set(importer.required_keys[import_mode]).issubset(set(item.keys()))): diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys())) error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import) else: for k in import_keys: if k not in item.keys() or k not in importer.validators.keys(): continue if not importer.validators[k](ds,mode=import_mode): error_string += ' ++ ' error_string += "%s: %s" % (k,self.translation_service(ds.getError(k), ds.getErrorMapping(k))) if error_string: error = error_string id = '' mapping = item else: temp_item = item.copy() temp_item.update(dm) results = import_method(temp_item) id = results[0] error = results[1] mapping = results[2] # # restore some values # for k in empty_value_keys: mapping[k] = EMPTY if mapping.has_key('sex'): #import pdb;pdb.set_trace() if mapping['sex'] == True: mapping['sex'] = 'F' elif mapping['sex'] == False: mapping['sex'] = 'M' if pending_only: info['imported_from'] = item['imported_from'] data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items()) info['error'] = error info['import_record_no'] = count + 1 mapping.update(info) log_list = [] if error: error_count += 1 digest = makeDigest(mapping,data_keys) if digest not in pending_digests: pending_digests += digest, pending.append(mapping) if not pending_only: log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(), else: already_in += 1 pass else: imported_count += 1 imported += mapping, log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(), if log_list: time_till_now = time.time() - elapse percent_finished = (error_count + imported_count)/tti_float*100 log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),) logger.info(' '.join(log_list)) finished = count > total_to_import - 1 must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after) if must_commit: old_commit_count = imported_count if must_commit or finished: if len(imported): transaction.commit() datafile = open(imported_path,"a") writer = csv.DictWriter(datafile, csv_keys, extrasaction='ignore') writer.writerows(imported) datafile.close() total_added_to_imported += len(imported) imported = [] if len(pending) > 0: datafile = open(pending_tmp,"a") writer = csv.DictWriter(datafile, csv_keys, extrasaction='ignore') writer.writerows(pending) datafile.close() total_pending += len(pending) #total_added_to_pending += len(pending) pending = [] if not finished: msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars() logger.info(msg) elapse = time.time() - elapse if os.path.exists(pending_path): copy2(pending_path,pending_backup) copy2(pending_tmp,pending_path) msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars() msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars() if pending_only: removed_pending = pending_at_start - total_pending msg += "%(removed_pending)d removed from %(pending_fn)s" % vars() else: added_pending = total_pending - pending_at_start msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars() #msg += "%(total_pending)d totally written" % vars() # this line does not make any sense logger.info(msg) if import_source_done: copy(import_source_path,import_source_done) os.remove(import_source_path) upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None) if upload is not None: upload_doc = upload.getContent() mapping = {} #mapping['import_date'] = DateTime.DateTime() mapping['import_date'] = import_date mapping['imported_by'] = importer.imported_by mapping['import_message'] = msg upload_doc.edit(mapping = mapping) os.remove(pending_tmp) return msg ###) security.declareProtected(ModifyPortalContent,"moveImagesToFS")###( def moveImagesToFS(self,student_id="O738726"): "move the images to the filesystem" images_dir = getImagesDir(student_id) #images_dir = os.path.join("%s" % images_base,student_id) student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) stool = getToolByName(self, 'portal_schemas') schemas = ['student_application', 'student_clearance', ] created = False for schema_id in schemas: schema = stool._getOb(schema_id) object = getattr(student_folder,schema_id[len('student_'):],None) if object is None: continue doc = object.getContent() for key in schema.keys(): if schema[key].meta_type != "CPS Image Field": continue #import pdb;pdb.set_trace() image = getattr(doc,key,None) if not image or not hasattr(image,"data"): continue if not created: if not os.path.exists(images_dir): os.mkdir(images_dir) created = True filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars()) open(filename,"wb").write(str(image.data)) ###) security.declareProtected(ModifyPortalContent,"movePassportToFS")###( def movePassportToFS(self,student_id="O738726"): "move the passports to the filesystem" images_dir = os.path.join("%s" % i_home,'passports') student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) stool = getToolByName(self, 'portal_schemas') schemas = ['student_application', #'student_clearance', ] created = False for schema_id in schemas: schema = stool._getOb(schema_id) object = getattr(student_folder,schema_id[len('student_'):],None) if object is None: continue doc = object.getContent() for key in schema.keys(): if schema[key].meta_type != "CPS Image Field": continue #import pdb;pdb.set_trace() image = getattr(doc,key) if not hasattr(image,"data"): continue if not created: if not os.path.exists(images_dir): os.mkdir(images_dir) created = True filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars()) open(filename,"wb").write(str(image.data)) ###) InitializeClass(WAeUPTool)