#-*- mode: python; mode: fold -*- # (C) Copyright 2005 The WAeUP group # Author: Joachim Schmitz (js@aixtraware.de) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA # 02111-1307, USA. # # $Id: WAeUPTool.py 2580 2007-11-08 10:31:00Z joachim $ """The WAeUP Tool Box. """ from AccessControl import ClassSecurityInfo from Acquisition import aq_inner from Acquisition import aq_parent from Globals import DTMLFile from Globals import InitializeClass from OFS.SimpleItem import SimpleItem from zExceptions import BadRequest from Products.CMFCore.utils import getToolByName from Products.CPSSchemas.DataStructure import DataStructure from Products.CPSSchemas.DataModel import DataModel from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter from Products.CMFCore.ActionProviderBase import ActionProviderBase from Products.CMFCore.permissions import View from Products.ZCatalog.ZCatalog import ZCatalog from Products.CMFCore.permissions import ModifyPortalContent from Products.CMFCore.permissions import ManagePortal from Products.CMFCore.utils import UniqueObject from Products.CMFCore.URLTool import URLTool from Products.CMFCore.utils import getToolByName from Students import makeCertificateCode from Globals import package_home,INSTANCE_HOME import DateTime,time import logging import transaction import csv,re,os,sys from shutil import copy2 from Products.AdvancedQuery import Eq, Between, Le,In p_home = package_home(globals()) i_home = INSTANCE_HOME images_base = os.path.join(i_home,"images") def getObject(object,name): if object.hasObject(name): return getattr(object,name) return None class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase): """WAeUP tool""" id = 'waeup_tool' meta_type = 'WAeUP Tool' _actions = () security = ClassSecurityInfo() security.declareObjectProtected(View) manage_options = ( ActionProviderBase.manage_options + SimpleItem.manage_options ) def rwrite(self,s): ###( response = self.REQUEST.RESPONSE response.setHeader('Content-type','text/html; charset=ISO-8859-15') response.write("%s
\r\n" % s) ###) def sleep(self,secs): ###( "sleep" import time time.sleep(secs) return ###) security.declareProtected(ModifyPortalContent,'openLog') ###( def openLog(self,name): """open a log file""" version = 1 path = "%s/log/%s_%d.log" % (i_home,name,version) while os.path.exists(path): version += 1 path = "%s/log/%s_%d.log" % (i_home,name,version) log = open(path,"w") return log ###) security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###( def bypassQueueCatalog(self,enable=True): """bypass the QueueCatalog by setting all indexes to process imediate, if enable is True (default) the old settings are restored """ ###) security.declareProtected(ModifyPortalContent,'measureOaT') ###( def measureOaT(self,method="a",probe="1000",nr_pts="1"): """measure Object access Time""" import random if hasattr(self,'portal_catalog_real'): aq_portal = self.portal_catalog_real.evalAdvancedQuery else: aq_portal = self.portal_catalog.evalAdvancedQuery nr_pts = int(nr_pts) probe = int(probe) intervall = probe/10 objects = ("application","clearance","personal") portal_types = ("StudentApplication","StudentClearance","StudentPersonal") #i = random.randrange(num_objects) count = 0 found = 0 not_found = 0 t_found = 0 t_not_found = 0 time_found = time_not_found = 0.0 t_time_found = t_time_not_found = 0.0 accessed = [] t_min = 1000 t_max = 0 #import pdb;pdb.set_trace() students = self.portal_catalog(portal_type="Student") num_students = len(students) if method == "d": query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts]) res = aq_portal(query) brains = {} for r in res: sid = r.relative_path.split('/')[-2] if brains.has_key(sid): brains[sid][r.portal_type] = r else: brains[sid] = {r.portal_type : r} brains_list = brains.keys() num_objects = len(brains_list) else: num_objects = num_students print "="*40 print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method, probe, nr_pts, num_objects) print "nr found/not time found/not min/max" elapse = time.time() i_elapse = time.time() c_elapse = time.clock() for c in range(1,probe + 1): i = random.randrange(num_objects) if method in ('a','b','c'): student_brain = students[i] elif method == "d": #import pdb;pdb.set_trace() student_brain = brains[brains_list[i]] if method == "c": query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts]) res = aq_portal(query) this_portal_types = [r.portal_type for r in res] for i in range(nr_pts): oid = objects[i] if method == "a": try: student_path = student_brain.getPath() path = "%s/%s" % (student_path,oid) doc = self.unrestrictedTraverse(path).getContent() found += 1 i_time = time.time() - i_elapse time_found += i_time except: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time pass elif method == "b": try: student_object = student_brain.getObject() doc = getattr(student_object,oid).getContent() found += 1 i_time = time.time() - i_elapse time_found += i_time except: i_time = time.time() - i_elapse time_not_found += i_time not_found += 1 pass elif method == "c": if portal_types[i] in this_portal_types: found += 1 doc = res[this_portal_types.index(portal_types[i])].getObject().getContent() i_time = time.time() - i_elapse time_found += i_time else: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time elif method == "d": if student_brain.has_key(portal_types[i]): found += 1 doc = student_brain[portal_types[i]].getObject().getContent() i_time = time.time() - i_elapse time_found += i_time else: not_found += 1 i_time = time.time() - i_elapse time_not_found += i_time i_elapse = time.time() if c and (c % intervall == 0): #i_time = time.time() - i_elapse t_per = 0.0 if found: t_per = time_found/found if t_per > t_max: t_max = t_per if t_per > 0.0 and t_per < t_min: t_min = t_per itf = 0.0 if found: itf = time_found/found itnf = 0.0 if not_found : itnf = time_not_found / not_found interval_time = time_found + time_not_found s = "%(c)d: %(found)d/%(not_found)d " % vars() s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() s += "%(t_min)6.4f/%(t_max)6.4f" % vars() print s t_found += found t_not_found += not_found t_time_found += time_found t_time_not_found += time_not_found time_found = time_not_found = 0.0 found = not_found = 0 # t_found += found # t_not_found += not_found elapse = time.time() - elapse itf = 0.0 if t_found: itf = t_time_found/t_found itnf = 0.0 if t_not_found: itnf = t_time_not_found / t_not_found #c_elapse = time.clock() - c_elapse s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars() s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() s += "%(t_min)6.4f/%(t_max)6.4f" % vars() print "-"*40 print s rel_found = float(t_found)/probe rel_not_found = float(t_not_found)/probe estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf) print estimated_total_time ###) security.declareProtected(ModifyPortalContent,'writeLog') ###( def writeLog(self,logfile,s): """write to the log file""" logfile.write(s) ###) def generateStudentId(self,letter): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if letter == '?': letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') sid = "%c%d" % (letter,r.randint(99999,1000000)) students = self.portal_url.getPortalObject().campus.students ## while hasattr(students, sid): ## sid = "%c%d" % (letter,r.randint(99999,1000000)) while self.students_catalog(id = sid): sid = "%c%d" % (letter,r.randint(99999,1000000)) return sid ###) def generatePassword(self,s=None): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if s is None: s = 'abcdefghklmnpqrstuvwxy23456789' pw = '' while len(pw) < 6: pw += r.choice(s) return pw ###) security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###( def removeDeletedDocIds(self, max=1000): """ remove deleted docids from repository commit after max """ logger = logging.getLogger('WAeUPTool.removeDeletedDocIds') repository = getToolByName(self, 'portal_repository') pxtool = getToolByName(self, 'portal_proxies') pxtool_infos = pxtool.getRevisionsUsed() nb_revs = 0 docids_d = {} # all docids unused_docids_d = {} # all docids that are unused ids_unused_revs_docids = [] # ids for revs of unused docids ids_unused_revs = [] # ids for unused revs total = 0 idlist = repository.objectIds() for id in idlist: docid, rev = repository._splitId(id) if docid is None: logger.info("invalid doc_id %s" % docid) continue nb_revs += 1 docids_d[docid] = None if not pxtool_infos.has_key(docid): unused_docids_d[docid] = None ids_unused_revs_docids.append(id) ids_unused_revs.append(id) elif not pxtool_infos[docid].has_key(rev): ids_unused_revs.append(id) if len(ids_unused_revs) >= max: repository.manage_delObjects(ids_unused_revs) #import pdb;pdb.set_trace() transaction.commit() total += max logger.info('removed %d total %d unused docids ' % (max,total)) anz = len(ids_unused_revs) if anz > 0: repository.manage_delObjects(ids_unused_revs) transaction.commit() total += anz logger.info('finished removing %d unused docids ' % (total)) ###) security.declareProtected(View,'getCredential') ###( def getCredential(self,student_id): "return a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if not self.isStaff(): mtool = self.portal_membership member = mtool.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.getCredential') logger.info('%s tried to access password of %s' % (member,student_id)) return None if student_entry is None: return None return getattr(student_entry,"password","not set") ###) security.declarePublic('checkPassword') ###( def checkPassword(self,student_id,password): "return a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return False return getattr(student_entry,"password","not set") == password ###) security.declareProtected(ModifyPortalContent,'editPassword') ###( def editPassword(self,student_id,password): "edit a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return setattr(student_entry,'password',password) ###) security.declareProtected(ModifyPortalContent,'doCommit') ###( def doCommit(self,logger=None): "commit some transactions" transaction.commit() ###) security.declarePublic('loadStudentFoto') ###( def loadStudentFoto(self,student,filename,folder): "return a student passport picture" #import pdb;pdb.set_trace() picture ="%s/import/%s/%s" % (i_home,folder,filename) student_id = student.getId() images_dir = os.path.join("%s" % images_base,student_id) if not os.path.exists(images_dir): os.mkdir(images_dir) image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) for extension in ('.jpg','.JPG'): fullname = "%(picture)s%(extension)s" % vars() if os.path.exists(fullname): copy2(fullname,image_name) return "successfully copied passport picture" return "passport picture not found: %s.jpg or .JPG" % picture ###) def old____loadStudentFoto(self,student,filename,folder): ###( "return a student passport picture" app = student.application app_doc = app.getContent() #clear = student.clearance #clear_doc = clear.getContent() #matric_no = clear_doc.matric_no.upper() picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename) picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename) #import pdb;pdb.set_trace() if os.path.exists(picture1): file = open(picture1) elif os.path.exists(picture2): file = open(picture2) else: return "passport picture not found %s" % picture1 reopened = False if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened': self.portal_workflow.doActionFor(app,'open') reopened = True outfile = file.read() app_doc.manage_addFile('passport', file=outfile, title="%s.jpg" % filename) if reopened: self.portal_workflow.doActionFor(app,'close') return "successfully loaded passport picture" ###) security.declareProtected(ModifyPortalContent,'createOne') ###( def createOne(self,students_folder,student_brain,letter,commit=False): sid = self.waeup_tool.generateStudentId(letter) students_folder.invokeFactory('Student', sid) student = getattr(students_folder,sid) self.portal_workflow.doActionFor(student,'return') student.manage_setLocalRoles(sid, ['Owner',]) matric_no = student_brain.matric_no jamb_reg_no = student_brain.Entryregno self.students_catalog.addRecord(id = sid, matric_no = matric_no, jamb_reg_no = jamb_reg_no, sex = student_brain.Sex == "F", name = "%s %s %s" % (student_brain.Firstname, student_brain.Middlename, student_brain.Lastname) ) if commit: transaction.commit() return sid,jamb_reg_no ###) security.declareProtected(ModifyPortalContent,'addStudent') ###( def addStudent(self,dict): students_folder = self.portal_url.getPortalObject().campus.students sid = self.waeup_tool.generateStudentId('?') students_folder.invokeFactory('Student', sid) student_obj = getattr(students_folder,sid) f2t = self.student_field2types #from pdb import set_trace; set_trace() d = {} #d['jamb_sex'] = 'M' #if dict.get('sex'): # d['jamb_sex'] = 'F' entry_session = dict.get('entry_session') if entry_session == self.getSessionId()[0]: wfaction = 'admit' wft = 'wf_transition_admit' password = None else: wfaction = 'return' wft = 'wf_transition_return' password = self.generatePassword() self.makeStudentMember(sid,password) for pt in f2t.keys(): student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj) d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = dict.get(field,'') sub_doc.edit(mapping = d) new_state = f2t[pt][wft] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) self.portal_workflow.doActionFor(student_obj,wfaction) student_obj.manage_setLocalRoles(sid, ['Owner',]) return sid,password ###) security.declarePublic('getCertificateBrain') ###( def getCertificateBrain(self,cert_id): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"Certificate", 'id': cert_id}) if res: return res[0] return None ###) security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###( def get_csv_filenames(self): "do it" files = [file for file in os.listdir("%s/import/" % (i_home)) if file.endswith('.csv') and file.find('imported') == -1] return files ###) security.declarePublic('findStudentByMatricelNo') ###( def findStudentByMatricelNo(self,matric_no): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"StudentClearance", 'SearchableText': matric_no}) if res: return res[0] return None ###) security.declarePublic('makeStudentMember') ###( def makeStudentMember(self,sid,password='uNsEt'): """make the student a member""" membership = self.portal_membership membership.addMember(sid, password , roles=('Member', 'Student', ), domains='', properties = {'memberareaCreationFlag': False, 'homeless': True},) member = membership.getMemberById(sid) self.portal_registration.afterAdd(member, sid, password, None) #self.manage_setLocalRoles(sid, ['Owner',]) ###) security.declareProtected(View,'makeStudentData') ###( def makeStudentData(self,student_id,email=None,phone_nr=None): "create Datastructure for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentData') students_folder = self.portal_url.getPortalObject().campus.students #res = self.students_catalog(id=student_id) #if res: # st = res[0] #res = self.returning_import(matric_no = st.matric_no) res = self.returning_import(id = student_id) if res: student = res[0] else: logger.info('Id %s not found in returning_import' % student_id) return logger.info('%s creates data structure' % student_id) s_results = self.results_import(matric_no = student.matric_no) if s_results: lnr = self.getLevelFromResultsCosCode(s_results) level = "%d00" % lnr verdict,eligible = self.getVerdict(s_results[0].Verdict) #if eligible: # level = "%d00" % (lnr + 1) else: logger.info('matric_no %s not found in results_import' % student.matric_no) level = 0 verdict = 'N/A' #student should not be allowed to perform this transition #wftool = self.portal_workflow #wftool.doActionFor(student,'return') certcode_org = student.Coursemajorcode certcode = makeCertificateCode(certcode_org) certificate_brain = self.getCertificateBrain(certcode) if not certificate_brain: em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org) logger.info(em) matric_no = student.matric_no sid = student_id student_obj = getattr(students_folder,sid) student_obj.invokeFactory('StudentApplication','application') application = student_obj.application self.portal_workflow.doActionFor(application,'open',dest_container=application) da = {'Title': 'Application Data'} student_obj.invokeFactory('StudentPersonal','personal') da['jamb_reg_no'] = student.Entryregno em = self.getEntryMode(student.Entryregno) da['entry_mode'] = em personal = student_obj.personal self.portal_workflow.doActionFor(personal,'open',dest_container=personal) dp = {'Title': 'Personal Data'} student_obj.invokeFactory('StudentClearance','clearance') clearance = student_obj.clearance self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) dc = {'Title': 'Clearance/Eligibility Record'} dc['matric_no'] = matric_no state = student.State lga = student.LGA if state and lga: lga = state + ' / ' + lga else: lga = "None" da['jamb_lga'] = dc['lga'] = lga da['app_email'] = dp['email'] = email da['app_mobile'] = dp['phone'] = phone_nr dp['firstname'] = student.Firstname dp['middlename'] = student.Middlename dp['lastname'] = student.Lastname da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname) da['jamb_sex'] = student.Sex dp['sex'] = student.Sex == 'F' dp['perm_address'] = student.Permanent_Address application.getContent().edit(mapping=da) self.portal_workflow.doActionFor(application,'close',dest_container=application) personal.getContent().edit(mapping=dp) clearance.getContent().edit(mapping=dc) self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) # # Study Course # student_obj.invokeFactory('StudentStudyCourse','study_course') studycourse = student_obj.study_course self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) dsc = {} dsc['study_course'] = certcode dsc['current_level'] = level dsc['current_verdict'] = verdict dsc['current_mode'] = em dsc['current_session'] = '05' studycourse.getContent().edit(mapping=dsc) # # Level # ## l = getattr(studycourse,level,None) ## if l is None: ## studycourse.invokeFactory('StudentStudyLevel', level) ## l = getattr(studycourse, level) ## self.portal_workflow.doActionFor(l,'open',dest_container=l) ## l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) security.declareProtected(ModifyPortalContent,'admitOneStudent') ###( def admitOneStudent(self,brain,entry_session,pin_password): "create Datastructure for an admitted Student" #import pdb;pdb.set_trace() students_folder = self.portal_url.getPortalObject().campus.students logger = logging.getLogger('WAeUPTool.admitStudent') if brain.status != "admitted": logger.info('status of %s is %s' % (brain.reg_no,brain.status)) return pin_parts = brain.pin.split('-') if len(pin_parts) != 3: logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no)) return student_id = self.generateStudentId('?') students_folder.invokeFactory('Student', student_id) student_object = getattr(students_folder,student_id) if pin_password: password = pin_parts[2] self.makeStudentMember(student_id,password = password) student_object.manage_setLocalRoles(student_id, ['Owner',]) #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no)) # # application # student_object.invokeFactory('StudentApplication','application') application = student_object.application #self.portal_workflow.doActionFor(application,'open',dest_container=application) da = {'Title': 'Application Data'} da['jamb_reg_no'] = brain.reg_no sex = 'M' if brain.sex: sex = 'F' da['jamb_sex'] = sex #da['app_ac_pin'] = brain.pin #if brain.lga: # state_lga = brain.lga.split('_') # da['state_lga'] = state_lga[0] # da['jamb_lga'] = state_lga[-1] da['state_lga'] = brain.jamb_lga da['jamb_lga'] = brain.jamb_state da['jamb_score'] = brain.aggregate da['app_email'] = brain.email da['app_mobile'] = brain.phone if brain.entry_mode: da['entry_mode'] = brain.entry_mode elif brain.screening_type == 'pume': da['entry_mode'] = 'ume_ft' elif brain.screening_type == 'pde': da['entry_mode'] = 'de_ft' else: da['entry_mode'] = brain.screening_type da['entry_session'] = entry_session da['jamb_lastname'] = brain.lastname da['jamb_middlename'] = brain.middlenames # different field names! da['firstname'] = brain.firstname da['screening_application_date'] = brain.application_date da['date_of_birth'] = brain.date_of_birth da['jamb_first_cos'] = brain.course1 da['jamb_second_cos'] = brain.course2 da['course3'] = brain.course3 da['screening_type'] = brain.screening_type da['screening_score'] = brain.screening_score da['screening_date'] = brain.screening_date da['hq_type'] = brain.hq_type da['hq_grade'] = brain.hq_grade da['aos'] = brain.aos application.getContent().edit(mapping=da) #self.portal_workflow.doActionFor(application,'close',dest_container=application) # # personal # student_object.invokeFactory('StudentPersonal','personal') personal = student_object.personal #self.portal_workflow.doActionFor(personal,'open',dest_container=personal) dp = {'Title': 'Personal Data'} dp['sex'] = brain.sex dp['email'] = brain.email dp['phone'] = brain.phone dp['lastname'] = brain.lastname dp['middlename'] = brain.middlenames # different field names! dp['firstname'] = brain.firstname personal.getContent().edit(mapping=dp) # # clearance # student_object.invokeFactory('StudentClearance','clearance') clearance = student_object.clearance #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) dc = {'Title': 'Clearance/Eligibility Record'} dc['lga'] = brain.lga clearance.getContent().edit(mapping=dc) #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) # # study Course # student_object.invokeFactory('StudentStudyCourse','study_course') studycourse = student_object.study_course #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) dsc = {} dsc['study_course'] = brain.course_admitted dsc['current_verdict'] = '' dsc['current_mode'] = da['entry_mode'] if da['entry_mode'].startswith('de'): dsc['current_level'] = '200' else: dsc['current_level'] = '100' dsc['current_session'] = entry_session studycourse.getContent().edit(mapping=dsc) # # payments folder student_object.invokeFactory('PaymentsFolder','payments') payments = getattr(student_object,'payments') dpay = {} dpay['Title'] = 'Payments' payments.getContent().edit(mapping=dpay) self.portal_workflow.doActionFor(payments,'open') # # passport foto app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home, brain.screening_type, brain.reg_no) images_dir = os.path.join("%s" % images_base,student_id) if not os.path.exists(images_dir): os.mkdir(images_dir) image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) if os.path.exists(app_picture): copy2(app_picture,image_name) else: logger.info('passport of %s/%s not found: %s' % (student_id, brain.reg_no, app_picture)) return student_id ###) security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###( def makeStudentLevel(self,student_id): "create the StudyLevel for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentLevel') students_folder = self.portal_url.getPortalObject().campus.students res = self.students_catalog(id=student_id) if res: st = res[0] course = st.course matric_no = st.matric_no level = st.level res = self.results_import(matric_no = matric_no) if res: results = res logger.info('%s creating Level %s' % (student_id,level)) # # Level # student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) studycourse = getattr(student_obj,"study_course",None) self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse) l = getattr(studycourse,level,None) if l is None: studycourse.invokeFactory('StudentStudyLevel', level) l = getattr(studycourse, level) self.portal_workflow.doActionFor(l,'open',dest_container=l) l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) security.declarePublic('getHallInfo') ###( def getHallInfo(self,bed): """return Hall Info""" info = {} hall,block,room,letter = bed.split('_') res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall) if res and len(res) == 1: hall_brain = res[0] hall_doc = hall_brain.getObject().getContent() else: return info info['hall_title'] = hall_brain.Title info['maintenance_code'] = hall_doc.maintenance_code res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch") batch_doc = None for brain in res: if brain.id.startswith(info['maintenance_code']): batch_doc = brain.getObject().getContent() break if batch_doc is None: info['maintenance_fee'] = None else: info['maintenance_fee'] = batch_doc.cost return info ###) security.declareProtected(ModifyPortalContent,'removePictureFolder') ###( def removePictureFolder(self,student_id): """remove picture_folder by renaming it""" path = 'images' picture_path = os.path.join(i_home,path,student_id) if not os.path.exists(picture_path): return False os.rename(picture_path,picture_path + "_removed") return True ###) security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###( def restorePictureFolder(self,student_id): """restore picture_folder by renaming it""" path = 'images' picture_path = os.path.join(i_home,path,student_id) if not os.path.exists(picture_path + "_removed"): return False os.rename(picture_path + "_removed",picture_path) return True ###) security.declarePublic('picturesExist') ###( def picturesExist(self, ids,student_id=None): """check if pictures exist in the filesystem""" if student_id is None: student_id = self.getStudentId() if student_id is None: return False picture_path = os.path.join(images_base,student_id) if not os.path.exists(picture_path): return False pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] return set(ids).issubset(set(pictures)) ###) security.declarePublic('picturesList') ###( def picturesList(self): """check if pictures exist in the filesystem""" path = 'images' student_id = self.getStudentId() picture_path = os.path.join(i_home,path,student_id) if not os.path.exists(picture_path): return [] return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] ###) security.declarePublic('showFsPicture') ###( def showFsPicture(self,path): """return a picture from the filesystem""" picture_path = os.path.join(i_home,path) response = self.REQUEST.RESPONSE #import pdb;pdb.set_trace() registry = getToolByName(self, 'mimetypes_registry') mimetype = str(registry.lookupExtension(path.lower()) or registry.lookupExtension('file.bin')) if os.path.exists(picture_path): response.setHeader('Content-type',mimetype) return open(picture_path).read() picture_path = os.path.join(i_home,'import',path) if os.path.exists(picture_path): return open(picture_path).read() ###) security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###( def deleteAllCourses(self,department="All"): ''' delete the courses''' pm = self.portal_membership member = pm.getAuthenticatedMember() if str(member) not in ("henrik","joachim"): return "not possible" if department == "All": res = self.portal_catalog({'meta_type': 'Department'}) if len(res) < 1: return "No Departments found" deleted = [] for dep in res: cf = dep.getObject().courses if cf: cf.manage_delObjects(ids=cf.objectIds()) deleted.append("deleted Courses in %s" % dep.getId) return "\r".join(deleted) ###) security.declareProtected(ModifyPortalContent,'getLogfileLines') ###( def getLogfileLines(self,filename="event.log",numlines=20): """Get last NUMLINES lines of logfile FILENAME. Return last lines' of a file in the instances logfile directory as a list. The number of returned lines equals `numlines' or less. If less than `numlines' lines are available, the whole file ist returned. If the file can not be opened or some other error occurs, empty list is returend. """ result = [] lines_hit = 0 # We only handle files in instances' log directory... logpath = os.path.join(i_home, "log") filename = str(os.path.abspath( os.path.join( logpath, filename ))) if not filename.startswith( logpath ): # Attempt to access file outside log-dir... return [] try: fd = file( filename, "rb" ) except IOError: return [] if not fd: return [] if os.linesep == None: linesep = '\n' else: linesep = os.linesep # Try to find 'numlines' times a lineseparator, searching from end # and moving to the beginning of file... fd.seek( 0, 2) # Move to end of file... while lines_hit < numlines: if fd.read(1) == linesep[-1]: # This moves filedescriptor # one step forward... lines_hit += 1 try: fd.seek( -2, 1) # Go two bytes back from current pos... except IOError: # We cannot go back two bytes. Maybe the file is too small... break fd.seek(2,1) # Read all lines from current position... result = fd.readlines() # Remove line endings... result = [x.strip() for x in result] fd.close() return result ###) security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###( def getCallbacksFromLog(self,filename): """fix Online Payment Transactions from Z2.log entries""" import transaction import random from cgi import parse_qs from urlparse import urlparse #from pdb import set_trace wftool = self.portal_workflow current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") students_folder = self.portal_url.getPortalObject().campus.students s = r'(?P\S+) - (?P\S+) \[' s += r'(?P.*)\] "(?P.*)" (?P\d+ \d+) "' s += r'(?P.*)" "(?P.*)"' data = re.compile(s) start = True tr_count = 1 total = 0 #name = 'pume_results' #name = 'epaymentsuccessful_z2log2' name = filename no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.getFailedTransactions') try: transactions = open("%s/import/%s" % (i_home,name),"rb").readlines() except: logger.error('Error reading %s' % name) return tas = [] for line in transactions: dict = {} items = data.search(line) dict['idict'] = idict = items.groupdict() #print idict #from pdb import set_trace;set_trace() urlparsed = urlparse(idict['get'][4:]) #print urlparsed path = urlparsed[2].split('/') dict['student_id'] = student_id = path[8] dict['payment_id'] = payment_id = path[10] dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4]) tas.append(dict) tr_count += 1 return tas ###) security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###( def importOnlinePaymentTransactions(self): """load Online Payment Transactions from CSV values""" import transaction import random #from pdb import set_trace current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") opt = self.online_payments_import students_folder = self.portal_url.getPortalObject().campus.students start = True tr_count = 1 total = 0 #name = 'pume_results' name = 'OnlineTransactions' no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions') try: transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) except: logger.error('Error reading %s.csv' % name) return for pay_transaction in transactions: if start: start = False logger.info('Start loading from %s.csv' % name) s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()]) no_import.append('%s,"Error"' % s) format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()]) format_error = format + ',"%(Error)s"' data = {} # format of the first file sent by Tayo #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date']) #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = pay_transaction['Resp Code'] #data['amount'] = amount = pay_transaction['Amount'] # format of the second file sent by Tayo #data['datetime'] = date = 0 #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = '00' #data['amount'] = amount = pay_transaction['Amount'] # format of the third file sent by Kehinde data['datetime'] = date = 0 data['student_id'] = student_id = pay_transaction['customer_id'] data['order_id'] = order_id = pay_transaction['merchant_reference'] data['response_code'] = response_code = '00' data['amount'] = amount = pay_transaction['Amount'] dup = False if response_code == "12": continue try: opt.addRecord(**data) except ValueError: dup = True #from pdb import set_trace;set_trace() if dup: if response_code == "00": try: opt.modifyRecord(**data) except: logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data) continue else: pay_transaction['Error'] = "Duplicate order_id" no_import.append( format_error % pay_transaction) logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data) continue tr_count += 1 if tr_count > 1000: if len(no_import) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import) + '\n') no_import = [] em = '%d transactions committed\n' % (tr_count) transaction.commit() regs = [] logger.info(em) total += tr_count tr_count = 0 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import)) return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) ###) security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###( def mass_create_faculty(self,mapping): "create a faculty" logger = logging.getLogger('WAeUPTool.mass_create_faculty') academics_folder = self.portal_url.getPortalObject().campus.academics fid = mapping['code'] if getattr(academics_folder,fid,None) is not None: return '', "Faculty with ID: %s exists" % fid logger.info('Creating Faculty %(code)s, %(title)s' % mapping) try: academics_folder.invokeFactory('Faculty', fid) except BadRequest,E: return '', "%s" % E f = getattr(academics_folder,fid,None) f.getContent().edit(mapping=mapping) return fid,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###( def mass_edit_faculty(self,mapping): "edit a faculty" logger = logging.getLogger('WAeUPTool.mass_edit_faculty') academics_folder = self.portal_url.getPortalObject().campus.academics fid = mapping['code'] f = getattr(academics_folder,fid,None) if f is None: return '', "Faculty with ID: %s does not exist" % fid logger.info('Editing Faculty %(code)s, %(title)s' % mapping) f.getContent().edit(mapping=mapping) return fid,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_department') ###( def mass_create_department(self,mapping): "create a department in the correct faculty" logger = logging.getLogger('WAeUPTool.mass_create_department') fid = mapping['faculty_code'] if getattr(self,'_v_faculties',None) is None: res = self.portal_catalog(portal_type = "Faculty") self._v_faculties = {} for f in res: self._v_faculties[f.getId] = f.getObject() f = self._v_faculties.get(fid,None) if f is None: return '', "No Faculty with ID: %s" % fid else: did = mapping.get('code') d = getattr(f,did,None) if d is None or d.portal_type == "Faculty": logger.info('Creating Department %(code)s, %(title)s' % mapping) try: f.invokeFactory('Department', did) except BadRequest,E: return '', "%s" % E d = getattr(f,did) d.invokeFactory('CoursesFolder','courses') courses = getattr(d,'courses') dict = {'Title': 'Courses'} courses.getContent().edit(mapping=dict) d.invokeFactory('CertificatesFolder','certificates') certificates = getattr(d,'certificates') dict = {'Title': 'Certificates'} certificates.getContent().edit(mapping=dict) d.getContent().edit(mapping=mapping) return did,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_department') ###( def mass_edit_department(self,mapping): "create a department in the correct faculty" logger = logging.getLogger('WAeUPTool.mass_create_department') academics_folder = self.portal_url.getPortalObject().campus.academics fid = mapping['faculty_code'] did = mapping.get('code') try: d = getattr(getattr(academics_folder,fid),did,None) except KeyError: return '', "Department %s or Faculty %s wrong" % (did,fid) else: if d is None or d.portal_type == "Faculty": logger.info('Editing Department %(code)s, %(title)s' % mapping) d.getContent().edit(mapping=mapping) return did,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_course') ###( def mass_create_course(self,mapping): #import pdb;pdb.set_trace() if getattr(self,'_v_course_list',None) is None: self._v_course_list = [] if getattr(self,'_v_departments',None) is None: res = self.portal_catalog(portal_type = "Department") self._v_department_courses = {} for d in res: self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None) did = mapping['department_code'] d = self._v_department_courses.get(did,None) if d is None: return '', "No Department with ID: %s" % did course_id = mapping.get('code') if course_id in self._v_course_list: return '', "Duplicate Course ID: %s" % did c = getattr(d,course_id,None) if c is not None: return '', "Duplicate Course ID: %s" % did try: d.invokeFactory('Course', course_id) except BadRequest,E: return '', "%s" % E self._v_course_list.append(course_id) c = getattr(d,course_id) c.getContent().edit(mapping=mapping) return course_id,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_course') ###( def mass_edit_course(self,mapping): #import pdb;pdb.set_trace() course_id = mapping.get('code') res = self.portal_catalog(id=course_id) if not res: return '', "No Course with ID: %s" % course_id c = res[0].getObject() c.getContent().edit(mapping=mapping) return course_id,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###( def mass_create_certificate(self,mapping): if getattr(self,'_v_certificate_list',None) is None: self._v_certificate_list = [] if getattr(self,'_v_department_certificates',None) is None: res = self.portal_catalog(portal_type = "Department") self._v_department_certificates = {} for d in res: self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None) did = mapping['department_code'] d = self._v_department_certificates.get(did,None) if d is None: return '', "No Department with ID: %s" % did certificate_id = mapping.get('code') if certificate_id in self._v_certificate_list: return '', "Duplicate Certificate ID: %s" % did c = getattr(d,certificate_id,None) if c is not None: return '', "Duplicate Certificate ID: %s" % did try: d.invokeFactory('Certificate', certificate_id) except BadRequest,E: return '', "%s" % E self._v_certificate_list.append(certificate_id) c = getattr(d,certificate_id) c.getContent().edit(mapping=mapping) return certificate_id,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###( def mass_edit_certificate(self,mapping): #import pdb;pdb.set_trace() certificate_id = mapping.get('code') res = self.portal_catalog(id=certificate_id) if not res: return '', "No Certificate with ID: %s" % did c = res[0].getObject() c.getContent().edit(mapping=mapping) return certificate_id,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_application') ###( def mass_create_application(self,mapping): #import pdb;pdb.set_trace() reg_no = mapping.get('reg_no') try: self.applicants_catalog.addRecord(**mapping) except ValueError: return '', "applicant record with reg_no %s already exists" % reg_no return reg_no,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_application') ###( def mass_edit_application(self,mapping): #import pdb;pdb.set_trace() reg_no = mapping.get('reg_no') try: self.applicants_catalog.modifyRecord(**mapping) except KeyError: return '', "applicant record with reg_no %s does not exist" % reg_no return reg_no,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###( def mass_create_course_result(self,mapping): #import pdb;pdb.set_trace() if getattr(self,'_v_courses',None) is None: res = self.courses_catalog() self._v_courses = {} for brain in res: self._v_courses[brain.code] = brain course_id = mapping.get('code') if course_id not in self._v_courses.keys(): return '', "No course with ID: %s" % did id_key = '' for id_key in ('student_id','matric_no'): id_field = mapping.get(id_key,None) if id_field is not None: student_id = id_field break query = Eq(id_key,id_field) res = self.students_catalog.evalAdvancedQuery(query) if not res: return '', "No student with %(id_field)s: %(id_key)s" % vars() if id_field != "student_id": mapping['student_id'] = res[0].id mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping for k in ('semester',): mapping[k] = getattr(self._v_courses[course_id],k) try: self.course_results.addRecord(**mapping) except ValueError: return '', "course result already exists: %s" % key return key,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###( def mass_edit_course_result(self,mapping): #import pdb;pdb.set_trace() # if getattr(self,'_v_courses',None) is None: # res = self.courses_catalog() # self._v_courses = {} # for brain in res: # self._v_courses[brain.code] = brain # course_id = mapping.get('code') # if course_id not in self._v_courses.keys(): # return '', "No course with ID: %s" % did id_key = '' for id_key in ('student_id','matric_no'): id_field = mapping.get(id_key,None) if id_field is not None: student_id = id_field break query = Eq(id_key,id_field) res = self.students_catalog.evalAdvancedQuery(query) if not res: return '', "No student with %(id_field)s: %(id_key)s" % vars() if id_field != "student_id": mapping['student_id'] = res[0].id mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping # for k in ('semester',): # mapping[k] = getattr(self._v_courses[course_id],k) try: self.course_results.modifyRecord(**mapping) except KeyError: return '', "No course result to edit: %s" % key return key,'' ###) security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###( def mass_create_certificate_course(self,mapping): if getattr(self,'_v_courses',None) is None: res = self.courses_catalog() self._v_courses= [course.code for course in res] if getattr(self,'_v_ceritficates',None) is None: res = self.portal_catalog(portal_type = "Certificate") self._v_certificates = {} for cert in res: self._v_certificates[cert.getId] = cert.getObject() certificate_course_id = mapping.get('code') if certificate_course_id not in self._v_courses: return '', "No Course with ID: %s" % certificate_course_id cert_id = mapping['certificate_code'] cert = self._v_certificates.get(cert_id,None) if cert is None: return '', "No Certificate with ID: %s" % cert_id level_id = mapping.get('level') level = getattr(cert,level_id,None) if level is None: cert.invokeFactory('StudyLevel', level_id) level = getattr(cert,level_id,None) elif hasattr(level,certificate_course_id): return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\ (certificate_course_id,cert_id,level_id) level.invokeFactory('CertificateCourse', certificate_course_id) c = getattr(level,certificate_course_id) c.getContent().edit(mapping=mapping) return certificate_course_id,'' ###) field2types_student = { ###( 'StudentApplication': {'id': 'application', 'title': 'Application Data', 'wf_transition_return': 'close', 'wf_transition_admit': 'remain', 'fields': ('jamb_reg_no', 'entry_mode', 'entry_session', 'jamb_score', 'app_email', 'jamb_age', 'jamb_state', 'jamb_lga', ) }, #'StudentPume': # {'id': 'pume', # 'title': 'Pume Data', # 'wf_transition_return': 'close', # 'wf_transition_admit': 'close', # 'fields': # ('pume_score', # ) # }, 'StudentClearance': {'id': 'clearance', 'title': 'Clearance/Eligibility Record', 'wf_transition_return': 'close', 'wf_transition_admit': 'remain', 'fields': ('matric_no', 'nationality', 'lga', 'birthday', ) }, 'StudentPersonal': {'id': 'personal', 'title': 'Personal Data', 'wf_transition_return': 'open', 'wf_transition_admit': 'remain', 'fields': ('firstname', 'middlename', 'lastname', 'sex', 'email', 'phone', 'perm_address', ) }, 'StudentStudyCourse': {'id': 'study_course', 'title': 'Study Course', 'wf_transition_return': 'open', 'wf_transition_admit': 'remain', 'fields': ('study_course', 'current_level', 'current_session', 'current_mode', 'current_verdict', 'previous_verdict', ) }, 'PaymentsFolder': {'id': 'payments', 'title': 'Payments', 'wf_transition_return': 'open', 'wf_transition_admit': 'open', 'fields': () }, } ###) security.declareProtected(ModifyPortalContent,'mass_create_student') ###( def mass_create_student(self,mapping): "create a students record due import" logger = logging.getLogger('WAeUPTool.mass_create_student') students_folder = self.portal_url.getPortalObject().campus.students jamb_reg_no = mapping.get('jamb_reg_no',None) if jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if res: return '',"jamb_reg_no exists" matric_no = mapping.get('matric_no',None) if matric_no: res = self.students_catalog(matric_no = matric_no) if res: return '',"matric_no exists" sid = self.waeup_tool.generateStudentId('?') students_folder.invokeFactory('Student', sid) student_obj = getattr(students_folder,sid) f2t = self.field2types_student d = {} transition = mapping.get('reg_transition','admit') if transition not in ('admit','return'): return '',"no valid transition provided" for pt in f2t.keys(): student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = mapping.get(field,'') if pt == "StudentApplication": d['jamb_sex'] = 'M' if mapping.get('sex'): d['jamb_sex'] = 'F' d['jamb_firstname'] = mapping.get('firstname',None) d['jamb_middlename'] = mapping.get('middlename',None) d['jamb_lastname'] = mapping.get('lastname',None) # if pt == "StudyCourse": # for von,zu in (('entry_mode','current_mode'), # ('entry_session','current_session')): # if mapping.get(zu,None) is None and mapping.get(von,None) is not None: # d[zu] = mapping[von] sub_doc.edit(mapping = d) #import pdb;pdb.set_trace() new_state = f2t[pt]['wf_transition_%(transition)s' % vars()] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) self.portal_workflow.doActionFor(student_obj,transition) student_obj.manage_setLocalRoles(sid, ['Owner',]) return sid,'' ###) security.declareProtected(ModifyPortalContent,'mass_edit_student') ###( def mass_edit_student(self,mapping): "edit a students record due import" logger = logging.getLogger('WAeUPTool.mass_edit_student') students_folder = self.portal_url.getPortalObject().campus.students sid = mapping.get('id',None) jamb_reg_no = mapping.get('jamb_reg_no',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() if sid: res = self.students_catalog(id = sid) if not res: return '',"no student with id %s" % sid if matric_no and res[0].matric_no and\ matric_no != res[0].matric_no: logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no)) if jamb_reg_no and res[0].jamb_reg_no and\ jamb_reg_no != res[0].jamb_reg_no: logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no)) elif jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if not res: return '',"no student with jamb_reg_no %s" % jamb_reg_no editable_keys.remove('jamb_reg_no') elif matric_no: res = self.students_catalog(matric_no = matric_no) if not res: return '',"no student with matric_no %s" % matric_no editable_keys.remove('matric_no') ## included only to change wf state from admitted to returning #if res[0].review_state not in ('admitted','objection_raised'): # return '%s' % res[0].id ,"student is not in state admitted or objection_raised" ## end inclusion sid = res[0].id student_obj = getattr(students_folder,sid) f2t = self.field2types_student d = {} #import pdb;pdb.set_trace() any_change = False for pt in f2t.keys(): if pt == "student_application": d['jamb_sex'] = 'M' if mapping.get('sex'): d['jamb_sex'] = 'F' intersect = set(f2t[pt]['fields']).intersection(set(editable_keys)) if intersect: sub_obj = getattr(student_obj,f2t[pt]['id'],None) if sub_obj is None: try: student_obj.invokeFactory(pt,f2t[pt]['id']) except: continue sub_obj = getattr(student_obj,f2t[pt]['id']) d['Title'] = f2t[pt]['title'] sub_doc = sub_obj.getContent() for field in intersect: changed = False if getattr(sub_doc,field,None) != mapping.get(field,''): any_change = True changed = True d[field] = mapping.get(field,'') if changed: sub_doc.edit(mapping = d) ## included only to change wf state from admitted to returning # if res[0].review_state in ('admitted','objection_raised'): # new_state = f2t[pt]['wf_transition_return'] # sub_obj = getattr(student_obj,f2t[pt]['id'],None) # if sub_obj and new_state != "remain": # try: # self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) # except: # #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id)) # pass #if res[0].review_state in ('admitted','objection_raised'): # wfaction = 'return' # try: # self.portal_workflow.doActionFor(student_obj,wfaction) # logger.info('%s, wf state changed' % sid) # any_change = True # except: # logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state)) # pass ## end inclusion if any_change: return sid,'' else: return sid,'not modified' ###) security.declareProtected(ModifyPortalContent,"importData")###( def importData(self,filename,name,edit=False,bypass_queue_catalog=False): """load data from CSV values""" import transaction import random pm = self.portal_membership member = pm.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.importData') current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") students_folder = self.portal_url.getPortalObject().campus.students start = True tr_count = 0 total_imported = 0 total_not_imported = 0 total = 0 iname = "import_%s" % name if iname == 'import_application': commit_after = 2000 else: commit_after = 100 stool = getToolByName(self, 'portal_schemas') ltool = getToolByName(self, 'portal_layouts') schema = stool._getOb(iname) if schema is None: em = 'No such schema %s' % iname logger.error('No such schema %s' % iname) return em layout = ltool._getOb(iname) if layout is None: em = 'No such layout %s' % iname logger.error(em) return em validators = {} for widget in layout.keys(): validators[widget] = layout[widget].validate #import pdb;pdb.set_trace() mode = "create" if edit: mode = "edit" importer_name = "mass_%(mode)s_%(name)s" % vars() importer = getattr(self, '%s' % importer_name,None) if importer is None: em = 'No importer function %s' % importer_name logger.error(em) return em not_imported = [] imported = [] try: items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb")) except: em = 'Error reading %s.csv' % filename logger.error(em) return em for item in items: if start: start = False adapters = [MappingStorageAdapter(schema, item)] logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname)) attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next() import_keys = [k for k in attrs if not k.startswith('ignore')] diff2schema = set(import_keys).difference(set(schema.keys())) diff2layout = set(import_keys).difference(set(layout.keys())) if diff2schema: em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) return em if mode == "create": required_keys = [layout.getIdUnprefixed(id) for id,widget in layout.objectItems() if widget.is_required] if not set(required_keys).issubset(set(import_keys)): diff2import = set(required_keys).difference(set(import_keys)) em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import) return em s = ','.join(['"%s"' % fn for fn in import_keys]) open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n') s = '"id",' + s open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n') format = ','.join(['"%%(%s)s"' % fn for fn in import_keys]) format_error = format + ',"%(Error)s"' format = '"%(id)s",'+ format dm = DataModel(item, adapters,context=self) ds = DataStructure(data=item,datamodel=dm) error_string = "" total += 1 for k in import_keys: if not validators[k](ds,mode=mode): error_string += " %s : %s" % (k, self.translation_service(ds.getError(k), ds.getErrorMapping(k))) if not error_string: temp_item = item.copy() temp_item.update(dm) temp_item['id'],error = importer(temp_item) if error: error_string += error else: item = temp_item if error_string: item['Error'] = error_string not_imported.append(format_error % item) total_not_imported += 1 else: em = format % item imported.append(em) tr_count += 1 total_imported += 1 logger.info("%(total_imported)d of %(total)d %(em)s" % vars()) if total and not total % commit_after: transaction.commit() if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported) + '\n') not_imported = [] if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported) + '\n') imported = [] em = '%d transactions committed\n' % (tr_count) regs = [] logger.info(em) tr_count = 0 #if total > 100: # return if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported)) if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported)) em = "Finished: %d imported, %d not imported (of total %d)" % (total_imported,total_not_imported,total) logger.info(em) return em ###) security.declareProtected(ModifyPortalContent,"moveImagesToFS")###( def moveImagesToFS(self,student_id="O738726"): "move the images to the filesystem" images_dir = os.path.join("%s" % images_base,student_id) student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) stool = getToolByName(self, 'portal_schemas') schemas = ['student_application', 'student_clearance', ] created = False for schema_id in schemas: schema = stool._getOb(schema_id) object = getattr(student_folder,schema_id[len('student_'):],None) if object is None: continue doc = object.getContent() for key in schema.keys(): if schema[key].meta_type != "CPS Image Field": continue #import pdb;pdb.set_trace() image = getattr(doc,key,None) if not image or not hasattr(image,"data"): continue if not created: if not os.path.exists(images_dir): os.mkdir(images_dir) created = True filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars()) open(filename,"wb").write(str(image.data)) ###) security.declareProtected(ModifyPortalContent,"movePassportToFS")###( def movePassportToFS(self,student_id="O738726"): "move the passports to the filesystem" images_dir = os.path.join("%s" % i_home,'passports') student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) stool = getToolByName(self, 'portal_schemas') schemas = ['student_application', #'student_clearance', ] created = False for schema_id in schemas: schema = stool._getOb(schema_id) object = getattr(student_folder,schema_id[len('student_'):],None) if object is None: continue doc = object.getContent() for key in schema.keys(): if schema[key].meta_type != "CPS Image Field": continue #import pdb;pdb.set_trace() image = getattr(doc,key) if not hasattr(image,"data"): continue if not created: if not os.path.exists(images_dir): os.mkdir(images_dir) created = True filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars()) open(filename,"wb").write(str(image.data)) ###) InitializeClass(WAeUPTool)