3#-*- mode: python; mode: fold -*- # (C) Copyright 2005 The WAeUP group # Author: Joachim Schmitz (js@aixtraware.de) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA # 02111-1307, USA. # # $Id: WAeUPTool.py 1988 2007-07-05 11:14:12Z joachim $ """The WAeUP Tool Box. """ from AccessControl import ClassSecurityInfo from Acquisition import aq_inner from Acquisition import aq_parent from Globals import DTMLFile from Globals import InitializeClass from OFS.SimpleItem import SimpleItem from Products.CMFCore.utils import getToolByName from Products.CPSSchemas.DataStructure import DataStructure from Products.CPSSchemas.DataModel import DataModel from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter from Products.CMFCore.ActionProviderBase import ActionProviderBase from Products.CMFCore.permissions import View from Products.ZCatalog.ZCatalog import ZCatalog from Products.CMFCore.permissions import ModifyPortalContent from Products.CMFCore.permissions import ManagePortal from Products.CMFCore.utils import UniqueObject from Products.CMFCore.URLTool import URLTool from Products.CMFCore.utils import getToolByName from Students import makeCertificateCode from Globals import package_home,INSTANCE_HOME p_home = package_home(globals()) i_home = INSTANCE_HOME import DateTime import logging import transaction import csv,re,os from Products.AdvancedQuery import Eq, Between, Le,In def getObject(object,name): if object.hasObject(name): return getattr(object,name) return None class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase): """WAeUP tool""" id = 'waeup_tool' meta_type = 'WAeUP Tool' _actions = () security = ClassSecurityInfo() security.declareObjectProtected(View) manage_options = ( ActionProviderBase.manage_options + SimpleItem.manage_options ) def rwrite(self,s): ###( response = self.REQUEST.RESPONSE response.setHeader('Content-type','text/html; charset=ISO-8859-15') response.write("%s
\r\n" % s) ###) def sleep(self,secs): ###( "sleep" import time time.sleep(secs) return ###) security.declareProtected(ModifyPortalContent,'openLog') ###( def openLog(self,name): """open a log file""" version = 1 path = "%s/log/%s_%d.log" % (i_home,name,version) while os.path.exists(path): version += 1 path = "%s/log/%s_%d.log" % (i_home,name,version) log = open(path,"w") return log ###) security.declareProtected(ModifyPortalContent,'writeLog') ###( def writeLog(self,logfile,s): """write to the log file""" logfile.write(s) ###) def generateStudentId(self,letter): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if letter == '?': letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') sid = "%c%d" % (letter,r.randint(99999,1000000)) students = self.portal_url.getPortalObject().campus.students ## while hasattr(students, sid): ## sid = "%c%d" % (letter,r.randint(99999,1000000)) while self.students_catalog(id = sid): sid = "%c%d" % (letter,r.randint(99999,1000000)) return sid ###) def generatePassword(self,s=None): ###( import random r = random ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): if s is None: s = 'abcdefghklmnpqrstuvwxy23456789' pw = '' while len(pw) < 6: pw += r.choice(s) return pw ###) security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###( def removeDeletedDocIds(self, max=1000): """ remove deleted docids from repository commit after max """ logger = logging.getLogger('WAeUPTool.removeDeletedDocIds') repository = getToolByName(self, 'portal_repository') pxtool = getToolByName(self, 'portal_proxies') pxtool_infos = pxtool.getRevisionsUsed() nb_revs = 0 docids_d = {} # all docids unused_docids_d = {} # all docids that are unused ids_unused_revs_docids = [] # ids for revs of unused docids ids_unused_revs = [] # ids for unused revs total = 0 idlist = repository.objectIds() for id in idlist: docid, rev = repository._splitId(id) if docid is None: logger.info("invalid doc_id %s" % docid) continue nb_revs += 1 docids_d[docid] = None if not pxtool_infos.has_key(docid): unused_docids_d[docid] = None ids_unused_revs_docids.append(id) ids_unused_revs.append(id) elif not pxtool_infos[docid].has_key(rev): ids_unused_revs.append(id) if len(ids_unused_revs) >= max: repository.manage_delObjects(ids_unused_revs) #import pdb;pdb.set_trace() transaction.commit() total += max logger.info('removed %d total %d unused docids ' % (max,total)) anz = len(ids_unused_revs) if anz > 0: repository.manage_delObjects(ids_unused_revs) transaction.commit() total += anz logger.info('finished removing %d unused docids ' % (total)) ###) security.declareProtected(ModifyPortalContent,'getCredential') ###( def getCredential(self,student_id): "return a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return None return getattr(student_entry,"password","not set") ###) security.declarePublic('checkPassword') ###( def checkPassword(self,student_id,password): "return a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return False return getattr(student_entry,"password","not set") == password ###) security.declarePublic('editPassword') ###( def editPassword(self,student_id,password): "edit a student password" student_entry = getattr(self.portal_directories.students,student_id,None) if student_entry is None: return setattr(student_entry,'password',password) ###) security.declareProtected(View,'doCommit') ###( def doCommit(self,logger=None): "commit some transactions" transaction.commit() ###) security.declarePublic('loadStudentFoto') ###( def loadStudentFoto(self,student,filename,folder): "return a student passport picture" app = student.application app_doc = app.getContent() #clear = student.clearance #clear_doc = clear.getContent() #matric_no = clear_doc.matric_no.upper() picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename) picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename) #import pdb;pdb.set_trace() if os.path.exists(picture1): file = open(picture1) elif os.path.exists(picture2): file = open(picture2) else: return "passport picture not found %s" % picture1 reopened = False if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened': self.portal_workflow.doActionFor(app,'open') reopened = True outfile = file.read() app_doc.manage_addFile('passport', file=outfile, title="%s.jpg" % filename) if reopened: self.portal_workflow.doActionFor(app,'close') return "successfully loaded passport picture" ###) security.declareProtected(ModifyPortalContent,'createOne') ###( def createOne(self,students_folder,student_brain,letter,commit=False): sid = self.waeup_tool.generateStudentId(letter) students_folder.invokeFactory('Student', sid) student = getattr(students_folder,sid) self.portal_workflow.doActionFor(student,'return') student.manage_setLocalRoles(sid, ['Owner',]) matric_no = student_brain.matric_no jamb_reg_no = student_brain.Entryregno self.students_catalog.addRecord(id = sid, matric_no = matric_no, jamb_reg_no = jamb_reg_no, sex = student_brain.Sex == "F", name = "%s %s %s" % (student_brain.Firstname, student_brain.Middlename, student_brain.Lastname) ) if commit: transaction.commit() return sid,jamb_reg_no ###) security.declareProtected(ModifyPortalContent,'addStudent') ###( def addStudent(self,dict): students_folder = self.portal_url.getPortalObject().campus.students sid = self.waeup_tool.generateStudentId('?') students_folder.invokeFactory('Student', sid) student_obj = getattr(students_folder,sid) f2t = self.student_field2types #from pdb import set_trace; set_trace() d = {} d['jamb_sex'] = 'M' if dict.get('sex'): d['jamb_sex'] = 'F' entry_session = dict.get('entry_session') if entry_session == self.getSessionId()[-2:]: wfaction = 'admit' wft = 'wf_transition_admitted' password = None else: wfaction = 'return' wft = 'wf_transition_returning' password = self.generatePassword() self.makeStudentMember(sid,password) for pt in f2t.keys(): student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj) d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = dict.get(field,'') sub_doc.edit(mapping = d) new_state = f2t[pt][wft] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) self.portal_workflow.doActionFor(student_obj,wfaction) student_obj.manage_setLocalRoles(sid, ['Owner',]) return sid,password ###) security.declarePublic('getCertificateBrain') ###( def getCertificateBrain(self,cert_id): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"Certificate", 'id': cert_id}) if res: return res[0] return None ###) security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###( def get_csv_filenames(self): "do it" files = [file for file in os.listdir("%s/import/" % (i_home)) if file.endswith('.csv') and file.find('imported') == -1] return files ###) security.declarePublic('findStudentByMatricelNo') ###( def findStudentByMatricelNo(self,matric_no): "do it" res = ZCatalog.searchResults(self.portal_catalog_real, {'portal_type':"StudentClearance", 'SearchableText': matric_no}) if res: return res[0] return None ###) security.declarePublic('makeStudentMember') ###( def makeStudentMember(self,sid,password='uNsEt'): """make the student a member""" membership = self.portal_membership membership.addMember(sid, password , roles=('Member', 'Student', ), domains='', properties = {'memberareaCreationFlag': False, 'homeless': True},) member = membership.getMemberById(sid) self.portal_registration.afterAdd(member, sid, password, None) #self.manage_setLocalRoles(sid, ['Owner',]) ###) security.declarePublic('makeStudentData') ###( def makeStudentData(self,student_id,email=None,phone_nr=None): "create Datastructure for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentData') students_folder = self.portal_url.getPortalObject().campus.students #res = self.students_catalog(id=student_id) #if res: # st = res[0] #res = self.returning_import(matric_no = st.matric_no) res = self.returning_import(id = student_id) if res: student = res[0] else: logger.info('Id %s not found in returning_import' % student_id) return logger.info('%s creates data structure' % student_id) s_results = self.results_import(matric_no = student.matric_no) if s_results: lnr = self.getLevelFromResultsCosCode(s_results) level = "%d00" % lnr verdict,eligible = self.getVerdict(s_results[0].Verdict) if eligible: level = "%d00" % (lnr + 1) else: logger.info('matric_no %s not found in results_import' % student.matric_no) level = 0 verdict = 'N/A' #student should not be allowed to perform this transition #wftool = self.portal_workflow #wftool.doActionFor(student,'return') certcode_org = student.Coursemajorcode certcode = makeCertificateCode(certcode_org) certificate_brain = self.getCertificateBrain(certcode) if not certificate_brain: em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org) logger.info(em) matric_no = student.matric_no sid = student_id student_obj = getattr(students_folder,sid) student_obj.invokeFactory('StudentApplication','application') application = student_obj.application self.portal_workflow.doActionFor(application,'open',dest_container=application) da = {'Title': 'Application Data'} student_obj.invokeFactory('StudentPersonal','personal') da['jamb_reg_no'] = student.Entryregno em = self.getEntryMode(student.Entryregno) ## em = student.Mode_of_Entry ## if em in ('DIRECT', 'DIRECT ENTRY',): ## em = 'DE' ## elif em in ('U.M.E', 'UNE',): ## em = 'UME' ## elif not em: ## em = "unknown" da['entry_mode'] = em personal = student_obj.personal self.portal_workflow.doActionFor(personal,'open',dest_container=personal) dp = {'Title': 'Personal Data'} student_obj.invokeFactory('StudentClearance','clearance') clearance = student_obj.clearance self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) dc = {'Title': 'Clearance/Eligibility Record'} dc['matric_no'] = matric_no state = student.State lga = student.LGA if state and lga: lga = state + ' / ' + lga else: lga = "None" da['jamb_lga'] = dc['lga'] = lga da['app_email'] = dp['email'] = email da['app_mobile'] = dp['phone'] = phone_nr dp['firstname'] = student.Firstname dp['middlename'] = student.Middlename dp['lastname'] = student.Lastname da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname) da['jamb_sex'] = student.Sex dp['sex'] = student.Sex == 'F' dp['perm_address'] = student.Permanent_Address application.getContent().edit(mapping=da) self.portal_workflow.doActionFor(application,'close',dest_container=application) personal.getContent().edit(mapping=dp) clearance.getContent().edit(mapping=dc) self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) ## catd = {} ## catd['id'] = sid ## catd['entry_mode']= da['entry_mode'] ## catd['matric_no'] = matric_no ## catd['jamb_reg_no'] = da['jamb_reg_no'] ## catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp ## catd['sex'] = dp['sex'] ## catd['level'] = level ## catd['verdict'] = verdict ## if certificate_brain: ## cpath = certificate_brain.getPath().split('/') ## catd['faculty'] = cpath[-4] ## catd['department'] = cpath[-3] ## catd['course'] = certcode ## self.students_catalog.modifyRecord(**catd) # # Study Course # student_obj.invokeFactory('StudentStudyCourse','study_course') studycourse = student_obj.study_course self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) dsc = {} dsc['study_course'] = certcode dsc['current_level'] = level dsc['current_verdict'] = verdict dsc['current_mode'] = em dsc['current_session'] = '06' studycourse.getContent().edit(mapping=dsc) # # Level # ## l = getattr(studycourse,level,None) ## if l is None: ## studycourse.invokeFactory('StudentStudyLevel', level) ## l = getattr(studycourse, level) ## self.portal_workflow.doActionFor(l,'open',dest_container=l) ## l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) security.declarePublic('makeStudentLevel') ###( def makeStudentLevel(self,student_id): "create the StudyLevel for a returning Student" #import pdb;pdb.set_trace() logger = logging.getLogger('WAeUPTool.makeStudentLevel') students_folder = self.portal_url.getPortalObject().campus.students res = self.students_catalog(id=student_id) if res: st = res[0] course = st.course matric_no = st.matric_no level = st.level res = self.results_import(matric_no = matric_no) if res: results = res logger.info('%s creating Level %s' % (student_id,level)) # # Level # student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) studycourse = getattr(student_obj,"study_course",None) self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse) l = getattr(studycourse,level,None) if l is None: studycourse.invokeFactory('StudentStudyLevel', level) l = getattr(studycourse, level) self.portal_workflow.doActionFor(l,'open',dest_container=l) l.getContent().edit(mapping={'Title': "Level %s" % level}) ###) security.declarePublic('getAccommodationInfo') ###( def getAccommodationInfo(self,bed): """return Accommodation Info""" info = {} hall,block,room,letter = bed.split('_') res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall) if res and len(res) == 1: hall_brain = res[0] hall_doc = hall_brain.getObject().getContent() else: return info info['hall_title'] = hall_brain.Title info['maintenance_code'] = hall_doc.maintenance_code res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch") batch_doc = None for brain in res: if brain.id.startswith(info['maintenance_code']): batch_doc = brain.getObject().getContent() break if batch_doc is None: info['maintenance_fee'] = None else: info['maintenance_fee'] = batch_doc.cost return info ###) security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###( def deleteAllCourses(self,department="All"): ''' delete the courses''' pm = self.portal_membership member = pm.getAuthenticatedMember() if str(member) not in ("henrik","joachim"): return "not possible" if department == "All": res = self.portal_catalog({'meta_type': 'Department'}) if len(res) < 1: return "No Departments found" deleted = [] for dep in res: cf = dep.getObject().courses if cf: cf.manage_delObjects(ids=cf.objectIds()) deleted.append("deleted Courses in %s" % dep.getId) return "\r".join(deleted) ###) security.declareProtected(ModifyPortalContent,'getLogfileLines') ###( def getLogfileLines(self,filename="event.log",numlines=20): """Get last NUMLINES lines of logfile FILENAME. Return last lines' of a file in the instances logfile directory as a list. The number of returned lines equals `numlines' or less. If less than `numlines' lines are available, the whole file ist returned. If the file can not be opened or some other error occurs, empty list is returend. """ result = [] lines_hit = 0 # We only handle files in instances' log directory... logpath = os.path.join(i_home, "log") filename = str(os.path.abspath( os.path.join( logpath, filename ))) if not filename.startswith( logpath ): # Attempt to access file outside log-dir... return [] try: fd = file( filename, "rb" ) except IOError: return [] if not fd: return [] if os.linesep == None: linesep = '\n' else: linesep = os.linesep # Try to find 'numlines' times a lineseparator, searching from end # and moving to the beginning of file... fd.seek( 0, 2) # Move to end of file... while lines_hit < numlines: if fd.read(1) == linesep[-1]: # This moves filedescriptor # one step forward... lines_hit += 1 try: fd.seek( -2, 1) # Go two bytes back from current pos... except IOError: # We cannot go back two bytes. Maybe the file is too small... break fd.seek(2,1) # Read all lines from current position... result = fd.readlines() # Remove line endings... result = [x.strip() for x in result] fd.close() return result ###) security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###( def getCallbacksFromLog(self,filename): """fix Online Payment Transactions from Z2.log entries""" import transaction import random from cgi import parse_qs from urlparse import urlparse #from pdb import set_trace wftool = self.portal_workflow current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") students_folder = self.portal_url.getPortalObject().campus.students s = r'(?P\S+) - (?P\S+) \[' s += r'(?P.*)\] "(?P.*)" (?P\d+ \d+) "' s += r'(?P.*)" "(?P.*)"' data = re.compile(s) start = True tr_count = 1 total = 0 #name = 'pume_results' #name = 'epaymentsuccessful_z2log2' name = filename no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.getFailedTransactions') try: transactions = open("%s/import/%s" % (i_home,name),"rb").readlines() except: logger.error('Error reading %s' % name) return tas = [] for line in transactions: dict = {} items = data.search(line) dict['idict'] = idict = items.groupdict() #print idict #from pdb import set_trace;set_trace() urlparsed = urlparse(idict['get'][4:]) #print urlparsed path = urlparsed[2].split('/') dict['student_id'] = student_id = path[8] dict['payment_id'] = payment_id = path[10] dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4]) tas.append(dict) tr_count += 1 return tas ###) security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###( def importOnlinePaymentTransactions(self): """load Online Payment Transactions from CSV values""" import transaction import random #from pdb import set_trace current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") opt = self.online_payments_import students_folder = self.portal_url.getPortalObject().campus.students start = True tr_count = 1 total = 0 #name = 'pume_results' name = 'OnlineTransactions' no_import = [] imported = [] logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions') try: transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) except: logger.error('Error reading %s.csv' % name) return for pay_transaction in transactions: if start: start = False logger.info('Start loading from %s.csv' % name) s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()]) no_import.append('%s,"Error"' % s) format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()]) format_error = format + ',"%(Error)s"' data = {} # format of the first file sent by Tayo #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date']) #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = pay_transaction['Resp Code'] #data['amount'] = amount = pay_transaction['Amount'] # format of the second file sent by Tayo #data['datetime'] = date = 0 #data['student_id'] = student_id = pay_transaction['Payer ID'] #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] #data['response_code'] = response_code = '00' #data['amount'] = amount = pay_transaction['Amount'] # format of the third file sent by Kehinde data['datetime'] = date = 0 data['student_id'] = student_id = pay_transaction['customer_id'] data['order_id'] = order_id = pay_transaction['merchant_reference'] data['response_code'] = response_code = '00' data['amount'] = amount = pay_transaction['Amount'] dup = False if response_code == "12": continue try: opt.addRecord(**data) except ValueError: dup = True #from pdb import set_trace;set_trace() if dup: if response_code == "00": try: opt.modifyRecord(**data) except: logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data) continue else: pay_transaction['Error'] = "Duplicate order_id" no_import.append( format_error % pay_transaction) logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data) continue tr_count += 1 if tr_count > 1000: if len(no_import) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import) + '\n') no_import = [] em = '%d transactions committed\n' % (tr_count) transaction.commit() regs = [] logger.info(em) total += tr_count tr_count = 0 open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( '\n'.join(no_import)) return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) ###) # all workflow transitions refer to students in state returning student_field2types = { ###( 'StudentApplication': {'id': 'application', 'title': 'Application Data', 'wf_transition_returning': 'close', 'wf_transition_admitted': 'remain', 'fields': ('jamb_reg_no', 'entry_mode', 'entry_session', 'jamb_score', 'app_email', ) }, #'StudentPume': # {'id': 'pume', # 'title': 'Pume Data', # 'wf_transition_returning': 'close', # 'wf_transition_admitted': 'close', # 'fields': # ('pume_score', # ) # }, 'StudentClearance': {'id': 'clearance', 'title': 'Clearance Data', 'wf_transition_returning': 'close', 'wf_transition_admitted': 'remain', 'fields': ('matric_no', 'nationality', 'lga', 'birthday', ) }, 'StudentPersonal': {'id': 'personal', 'title': 'Personal Data', 'wf_transition_returning': 'open', 'wf_transition_admitted': 'remain', 'fields': ('firstname', 'middlename', 'lastname', 'sex', 'email', 'phone', 'perm_address', ) }, 'StudentStudyCourse': {'id': 'study_course', 'title': 'Study Course', 'wf_transition_returning': 'open', 'wf_transition_admitted': 'remain', 'fields': ('study_course', 'current_level', 'current_session', 'current_mode', 'current_verdict', ) }, 'PaymentsFolder': {'id': 'payments', 'title': 'Payments', 'wf_transition_returning': 'open', 'wf_transition_admitted': 'open', 'fields': () }, } ###) security.declareProtected(ModifyPortalContent,'importStudent') ###( def importStudent(self,mapping): "create a students record due import" logger = logging.getLogger('WAeUPTool.importStudent') students_folder = self.portal_url.getPortalObject().campus.students jamb_reg_no = mapping.get('jamb_reg_no',None) if jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if res: return '',"jamb_reg_no exists" matric_no = mapping.get('matric_no',None) if matric_no: res = self.students_catalog(matric_no = matric_no) if res: return '',"matric_no exists" sid = self.waeup_tool.generateStudentId('?') students_folder.invokeFactory('Student', sid) student_obj = getattr(students_folder,sid) f2t = self.student_field2types d = {} d['jamb_sex'] = 'M' if mapping.get('sex'): d['jamb_sex'] = 'F' for pt in f2t.keys(): student_obj.invokeFactory(pt,f2t[pt]['id']) sub_obj = getattr(student_obj,f2t[pt]['id']) sub_doc = sub_obj.getContent() d['Title'] = f2t[pt]['title'] for field in f2t[pt]['fields']: d[field] = mapping.get(field,'') if pt == "StudyCourse": for von,zu in (('entry_mode','current_mode'), ('entry_session','current_session')): if mapping.get(zu,None) is None and mapping.get(von,None) is not None: d[zu] = mapping[von] sub_doc.edit(mapping = d) transition = mapping.get('reg_transition','admitted') if transition not in ('admitted','returning'): transition = 'admitted' #import pdb;pdb.set_trace() new_state = f2t[pt]['wf_transition_%(transition)s' % vars()] if new_state != "remain": self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) wfaction = 'admit' self.portal_workflow.doActionFor(student_obj,wfaction) student_obj.manage_setLocalRoles(sid, ['Owner',]) return sid,'' ###) security.declareProtected(ModifyPortalContent,'importEditStudent') ###( def importEditStudent(self,mapping): "edit a students record due import" logger = logging.getLogger('WAeUPTool.importEditStudent') students_folder = self.portal_url.getPortalObject().campus.students sid = mapping.get('id',None) jamb_reg_no = mapping.get('jamb_reg_no',None) matric_no = mapping.get('matric_no',None) editable_keys = mapping.keys() if sid: res = self.students_catalog(id = sid) if not res: return '',"no student with id %s" % sid elif matric_no and res[0].matric_no and\ matric_no != res[0].matric_no: return '%s' % res[0].id ,"student has no matric_no %s" % matric_no elif jamb_reg_no and res[0].jamb_reg_no and\ jamb_reg_no != res[0].jamb_reg_no: return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no elif jamb_reg_no: res = self.students_catalog(jamb_reg_no = jamb_reg_no) if not res: return '',"no student with jamb_reg_no %s" % jamb_reg_no editable_keys.remove('jamb_reg_no') elif matric_no: res = self.students_catalog(matric_no = matric_no) if not res: return '',"no student with matric_no %s" % matric_no editable_keys.remove('matric_no') # included only to change wf state from admitted to returning if res[0].review_state != 'admitted': return '%s' % res[0].id ,"student is not in state admitted" # end inclusion sid = res[0].id student_obj = getattr(students_folder,sid) f2t = self.student_field2types d = {} #import pdb;pdb.set_trace() any_change = False for pt in f2t.keys(): if pt == "student_application": d['jamb_sex'] = 'M' if mapping.get('sex'): d['jamb_sex'] = 'F' intersect = set(f2t[pt]['fields']).intersection(set(editable_keys)) if intersect: sub_obj = getattr(student_obj,f2t[pt]['id'],None) if sub_obj is None: try: student_obj.invokeFactory(pt,f2t[pt]['id']) except: continue sub_obj = getattr(student_obj,f2t[pt]['id']) d['Title'] = f2t[pt]['title'] sub_doc = sub_obj.getContent() for field in intersect: changed = False if getattr(sub_doc,field) != mapping.get(field,''): any_change = True changed = True d[field] = mapping.get(field,'') if changed: sub_doc.edit(mapping = d) # included only to change wf state from admitted to returning if res[0].review_state == 'admitted': new_state = f2t[pt]['wf_transition_returning'] sub_obj = getattr(student_obj,f2t[pt]['id'],None) if sub_obj and new_state != "remain": try: self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) except: #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id)) pass if res[0].review_state == 'admitted': wfaction = 'return' try: self.portal_workflow.doActionFor(student_obj,wfaction) logger.info('%s, wf state changed' % sid) any_change = True except: logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state)) pass # end inclusion if any_change: return sid,'' else: return sid,'not modified' ###) security.declareProtected(ModifyPortalContent,"importData")###( def importData(self,filename,name,edit=False): """load data from CSV values""" import transaction import random pm = self.portal_membership member = pm.getAuthenticatedMember() logger = logging.getLogger('WAeUPTool.importData') current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") students_folder = self.portal_url.getPortalObject().campus.students start = True tr_count = 1 total_imported = 0 total_not_imported = 0 total = 0 iname = "import_%s" % name stool = getToolByName(self, 'portal_schemas') ltool = getToolByName(self, 'portal_layouts') schema = stool._getOb(iname) if schema is None: em = 'No such schema %s' % iname logger.error('No such schema %s' % iname) return em layout = ltool._getOb(iname) if layout is None: em = 'No such layout %s' % iname logger.error(em) return em validators = {} for widget in layout.keys(): validators[widget] = layout[widget].validate if edit: importer_name = "importEdit%s" % name.capitalize() else: importer_name = "import%s" % name.capitalize() importer = getattr(self, '%s' % importer_name,None) if importer is None: em = 'No importer function %s' % importer_name logger.error(em) return em not_imported = [] imported = [] try: items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb")) except: em = 'Error reading %s.csv' % filename logger.error(em) return em for item in items: if start: start = False adapters = [MappingStorageAdapter(schema, item)] dm = DataModel(item, adapters,context=self) logger.info('%s starts import from %s.csv' % (member,filename)) import_keys = [k for k in item.keys() if not k.startswith('ignore')] diff2schema = set(import_keys).difference(set(schema.keys())) diff2layout = set(import_keys).difference(set(layout.keys())) if diff2schema: em = "not ignorable key(s) %s found in heading" % diff2schema return em s = ','.join(['"%s"' % fn for fn in import_keys]) open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n') s = '"id",' + s open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n') format = ','.join(['"%%(%s)s"' % fn for fn in import_keys]) format_error = format + ',"%(Error)s"' format = '"%(id)s",'+ format ds = DataStructure(data=item,datamodel=dm) error_string = "" for k in import_keys: if not validators[k](ds): error_string += " %s : %s" % (k,ds.getError(k)) if not error_string: item.update(dm) item['id'],error = importer(item) if error: error_string += error if error_string: item['Error'] = error_string not_imported.append(format_error % item) total_not_imported += 1 else: em = format % item imported.append(em) logger.info("%(total_imported)d of %(total)d %(em)s" % vars()) tr_count += 1 total_imported += 1 total += 1 if total_imported and not total_imported % 100: transaction.commit() if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported) + '\n') not_imported = [] if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported) + '\n') imported = [] em = '%d transactions committed\n' % (tr_count) regs = [] logger.info(em) tr_count = 0 if len(imported) > 0: open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(imported)) if len(not_imported) > 0: open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write( '\n'.join(not_imported)) em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total) logger.info(em) return em ###) InitializeClass(WAeUPTool)