source: WAeUP_SRP/branches/joachim-azax-branch/WAeUPTool.py @ 11107

Last change on this file since 11107 was 1988, checked in by joachim, 17 years ago

merged r1979:r1987 from trunk

  • Property svn:keywords set to Id
File size: 43.3 KB
Line 
13#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 1988 2007-07-05 11:14:12Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29
30from Products.CMFCore.utils import getToolByName
31from Products.CPSSchemas.DataStructure import DataStructure
32from Products.CPSSchemas.DataModel import DataModel
33from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34from Products.CMFCore.ActionProviderBase import ActionProviderBase
35from Products.CMFCore.permissions import View
36from Products.ZCatalog.ZCatalog import ZCatalog
37from Products.CMFCore.permissions import ModifyPortalContent
38from Products.CMFCore.permissions import ManagePortal
39from Products.CMFCore.utils import UniqueObject
40from Products.CMFCore.URLTool import URLTool
41from Products.CMFCore.utils import getToolByName
42from Students import makeCertificateCode
43from Globals import package_home,INSTANCE_HOME
44p_home = package_home(globals())
45i_home = INSTANCE_HOME
46import DateTime
47import logging
48import transaction
49import csv,re,os
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52def getObject(object,name):
53    if object.hasObject(name):
54        return getattr(object,name)
55    return None
56
57class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58    """WAeUP tool"""
59
60    id = 'waeup_tool'
61    meta_type = 'WAeUP Tool'
62    _actions = ()
63
64    security = ClassSecurityInfo()
65    security.declareObjectProtected(View)
66
67    manage_options = ( ActionProviderBase.manage_options
68                     + SimpleItem.manage_options
69                     )
70
71    def rwrite(self,s): ###(
72        response = self.REQUEST.RESPONSE
73        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74        response.write("%s<br />\r\n" % s)
75    ###)
76
77    def sleep(self,secs): ###(
78        "sleep"
79        import time
80        time.sleep(secs)
81        return
82
83###)
84
85    security.declareProtected(ModifyPortalContent,'openLog') ###(
86    def openLog(self,name):
87        """open a log file"""
88        version = 1
89        path = "%s/log/%s_%d.log" % (i_home,name,version)
90        while os.path.exists(path):
91            version += 1
92            path = "%s/log/%s_%d.log" % (i_home,name,version)
93        log = open(path,"w")
94        return log
95
96###)
97
98    security.declareProtected(ModifyPortalContent,'writeLog') ###(
99    def writeLog(self,logfile,s):
100        """write to the log file"""
101        logfile.write(s)
102
103###)
104
105    def generateStudentId(self,letter): ###(
106        import random
107        r = random
108        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109        if letter == '?':
110            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111        sid = "%c%d" % (letter,r.randint(99999,1000000))
112        students = self.portal_url.getPortalObject().campus.students
113##        while hasattr(students, sid):
114##            sid = "%c%d" % (letter,r.randint(99999,1000000))
115        while self.students_catalog(id = sid):
116            sid = "%c%d" % (letter,r.randint(99999,1000000))
117        return sid
118    ###)
119
120    def generatePassword(self,s=None): ###(
121        import random
122        r = random
123        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124        if s is None:
125            s = 'abcdefghklmnpqrstuvwxy23456789'
126        pw = ''
127        while len(pw) < 6:
128            pw += r.choice(s)
129        return pw
130    ###)
131
132    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133    def removeDeletedDocIds(self, max=1000):
134        """
135        remove deleted docids from repository commit after max
136        """
137        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138        repository = getToolByName(self, 'portal_repository')
139        pxtool = getToolByName(self, 'portal_proxies')
140        pxtool_infos = pxtool.getRevisionsUsed()
141
142        nb_revs = 0
143        docids_d = {} # all docids
144        unused_docids_d = {} # all docids that are unused
145        ids_unused_revs_docids = [] # ids for revs of unused docids
146        ids_unused_revs = [] # ids for unused revs
147        total = 0
148        idlist = repository.objectIds()
149        for id in idlist:
150            docid, rev = repository._splitId(id)
151            if docid is None:
152                logger.info("invalid doc_id %s" % docid)
153                continue
154            nb_revs += 1
155            docids_d[docid] = None
156            if not pxtool_infos.has_key(docid):
157                unused_docids_d[docid] = None
158                ids_unused_revs_docids.append(id)
159                ids_unused_revs.append(id)
160            elif not pxtool_infos[docid].has_key(rev):
161                ids_unused_revs.append(id)
162            if len(ids_unused_revs) >= max:
163                repository.manage_delObjects(ids_unused_revs)
164                #import pdb;pdb.set_trace()
165                transaction.commit()
166                total += max
167                logger.info('removed %d total %d unused docids ' % (max,total))
168        anz = len(ids_unused_revs)
169        if anz > 0:
170            repository.manage_delObjects(ids_unused_revs)
171            transaction.commit()
172            total += anz
173            logger.info('finished removing %d unused docids ' % (total))
174
175
176###)
177
178    security.declareProtected(ModifyPortalContent,'getCredential') ###(
179    def getCredential(self,student_id):
180        "return a student password"
181        student_entry = getattr(self.portal_directories.students,student_id,None)
182        if student_entry is None:
183            return None
184        return getattr(student_entry,"password","not set")
185    ###)
186
187    security.declarePublic('checkPassword') ###(
188    def checkPassword(self,student_id,password):
189        "return a student password"
190        student_entry = getattr(self.portal_directories.students,student_id,None)
191        if student_entry is None:
192            return False
193        return getattr(student_entry,"password","not set") == password
194    ###)
195
196    security.declarePublic('editPassword') ###(
197    def editPassword(self,student_id,password):
198        "edit a student password"
199        student_entry = getattr(self.portal_directories.students,student_id,None)
200        if student_entry is None:
201            return
202        setattr(student_entry,'password',password)
203    ###)
204
205    security.declareProtected(View,'doCommit') ###(
206    def doCommit(self,logger=None):
207        "commit some transactions"
208        transaction.commit()
209    ###)
210
211    security.declarePublic('loadStudentFoto') ###(
212    def loadStudentFoto(self,student,filename,folder):
213        "return a student passport picture"
214        app = student.application
215        app_doc = app.getContent()
216        #clear = student.clearance
217        #clear_doc = clear.getContent()
218        #matric_no = clear_doc.matric_no.upper()
219        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
220        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
221        #import pdb;pdb.set_trace()
222        if os.path.exists(picture1):
223            file = open(picture1)
224        elif os.path.exists(picture2):
225            file = open(picture2)
226        else:
227            return "passport picture not found %s" % picture1
228        reopened = False
229        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
230            self.portal_workflow.doActionFor(app,'open')
231            reopened = True
232        outfile = file.read()
233        app_doc.manage_addFile('passport',
234                               file=outfile,
235                               title="%s.jpg" % filename)
236        if reopened:
237            self.portal_workflow.doActionFor(app,'close')
238        return "successfully loaded passport picture"
239    ###)
240
241    security.declareProtected(ModifyPortalContent,'createOne') ###(
242    def createOne(self,students_folder,student_brain,letter,commit=False):
243        sid = self.waeup_tool.generateStudentId(letter)
244        students_folder.invokeFactory('Student', sid)
245        student = getattr(students_folder,sid)
246        self.portal_workflow.doActionFor(student,'return')
247        student.manage_setLocalRoles(sid, ['Owner',])
248        matric_no = student_brain.matric_no
249        jamb_reg_no = student_brain.Entryregno
250        self.students_catalog.addRecord(id = sid,
251                                           matric_no = matric_no,
252                                           jamb_reg_no = jamb_reg_no,
253                                           sex = student_brain.Sex == "F",
254                                           name = "%s %s %s" % (student_brain.Firstname,
255                                                                student_brain.Middlename,
256                                                                student_brain.Lastname)
257                                        )
258        if commit:
259            transaction.commit()
260        return sid,jamb_reg_no
261    ###)
262
263    security.declareProtected(ModifyPortalContent,'addStudent') ###(
264    def addStudent(self,dict):
265        students_folder = self.portal_url.getPortalObject().campus.students
266        sid = self.waeup_tool.generateStudentId('?')
267        students_folder.invokeFactory('Student', sid)
268        student_obj = getattr(students_folder,sid)
269        f2t = self.student_field2types
270        #from pdb import set_trace; set_trace()
271        d = {}
272        d['jamb_sex']  = 'M'
273        if dict.get('sex'):
274            d['jamb_sex']  = 'F'
275
276        entry_session = dict.get('entry_session')
277        if entry_session == self.getSessionId()[-2:]:
278            wfaction = 'admit'
279            wft = 'wf_transition_admitted'
280            password = None
281        else:
282            wfaction = 'return'
283            wft = 'wf_transition_returning'
284            password = self.generatePassword()
285            self.makeStudentMember(sid,password)
286
287        for pt in f2t.keys():
288            student_obj.invokeFactory(pt,f2t[pt]['id'])
289            sub_obj = getattr(student_obj,f2t[pt]['id'])
290            sub_doc = sub_obj.getContent()
291            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
292            d['Title'] = f2t[pt]['title']
293            for field in f2t[pt]['fields']:
294                d[field] = dict.get(field,'')
295            sub_doc.edit(mapping = d)
296            new_state = f2t[pt][wft]
297            if new_state != "remain":
298                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
299        self.portal_workflow.doActionFor(student_obj,wfaction)
300        student_obj.manage_setLocalRoles(sid, ['Owner',])
301        return sid,password
302    ###)
303
304    security.declarePublic('getCertificateBrain') ###(
305    def getCertificateBrain(self,cert_id):
306        "do it"
307        res = ZCatalog.searchResults(self.portal_catalog_real,
308                                {'portal_type':"Certificate",
309                                      'id': cert_id})
310        if res:
311            return res[0]
312        return None
313    ###)
314
315    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
316    def get_csv_filenames(self):
317        "do it"
318        files = [file for file in os.listdir("%s/import/" % (i_home))
319                 if file.endswith('.csv') and file.find('imported') == -1]
320        return files
321    ###)
322
323    security.declarePublic('findStudentByMatricelNo') ###(
324    def findStudentByMatricelNo(self,matric_no):
325        "do it"
326        res = ZCatalog.searchResults(self.portal_catalog_real,
327                                {'portal_type':"StudentClearance",
328                                 'SearchableText': matric_no})
329        if res:
330            return res[0]
331        return None
332    ###)
333
334    security.declarePublic('makeStudentMember') ###(
335    def makeStudentMember(self,sid,password='uNsEt'):
336        """make the student a member"""
337        membership = self.portal_membership
338        membership.addMember(sid,
339                             password ,
340                             roles=('Member',
341                                     'Student',
342                                     ),
343                             domains='',
344                             properties = {'memberareaCreationFlag': False,
345                                           'homeless': True},)
346        member = membership.getMemberById(sid)
347        self.portal_registration.afterAdd(member, sid, password, None)
348        #self.manage_setLocalRoles(sid, ['Owner',])
349    ###)
350
351    security.declarePublic('makeStudentData') ###(
352    def makeStudentData(self,student_id,email=None,phone_nr=None):
353        "create Datastructure for a returning Student"
354        #import pdb;pdb.set_trace()
355        logger = logging.getLogger('WAeUPTool.makeStudentData')
356        students_folder = self.portal_url.getPortalObject().campus.students
357        #res = self.students_catalog(id=student_id)
358        #if res:
359        #    st = res[0]
360        #res = self.returning_import(matric_no = st.matric_no)
361        res = self.returning_import(id = student_id)
362        if res:
363            student = res[0]
364        else:
365            logger.info('Id %s not found in returning_import' % student_id)
366            return
367        logger.info('%s creates data structure' % student_id)
368        s_results = self.results_import(matric_no = student.matric_no)
369        if s_results:
370            lnr = self.getLevelFromResultsCosCode(s_results)
371            level = "%d00" % lnr
372            verdict,eligible = self.getVerdict(s_results[0].Verdict)
373            if eligible:
374                level = "%d00" % (lnr + 1)
375        else:
376            logger.info('matric_no %s not found in results_import' % student.matric_no)
377            level = 0
378            verdict = 'N/A'
379        #student should not be allowed to perform this transition
380        #wftool = self.portal_workflow
381        #wftool.doActionFor(student,'return')
382        certcode_org = student.Coursemajorcode
383        certcode = makeCertificateCode(certcode_org)
384        certificate_brain = self.getCertificateBrain(certcode)
385        if not certificate_brain:
386            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
387            logger.info(em)
388        matric_no = student.matric_no
389        sid = student_id
390        student_obj = getattr(students_folder,sid)
391        student_obj.invokeFactory('StudentApplication','application')
392        application = student_obj.application
393        self.portal_workflow.doActionFor(application,'open',dest_container=application)
394        da = {'Title': 'Application Data'}
395        student_obj.invokeFactory('StudentPersonal','personal')
396        da['jamb_reg_no'] = student.Entryregno
397        em = self.getEntryMode(student.Entryregno)
398##        em = student.Mode_of_Entry
399##        if em in ('DIRECT', 'DIRECT ENTRY',):
400##            em = 'DE'
401##        elif em in ('U.M.E', 'UNE',):
402##            em = 'UME'
403##        elif not em:
404##            em = "unknown"
405        da['entry_mode'] = em
406        personal = student_obj.personal
407        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
408        dp = {'Title': 'Personal Data'}
409        student_obj.invokeFactory('StudentClearance','clearance')
410        clearance = student_obj.clearance
411        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
412        dc = {'Title': 'Clearance/Eligibility Record'}
413        dc['matric_no'] = matric_no
414        state = student.State
415        lga = student.LGA
416        if state and lga:
417            lga =  state + ' / ' + lga
418        else:
419            lga = "None"
420        da['jamb_lga'] = dc['lga'] = lga
421        da['app_email'] = dp['email'] = email
422        da['app_mobile'] = dp['phone'] = phone_nr
423        dp['firstname'] = student.Firstname
424        dp['middlename'] = student.Middlename
425        dp['lastname'] = student.Lastname
426        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
427        da['jamb_sex'] = student.Sex
428        dp['sex'] = student.Sex == 'F'
429        dp['perm_address'] = student.Permanent_Address
430        application.getContent().edit(mapping=da)
431        self.portal_workflow.doActionFor(application,'close',dest_container=application)
432        personal.getContent().edit(mapping=dp)
433        clearance.getContent().edit(mapping=dc)
434        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
435##        catd = {}
436##        catd['id'] = sid
437##        catd['entry_mode']= da['entry_mode']
438##        catd['matric_no'] = matric_no
439##        catd['jamb_reg_no'] = da['jamb_reg_no']
440##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
441##        catd['sex'] = dp['sex']
442##        catd['level'] = level
443##        catd['verdict'] = verdict
444##        if certificate_brain:
445##            cpath = certificate_brain.getPath().split('/')
446##            catd['faculty'] = cpath[-4]
447##            catd['department'] = cpath[-3]
448##            catd['course'] = certcode
449##        self.students_catalog.modifyRecord(**catd)
450        #
451        # Study Course
452        #
453        student_obj.invokeFactory('StudentStudyCourse','study_course')
454        studycourse = student_obj.study_course
455        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
456        dsc = {}
457        dsc['study_course'] = certcode
458        dsc['current_level'] = level
459        dsc['current_verdict'] = verdict
460        dsc['current_mode'] = em
461        dsc['current_session'] = '06'
462        studycourse.getContent().edit(mapping=dsc)
463        #
464        # Level
465        #
466##        l = getattr(studycourse,level,None)
467##        if l is None:
468##            studycourse.invokeFactory('StudentStudyLevel', level)
469##            l = getattr(studycourse, level)
470##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
471##            l.getContent().edit(mapping={'Title': "Level %s" % level})
472###)
473
474    security.declarePublic('makeStudentLevel') ###(
475    def makeStudentLevel(self,student_id):
476        "create the StudyLevel for a returning Student"
477        #import pdb;pdb.set_trace()
478        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
479        students_folder = self.portal_url.getPortalObject().campus.students
480        res = self.students_catalog(id=student_id)
481        if res:
482            st = res[0]
483        course = st.course
484        matric_no = st.matric_no
485        level = st.level
486        res = self.results_import(matric_no = matric_no)
487        if res:
488            results = res
489        logger.info('%s creating Level %s' % (student_id,level))
490        #
491        # Level
492        #
493        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
494        studycourse = getattr(student_obj,"study_course",None)
495        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
496        l = getattr(studycourse,level,None)
497        if l is None:
498            studycourse.invokeFactory('StudentStudyLevel', level)
499            l = getattr(studycourse, level)
500            self.portal_workflow.doActionFor(l,'open',dest_container=l)
501            l.getContent().edit(mapping={'Title': "Level %s" % level})
502        ###)
503
504    security.declarePublic('getAccommodationInfo') ###(
505    def getAccommodationInfo(self,bed):
506        """return Accommodation Info"""
507        info = {}
508        hall,block,room,letter = bed.split('_')
509        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
510        if res and len(res) == 1:
511            hall_brain = res[0]
512            hall_doc = hall_brain.getObject().getContent()
513        else:
514            return info
515        info['hall_title'] = hall_brain.Title
516        info['maintenance_code'] = hall_doc.maintenance_code
517        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
518        batch_doc = None
519        for brain in res:
520            if brain.id.startswith(info['maintenance_code']):
521                batch_doc = brain.getObject().getContent()
522                break
523        if batch_doc is None:
524            info['maintenance_fee'] = None
525        else:
526            info['maintenance_fee'] = batch_doc.cost
527        return info
528    ###)
529
530    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
531    def deleteAllCourses(self,department="All"):
532        ''' delete the courses'''
533        pm = self.portal_membership
534        member = pm.getAuthenticatedMember()
535
536        if str(member) not in ("henrik","joachim"):
537            return "not possible"
538        if department == "All":
539            res = self.portal_catalog({'meta_type': 'Department'})
540        if len(res) < 1:
541            return "No Departments found"
542
543        deleted = []
544        for dep in res:
545            cf = dep.getObject().courses
546            if cf:
547                cf.manage_delObjects(ids=cf.objectIds())
548                deleted.append("deleted Courses in %s" % dep.getId)
549        return "\r".join(deleted)
550    ###)
551
552    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
553    def getLogfileLines(self,filename="event.log",numlines=20):
554        """Get last NUMLINES lines of logfile FILENAME.
555
556        Return last lines' of a file in the instances logfile directory as
557        a list. The number of returned lines equals `numlines' or less. If
558        less than `numlines' lines are available, the whole file ist
559        returned. If the file can not be opened or some other error
560        occurs, empty list is returend.
561        """
562        result = []
563        lines_hit = 0
564
565        # We only handle files in instances' log directory...
566        logpath = os.path.join(i_home, "log")
567        filename = str(os.path.abspath( os.path.join( logpath, filename )))
568        if not filename.startswith( logpath ):
569            # Attempt to access file outside log-dir...
570            return []
571
572        try:
573            fd = file( filename, "rb" )
574        except IOError:
575            return []
576        if not fd:
577            return []
578
579        if os.linesep == None:
580            linesep = '\n'
581        else:
582            linesep = os.linesep
583
584        # Try to find 'numlines' times a lineseparator, searching from end
585        # and moving to the beginning of file...
586        fd.seek( 0, 2) # Move to end of file...
587        while lines_hit < numlines:
588            if fd.read(1) == linesep[-1]: # This moves filedescriptor
589                                          # one step forward...
590                lines_hit += 1
591            try:
592                fd.seek( -2, 1) # Go two bytes back from current pos...
593            except IOError:
594                # We cannot go back two bytes. Maybe the file is too small...
595                break
596        fd.seek(2,1)
597
598        # Read all lines from current position...
599        result = fd.readlines()
600        # Remove line endings...
601        result = [x.strip() for x in result]
602        fd.close()
603        return result
604    ###)
605
606    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
607    def getCallbacksFromLog(self,filename):
608        """fix Online Payment Transactions from Z2.log entries"""
609        import transaction
610        import random
611        from cgi import parse_qs
612        from urlparse import urlparse
613        #from pdb import set_trace
614        wftool = self.portal_workflow
615        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
616        students_folder = self.portal_url.getPortalObject().campus.students
617        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
618        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
619        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
620        data = re.compile(s)
621        start = True
622        tr_count = 1
623        total = 0
624        #name = 'pume_results'
625        #name = 'epaymentsuccessful_z2log2'
626        name = filename
627        no_import = []
628        imported = []
629        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
630        try:
631            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
632        except:
633            logger.error('Error reading %s' % name)
634            return
635        tas = []
636        for line in transactions:
637            dict = {}
638            items = data.search(line)
639            dict['idict'] = idict = items.groupdict()
640            #print idict
641            #from pdb import set_trace;set_trace()
642            urlparsed = urlparse(idict['get'][4:])
643            #print urlparsed
644            path = urlparsed[2].split('/')
645            dict['student_id'] = student_id = path[8]
646            dict['payment_id'] = payment_id = path[10]
647            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
648            tas.append(dict)
649            tr_count += 1
650        return tas
651    ###)
652
653    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
654    def importOnlinePaymentTransactions(self):
655        """load Online Payment Transactions from CSV values"""
656        import transaction
657        import random
658        #from pdb import set_trace
659        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
660        opt = self.online_payments_import
661        students_folder = self.portal_url.getPortalObject().campus.students
662        start = True
663        tr_count = 1
664        total = 0
665        #name = 'pume_results'
666        name = 'OnlineTransactions'
667        no_import = []
668        imported = []
669        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
670        try:
671            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
672        except:
673            logger.error('Error reading %s.csv' % name)
674            return
675        for pay_transaction in transactions:
676            if start:
677                start = False
678                logger.info('Start loading from %s.csv' % name)
679                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
680                no_import.append('%s,"Error"' % s)
681                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
682                format_error = format + ',"%(Error)s"'
683            data = {}
684
685            # format of the first file sent by Tayo
686            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
687            #data['student_id'] = student_id = pay_transaction['Payer ID']
688            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
689            #data['response_code'] = response_code = pay_transaction['Resp Code']
690            #data['amount'] = amount = pay_transaction['Amount']
691
692            # format of the second file sent by Tayo
693            #data['datetime'] = date = 0
694            #data['student_id'] = student_id = pay_transaction['Payer ID']
695            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
696            #data['response_code'] = response_code = '00'
697            #data['amount'] = amount = pay_transaction['Amount']
698
699            # format of the third file sent by Kehinde
700            data['datetime'] = date = 0
701            data['student_id'] = student_id = pay_transaction['customer_id']
702            data['order_id'] = order_id = pay_transaction['merchant_reference']
703            data['response_code'] = response_code = '00'
704            data['amount'] = amount = pay_transaction['Amount']
705
706            dup = False
707            if response_code == "12":
708                continue
709            try:
710                opt.addRecord(**data)
711            except ValueError:
712                dup = True
713            #from pdb import set_trace;set_trace()
714            if dup:
715                if response_code == "00":
716                    try:
717                        opt.modifyRecord(**data)
718                    except:
719                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
720                        continue
721                else:
722                    pay_transaction['Error'] = "Duplicate order_id"
723                    no_import.append( format_error % pay_transaction)
724                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
725                    continue
726            tr_count += 1
727            if tr_count > 1000:
728                if len(no_import) > 0:
729                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
730                             '\n'.join(no_import) + '\n')
731                    no_import = []
732                em = '%d transactions committed\n' % (tr_count)
733                transaction.commit()
734                regs = []
735                logger.info(em)
736                total += tr_count
737                tr_count = 0
738        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
739                                                '\n'.join(no_import))
740        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
741    ###)
742
743
744    # all workflow transitions refer to students in state returning
745    student_field2types = {   ###(
746                      'StudentApplication':
747                          {'id': 'application',
748                           'title': 'Application Data',
749                           'wf_transition_returning': 'close',
750                           'wf_transition_admitted': 'remain',
751                           'fields':
752                             ('jamb_reg_no',
753                              'entry_mode',
754                              'entry_session',
755                              'jamb_score',
756                              'app_email',
757                              )
758                              },
759                      #'StudentPume':
760                      #    {'id': 'pume',
761                      #     'title': 'Pume Data',
762                      #     'wf_transition_returning': 'close',
763                      #     'wf_transition_admitted': 'close',
764                      #     'fields':
765                      #       ('pume_score',
766                      #        )
767                      #        },
768                      'StudentClearance':
769                          {'id': 'clearance',
770                           'title': 'Clearance Data',
771                           'wf_transition_returning': 'close',
772                           'wf_transition_admitted': 'remain',
773                           'fields':
774                             ('matric_no',
775                              'nationality',
776                              'lga',
777                              'birthday',
778                              )
779                              },
780                         'StudentPersonal':
781                          {'id': 'personal',
782                           'title': 'Personal Data',
783                           'wf_transition_returning': 'open',
784                           'wf_transition_admitted': 'remain',
785                           'fields':
786                             ('firstname',
787                              'middlename',
788                              'lastname',
789                              'sex',
790                              'email',
791                              'phone',
792                              'perm_address',
793                              )
794                              },
795                         'StudentStudyCourse':
796                          {'id': 'study_course',
797                           'title': 'Study Course',
798                           'wf_transition_returning': 'open',
799                           'wf_transition_admitted': 'remain',
800                           'fields':
801                             ('study_course',
802                              'current_level',
803                              'current_session',
804                              'current_mode',
805                              'current_verdict',
806                              )
807                              },
808
809                         'PaymentsFolder':
810                          {'id': 'payments',
811                           'title': 'Payments',
812                           'wf_transition_returning': 'open',
813                           'wf_transition_admitted': 'open',
814                           'fields':
815                             ()
816                              },
817                         }
818    ###)
819
820
821    security.declareProtected(ModifyPortalContent,'importStudent') ###(
822    def importStudent(self,mapping):
823        "create a students record due import"
824        logger = logging.getLogger('WAeUPTool.importStudent')
825        students_folder = self.portal_url.getPortalObject().campus.students
826        jamb_reg_no = mapping.get('jamb_reg_no',None)
827        if jamb_reg_no:
828            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
829            if res:
830                return '',"jamb_reg_no exists"
831        matric_no = mapping.get('matric_no',None)
832        if matric_no:
833            res = self.students_catalog(matric_no = matric_no)
834            if res:
835                return '',"matric_no exists"
836        sid = self.waeup_tool.generateStudentId('?')
837        students_folder.invokeFactory('Student', sid)
838        student_obj = getattr(students_folder,sid)
839        f2t = self.student_field2types
840        d = {}
841        d['jamb_sex']  = 'M'
842        if mapping.get('sex'):
843            d['jamb_sex']  = 'F'
844        for pt in f2t.keys():
845            student_obj.invokeFactory(pt,f2t[pt]['id'])
846            sub_obj = getattr(student_obj,f2t[pt]['id'])
847            sub_doc = sub_obj.getContent()
848            d['Title'] = f2t[pt]['title']
849            for field in f2t[pt]['fields']:
850                d[field] = mapping.get(field,'')
851            if pt == "StudyCourse":
852                for von,zu in (('entry_mode','current_mode'),
853                               ('entry_session','current_session')):
854                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
855                        d[zu] = mapping[von]
856            sub_doc.edit(mapping = d)
857            transition = mapping.get('reg_transition','admitted')
858            if transition not in ('admitted','returning'):
859                transition = 'admitted'
860            #import pdb;pdb.set_trace()
861            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
862            if new_state != "remain":
863                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
864        wfaction = 'admit'
865        self.portal_workflow.doActionFor(student_obj,wfaction)
866        student_obj.manage_setLocalRoles(sid, ['Owner',])
867        return sid,''
868    ###)
869
870    security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
871    def importEditStudent(self,mapping):
872        "edit a students record due import"
873        logger = logging.getLogger('WAeUPTool.importEditStudent')
874        students_folder = self.portal_url.getPortalObject().campus.students
875        sid = mapping.get('id',None)
876        jamb_reg_no = mapping.get('jamb_reg_no',None)
877        matric_no = mapping.get('matric_no',None)
878        editable_keys = mapping.keys()
879        if sid:
880            res = self.students_catalog(id = sid)
881            if not res:
882                return '',"no student with id %s" % sid
883            elif matric_no and res[0].matric_no and\
884              matric_no != res[0].matric_no:
885                return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
886            elif jamb_reg_no and res[0].jamb_reg_no and\
887              jamb_reg_no != res[0].jamb_reg_no:
888                return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
889        elif jamb_reg_no:
890            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
891            if not res:
892                return '',"no student with jamb_reg_no %s" % jamb_reg_no
893            editable_keys.remove('jamb_reg_no')
894        elif matric_no:
895            res = self.students_catalog(matric_no = matric_no)
896            if not res:
897                return '',"no student with matric_no %s" % matric_no
898            editable_keys.remove('matric_no')
899
900        # included only to change wf state from admitted to returning
901        if res[0].review_state != 'admitted':
902            return '%s' % res[0].id ,"student is not in state admitted"
903        # end inclusion   
904
905        sid = res[0].id
906        student_obj = getattr(students_folder,sid)
907        f2t = self.student_field2types
908        d = {}
909        #import pdb;pdb.set_trace()
910        any_change = False
911        for pt in f2t.keys():
912            if pt == "student_application":
913                d['jamb_sex']  = 'M'
914                if mapping.get('sex'):
915                    d['jamb_sex']  = 'F'
916            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
917            if intersect:
918                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
919                if sub_obj is None:
920                    try:
921                        student_obj.invokeFactory(pt,f2t[pt]['id'])
922                    except:
923                        continue
924                    sub_obj = getattr(student_obj,f2t[pt]['id'])
925                    d['Title'] = f2t[pt]['title']
926                sub_doc = sub_obj.getContent()
927                for field in intersect:
928                    changed = False
929                    if getattr(sub_doc,field) != mapping.get(field,''):
930                        any_change = True
931                        changed = True
932                        d[field] = mapping.get(field,'')
933                    if changed:
934                        sub_doc.edit(mapping = d)
935
936
937        # included only to change wf state from admitted to returning
938            if res[0].review_state == 'admitted':
939                new_state = f2t[pt]['wf_transition_returning']
940                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
941                if sub_obj and new_state != "remain":
942                    try:
943                        self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
944                    except:
945                        #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
946                        pass
947        if res[0].review_state == 'admitted':
948            wfaction = 'return'
949            try:
950                self.portal_workflow.doActionFor(student_obj,wfaction)
951                logger.info('%s, wf state changed' % sid)
952                any_change = True
953            except:
954                logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
955                pass
956        # end inclusion
957
958
959        if any_change:
960            return sid,''
961        else:
962            return sid,'not modified'
963    ###)
964
965
966    security.declareProtected(ModifyPortalContent,"importData")###(
967    def importData(self,filename,name,edit=False):
968        """load data from CSV values"""
969        import transaction
970        import random
971
972        pm = self.portal_membership
973        member = pm.getAuthenticatedMember()
974
975        logger = logging.getLogger('WAeUPTool.importData')
976        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
977        students_folder = self.portal_url.getPortalObject().campus.students
978        start = True
979        tr_count = 1
980        total_imported = 0
981        total_not_imported = 0
982        total = 0
983        iname = "import_%s" % name
984        stool = getToolByName(self, 'portal_schemas')
985        ltool = getToolByName(self, 'portal_layouts')
986        schema = stool._getOb(iname)
987        if schema is None:
988            em = 'No such schema %s' % iname
989            logger.error('No such schema %s' % iname)
990            return em
991        layout = ltool._getOb(iname)
992        if layout is None:
993            em = 'No such layout %s' % iname
994            logger.error(em)
995            return em
996        validators = {}
997        for widget in layout.keys():
998            validators[widget] = layout[widget].validate
999        if edit:
1000            importer_name = "importEdit%s" % name.capitalize()
1001        else:
1002            importer_name = "import%s" % name.capitalize()
1003        importer = getattr(self, '%s' % importer_name,None)
1004        if importer is None:
1005            em = 'No importer function %s' % importer_name
1006            logger.error(em)
1007            return em
1008        not_imported = []
1009        imported = []
1010        try:
1011            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1012        except:
1013            em = 'Error reading %s.csv' % filename
1014            logger.error(em)
1015            return em
1016        for item in items:
1017            if start:
1018                start = False
1019                adapters = [MappingStorageAdapter(schema, item)]
1020                dm = DataModel(item, adapters,context=self)
1021                logger.info('%s starts import from %s.csv' % (member,filename))
1022                import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1023                diff2schema = set(import_keys).difference(set(schema.keys()))
1024                diff2layout = set(import_keys).difference(set(layout.keys()))
1025                if diff2schema:
1026                    em = "not ignorable key(s) %s found in heading" % diff2schema
1027                    return em
1028                s = ','.join(['"%s"' % fn for fn in import_keys])
1029                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1030                s = '"id",' + s
1031                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1032                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1033                format_error = format + ',"%(Error)s"'
1034                format = '"%(id)s",'+ format
1035            ds = DataStructure(data=item,datamodel=dm)
1036            error_string = ""
1037            for k in import_keys:
1038                if not validators[k](ds):
1039                    error_string += " %s : %s" % (k,ds.getError(k))
1040            if not error_string:
1041                item.update(dm)
1042                item['id'],error = importer(item)
1043                if error:
1044                    error_string += error
1045            if error_string:
1046                item['Error'] = error_string
1047                not_imported.append(format_error % item)
1048                total_not_imported += 1
1049            else:
1050                em = format % item
1051                imported.append(em)
1052                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1053                tr_count += 1
1054                total_imported += 1
1055            total += 1
1056            if total_imported and not total_imported % 100:
1057                transaction.commit()
1058                if len(not_imported) > 0:
1059                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1060                             '\n'.join(not_imported) + '\n')
1061                    not_imported = []
1062                if len(imported) > 0:
1063                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1064                             '\n'.join(imported) + '\n')
1065                    imported = []
1066                em = '%d transactions committed\n' % (tr_count)
1067                regs = []
1068                logger.info(em)
1069                tr_count = 0
1070        if len(imported) > 0:
1071            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1072                                                '\n'.join(imported))
1073        if len(not_imported) > 0:
1074            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1075                                                '\n'.join(not_imported))
1076        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1077        logger.info(em)
1078        return em
1079    ###)
1080
1081InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.