source: WAeUP_SRP/trunk/WAeUPTool.py @ 1920

Last change on this file since 1920 was 1911, checked in by Henrik Bettermann, 18 years ago

fixed

  • Property svn:keywords set to Id
File size: 42.4 KB
Line 
13#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 1911 2007-06-17 05:26:26Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29
30from Products.CMFCore.utils import getToolByName
31from Products.CPSSchemas.DataStructure import DataStructure
32from Products.CPSSchemas.DataModel import DataModel
33from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34from Products.CMFCore.ActionProviderBase import ActionProviderBase
35from Products.CMFCore.permissions import View
36from Products.ZCatalog.ZCatalog import ZCatalog
37from Products.CMFCore.permissions import ModifyPortalContent
38from Products.CMFCore.permissions import ManagePortal
39from Products.CMFCore.utils import UniqueObject
40from Products.CMFCore.URLTool import URLTool
41from Products.CMFCore.utils import getToolByName
42from Students import makeCertificateCode
43from Globals import package_home,INSTANCE_HOME
44p_home = package_home(globals())
45i_home = INSTANCE_HOME
46import DateTime
47import logging
48import transaction
49import csv,re,os
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52def getObject(object,name):
53    if object.hasObject(name):
54        return getattr(object,name)
55    return None
56
57class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58    """WAeUP tool"""
59
60    id = 'waeup_tool'
61    meta_type = 'WAeUP Tool'
62    _actions = ()
63
64    security = ClassSecurityInfo()
65    security.declareObjectProtected(View)
66
67    manage_options = ( ActionProviderBase.manage_options
68                     + SimpleItem.manage_options
69                     )
70
71    def rwrite(self,s): ###(
72        response = self.REQUEST.RESPONSE
73        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74        response.write("%s<br />\r\n" % s)
75    ###)
76
77    def sleep(self,secs): ###(
78        "sleep"
79        import time
80        time.sleep(secs)
81        return
82
83###)
84
85    security.declareProtected(ModifyPortalContent,'openLog') ###(
86    def openLog(self,name):
87        """open a log file"""
88        version = 1
89        path = "%s/log/%s_%d.log" % (i_home,name,version)
90        while os.path.exists(path):
91            version += 1
92            path = "%s/log/%s_%d.log" % (i_home,name,version)
93        log = open(path,"w")
94        return log
95
96###)
97
98    security.declareProtected(ModifyPortalContent,'writeLog') ###(
99    def writeLog(self,logfile,s):
100        """write to the log file"""
101        logfile.write(s)
102
103###)
104
105    def generateStudentId(self,letter): ###(
106        import random
107        r = random
108        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
109        if letter == '?':
110            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
111        sid = "%c%d" % (letter,r.randint(99999,1000000))
112        students = self.portal_url.getPortalObject().campus.students
113##        while hasattr(students, sid):
114##            sid = "%c%d" % (letter,r.randint(99999,1000000))
115        while self.students_catalog(id = sid):
116            sid = "%c%d" % (letter,r.randint(99999,1000000))
117        return sid
118    ###)
119
120    def generatePassword(self,s=None): ###(
121        import random
122        r = random
123        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
124        if s is None:
125            s = 'abcdefghklmnpqrstuvwxy23456789'
126        pw = ''
127        while len(pw) < 6:
128            pw += r.choice(s)
129        return pw
130    ###)
131
132    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
133    def removeDeletedDocIds(self, max=1000):
134        """
135        remove deleted docids from repository commit after max
136        """
137        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
138        repository = getToolByName(self, 'portal_repository')
139        pxtool = getToolByName(self, 'portal_proxies')
140        pxtool_infos = pxtool.getRevisionsUsed()
141
142        nb_revs = 0
143        docids_d = {} # all docids
144        unused_docids_d = {} # all docids that are unused
145        ids_unused_revs_docids = [] # ids for revs of unused docids
146        ids_unused_revs = [] # ids for unused revs
147        total = 0
148        idlist = repository.objectIds()
149        for id in idlist:
150            docid, rev = repository._splitId(id)
151            if docid is None:
152                logger.info("invalid doc_id %s" % docid)
153                continue
154            nb_revs += 1
155            docids_d[docid] = None
156            if not pxtool_infos.has_key(docid):
157                unused_docids_d[docid] = None
158                ids_unused_revs_docids.append(id)
159                ids_unused_revs.append(id)
160            elif not pxtool_infos[docid].has_key(rev):
161                ids_unused_revs.append(id)
162            if len(ids_unused_revs) >= max:
163                repository.manage_delObjects(ids_unused_revs)
164                #import pdb;pdb.set_trace()
165                transaction.commit()
166                total += max
167                logger.info('removed %d total %d unused docids ' % (max,total))
168        anz = len(ids_unused_revs)
169        if anz > 0:
170            repository.manage_delObjects(ids_unused_revs)
171            transaction.commit()
172            total += anz
173            logger.info('finished removing %d unused docids ' % (total))
174
175
176###)
177
178    security.declareProtected(ModifyPortalContent,'getCredential') ###(
179    def getCredential(self,student_id):
180        "return a student password"
181        student_entry = getattr(self.portal_directories.students,student_id,None)
182        if student_entry is None:
183            return None
184        return getattr(student_entry,"password","not set")
185    ###)
186
187    security.declarePublic('checkPassword') ###(
188    def checkPassword(self,student_id,password):
189        "return a student password"
190        student_entry = getattr(self.portal_directories.students,student_id,None)
191        if student_entry is None:
192            return False
193        return getattr(student_entry,"password","not set") == password
194    ###)
195
196    security.declarePublic('editPassword') ###(
197    def editPassword(self,student_id,password):
198        "edit a student password"
199        student_entry = getattr(self.portal_directories.students,student_id,None)
200        if student_entry is None:
201            return
202        setattr(student_entry,'password',password)
203    ###)
204
205    security.declareProtected(View,'doCommit') ###(
206    def doCommit(self,logger=None):
207        "commit some transactions"
208        transaction.commit()
209    ###)
210
211    security.declarePublic('loadStudentFoto') ###(
212    def loadStudentFoto(self,student,filename,folder):
213        "return a student passport picture"
214        app_doc = student.application.getContent()
215        #clear = student.clearance
216        #clear_doc = clear.getContent()
217        #matric_no = clear_doc.matric_no.upper()
218        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
219        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
220        #import pdb;pdb.set_trace()
221        if os.path.exists(picture1):
222            file = open(picture1)
223        elif os.path.exists(picture2):
224            file = open(picture2)
225        else:
226            return "passport picture not found %s" % picture1
227
228        outfile = file.read()
229        app_doc.manage_addFile('passport',
230                               file=outfile,
231                               title="%s.jpg" % filename)
232        return "successfully loaded passport picture"
233    ###)
234
235    security.declareProtected(ModifyPortalContent,'createOne') ###(
236    def createOne(self,students_folder,student_brain,letter,commit=False):
237        sid = self.waeup_tool.generateStudentId(letter)
238        students_folder.invokeFactory('Student', sid)
239        student = getattr(students_folder,sid)
240        self.portal_workflow.doActionFor(student,'return')
241        student.manage_setLocalRoles(sid, ['Owner',])
242        matric_no = student_brain.matric_no
243        jamb_reg_no = student_brain.Entryregno
244        self.students_catalog.addRecord(id = sid,
245                                           matric_no = matric_no,
246                                           jamb_reg_no = jamb_reg_no,
247                                           sex = student_brain.Sex == "F",
248                                           name = "%s %s %s" % (student_brain.Firstname,
249                                                                student_brain.Middlename,
250                                                                student_brain.Lastname)
251                                        )
252        if commit:
253            transaction.commit()
254        return sid,jamb_reg_no
255    ###)
256
257    security.declareProtected(ModifyPortalContent,'addStudent') ###(
258    def addStudent(self,dict):
259        students_folder = self.portal_url.getPortalObject().campus.students
260        sid = self.waeup_tool.generateStudentId('?')
261        students_folder.invokeFactory('Student', sid)
262        student_obj = getattr(students_folder,sid)
263        f2t = self.student_field2types
264        #from pdb import set_trace; set_trace()
265        d = {}
266        d['jamb_sex']  = 'M'
267        if dict.get('sex'):
268            d['jamb_sex']  = 'F'
269
270        entry_session = dict.get('entry_session')
271        if entry_session == self.getSessionId()[-2:]:
272            wfaction = 'admit'
273            wft = 'wf_transition_admitted'
274            password = None
275        else:
276            wfaction = 'return'
277            wft = 'wf_transition_returning'
278            password = self.generatePassword()
279            self.makeStudentMember(sid,password)
280
281        for pt in f2t.keys():
282            student_obj.invokeFactory(pt,f2t[pt]['id'])
283            sub_obj = getattr(student_obj,f2t[pt]['id'])
284            sub_doc = sub_obj.getContent()
285            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
286            d['Title'] = f2t[pt]['title']
287            for field in f2t[pt]['fields']:
288                d[field] = dict.get(field,'')
289            sub_doc.edit(mapping = d)
290            new_state = f2t[pt][wft]
291            if new_state != "remain":
292                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
293        self.portal_workflow.doActionFor(student_obj,wfaction)
294        student_obj.manage_setLocalRoles(sid, ['Owner',])
295        return sid,password
296    ###)
297
298    security.declarePublic('getCertificateBrain') ###(
299    def getCertificateBrain(self,cert_id):
300        "do it"
301        res = ZCatalog.searchResults(self.portal_catalog_real,
302                                {'portal_type':"Certificate",
303                                      'id': cert_id})
304        if res:
305            return res[0]
306        return None
307    ###)
308
309    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
310    def get_csv_filenames(self):
311        "do it"
312        files = [file for file in os.listdir("%s/import/" % (i_home))
313                 if file.endswith('.csv') and file.find('imported') == -1]
314        return files
315    ###)
316
317    security.declarePublic('findStudentByMatricelNo') ###(
318    def findStudentByMatricelNo(self,matric_no):
319        "do it"
320        res = ZCatalog.searchResults(self.portal_catalog_real,
321                                {'portal_type':"StudentClearance",
322                                 'SearchableText': matric_no})
323        if res:
324            return res[0]
325        return None
326    ###)
327
328    security.declarePublic('makeStudentMember') ###(
329    def makeStudentMember(self,sid,password='uNsEt'):
330        """make the student a member"""
331        membership = self.portal_membership
332        membership.addMember(sid,
333                             password ,
334                             roles=('Member',
335                                     'Student',
336                                     ),
337                             domains='',
338                             properties = {'memberareaCreationFlag': False,
339                                           'homeless': True},)
340        member = membership.getMemberById(sid)
341        self.portal_registration.afterAdd(member, sid, password, None)
342        #self.manage_setLocalRoles(sid, ['Owner',])
343    ###)
344
345    security.declarePublic('makeStudentData') ###(
346    def makeStudentData(self,student_id,email=None,phone_nr=None):
347        "create Datastructure for a returning Student"
348        #import pdb;pdb.set_trace()
349        logger = logging.getLogger('WAeUPTool.makeStudentData')
350        students_folder = self.portal_url.getPortalObject().campus.students
351        #res = self.students_catalog(id=student_id)
352        #if res:
353        #    st = res[0]
354        #res = self.returning_import(matric_no = st.matric_no)
355        res = self.returning_import(id = student_id)
356        if res:
357            student = res[0]
358        else:
359            logger.info('Id %s not found in returning_import' % student_id)
360            return
361        logger.info('%s creates data structure' % student_id)
362        s_results = self.results_import(matric_no = student.matric_no)
363        if s_results:
364            lnr = self.getLevelFromResultsCosCode(s_results)
365            level = "%d00" % lnr
366            verdict,eligible = self.getVerdict(s_results[0].Verdict)
367            if eligible:
368                level = "%d00" % (lnr + 1)
369        else:
370            logger.info('matric_no %s not found in results_import' % student.matric_no)
371            level = 0
372            verdict = 'N/A'
373        #student should not be allowed to perform this transition
374        #wftool = self.portal_workflow
375        #wftool.doActionFor(student,'return')
376        certcode_org = student.Coursemajorcode
377        certcode = makeCertificateCode(certcode_org)
378        certificate_brain = self.getCertificateBrain(certcode)
379        if not certificate_brain:
380            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
381            logger.info(em)
382        matric_no = student.matric_no
383        sid = student_id
384        student_obj = getattr(students_folder,sid)
385        student_obj.invokeFactory('StudentApplication','application')
386        application = student_obj.application
387        self.portal_workflow.doActionFor(application,'open',dest_container=application)
388        da = {'Title': 'Application Data'}
389        student_obj.invokeFactory('StudentPersonal','personal')
390        da['jamb_reg_no'] = student.Entryregno
391        em = self.getEntryMode(student.Entryregno)
392##        em = student.Mode_of_Entry
393##        if em in ('DIRECT', 'DIRECT ENTRY',):
394##            em = 'DE'
395##        elif em in ('U.M.E', 'UNE',):
396##            em = 'UME'
397##        elif not em:
398##            em = "unknown"
399        da['entry_mode'] = em
400        personal = student_obj.personal
401        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
402        dp = {'Title': 'Personal Data'}
403        student_obj.invokeFactory('StudentClearance','clearance')
404        clearance = student_obj.clearance
405        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
406        dc = {'Title': 'Clearance/Eligibility Record'}
407        dc['matric_no'] = matric_no
408        state = student.State
409        lga = student.LGA
410        if state and lga:
411            lga =  state + ' / ' + lga
412        else:
413            lga = "None"
414        da['jamb_lga'] = dc['lga'] = lga
415        da['app_email'] = dp['email'] = email
416        da['app_mobile'] = dp['phone'] = phone_nr
417        dp['firstname'] = student.Firstname
418        dp['middlename'] = student.Middlename
419        dp['lastname'] = student.Lastname
420        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
421        da['jamb_sex'] = student.Sex
422        dp['sex'] = student.Sex == 'F'
423        dp['perm_address'] = student.Permanent_Address
424        application.getContent().edit(mapping=da)
425        self.portal_workflow.doActionFor(application,'close',dest_container=application)
426        personal.getContent().edit(mapping=dp)
427        clearance.getContent().edit(mapping=dc)
428        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
429##        catd = {}
430##        catd['id'] = sid
431##        catd['entry_mode']= da['entry_mode']
432##        catd['matric_no'] = matric_no
433##        catd['jamb_reg_no'] = da['jamb_reg_no']
434##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
435##        catd['sex'] = dp['sex']
436##        catd['level'] = level
437##        catd['verdict'] = verdict
438##        if certificate_brain:
439##            cpath = certificate_brain.getPath().split('/')
440##            catd['faculty'] = cpath[-4]
441##            catd['department'] = cpath[-3]
442##            catd['course'] = certcode
443##        self.students_catalog.modifyRecord(**catd)
444        #
445        # Study Course
446        #
447        student_obj.invokeFactory('StudentStudyCourse','study_course')
448        studycourse = student_obj.study_course
449        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
450        dsc = {}
451        dsc['study_course'] = certcode
452        dsc['current_level'] = level
453        dsc['current_verdict'] = verdict
454        dsc['current_mode'] = em
455        dsc['current_session'] = '06'
456        studycourse.getContent().edit(mapping=dsc)
457        #
458        # Level
459        #
460##        l = getattr(studycourse,level,None)
461##        if l is None:
462##            studycourse.invokeFactory('StudentStudyLevel', level)
463##            l = getattr(studycourse, level)
464##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
465##            l.getContent().edit(mapping={'Title': "Level %s" % level})
466###)
467
468    security.declarePublic('makeStudentLevel') ###(
469    def makeStudentLevel(self,student_id):
470        "create the StudyLevel for a returning Student"
471        #import pdb;pdb.set_trace()
472        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
473        students_folder = self.portal_url.getPortalObject().campus.students
474        res = self.students_catalog(id=student_id)
475        if res:
476            st = res[0]
477        course = st.course
478        matric_no = st.matric_no
479        level = st.level
480        res = self.results_import(matric_no = matric_no)
481        if res:
482            results = res
483        logger.info('%s creating Level %s' % (student_id,level))
484        #
485        # Level
486        #
487        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
488        studycourse = getattr(student_obj,"study_course",None)
489        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
490        l = getattr(studycourse,level,None)
491        if l is None:
492            studycourse.invokeFactory('StudentStudyLevel', level)
493            l = getattr(studycourse, level)
494            self.portal_workflow.doActionFor(l,'open',dest_container=l)
495            l.getContent().edit(mapping={'Title': "Level %s" % level})
496        ###)
497
498    security.declarePublic('getAccommodationInfo') ###(
499    def getAccommodationInfo(self,bed):
500        """return Accommodation Info"""
501        info = {}
502        hall,block,room,letter = bed.split('_')
503        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
504        if res and len(res) == 1:
505            hall_brain = res[0]
506            hall_doc = hall_brain.getObject().getContent()
507        else:
508            return info
509        info['hall_title'] = hall_brain.Title
510        info['maintenance_code'] = hall_doc.maintenance_code
511        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
512        batch_doc = None
513        for brain in res:
514            if brain.id.startswith(info['maintenance_code']):
515                batch_doc = brain.getObject().getContent()
516                break
517        if batch_doc is None:
518            info['maintenance_fee'] = None
519        else:
520            info['maintenance_fee'] = batch_doc.cost
521        return info
522    ###)
523
524    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
525    def deleteAllCourses(self,department="All"):
526        ''' delete the courses'''
527        pm = self.portal_membership
528        member = pm.getAuthenticatedMember()
529
530        if str(member) not in ("henrik","joachim"):
531            return "not possible"
532        if department == "All":
533            res = self.portal_catalog({'meta_type': 'Department'})
534        if len(res) < 1:
535            return "No Departments found"
536
537        deleted = []
538        for dep in res:
539            cf = dep.getObject().courses
540            if cf:
541                cf.manage_delObjects(ids=cf.objectIds())
542                deleted.append("deleted Courses in %s" % dep.getId)
543        return "\r".join(deleted)
544    ###)
545
546    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
547    def getLogfileLines(self,filename="event.log",numlines=20):
548        """Get last NUMLINES lines of logfile FILENAME.
549
550        Return last lines' of a file in the instances logfile directory as
551        a list. The number of returned lines equals `numlines' or less. If
552        less than `numlines' lines are available, the whole file ist
553        returned. If the file can not be opened or some other error
554        occurs, empty list is returend.
555        """
556        result = []
557        lines_hit = 0
558
559        # We only handle files in instances' log directory...
560        logpath = os.path.join(i_home, "log")
561        filename = str(os.path.abspath( os.path.join( logpath, filename )))
562        if not filename.startswith( logpath ):
563            # Attempt to access file outside log-dir...
564            return []
565
566        try:
567            fd = file( filename, "rb" )
568        except IOError:
569            return []
570        if not fd:
571            return []
572
573        if os.linesep == None:
574            linesep = '\n'
575        else:
576            linesep = os.linesep
577
578        # Try to find 'numlines' times a lineseparator, searching from end
579        # and moving to the beginning of file...
580        fd.seek( 0, 2) # Move to end of file...
581        while lines_hit < numlines:
582            if fd.read(1) == linesep[-1]: # This moves filedescriptor
583                                          # one step forward...
584                lines_hit += 1
585            try:
586                fd.seek( -2, 1) # Go two bytes back from current pos...
587            except IOError:
588                # We cannot go back two bytes. Maybe the file is too small...
589                break
590        fd.seek(2,1)
591
592        # Read all lines from current position...
593        result = fd.readlines()
594        # Remove line endings...
595        result = [x.strip() for x in result]
596        fd.close()
597        return result
598    ###)
599
600    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
601    def getCallbacksFromLog(self,filename):
602        """fix Online Payment Transactions from Z2.log entries"""
603        import transaction
604        import random
605        from cgi import parse_qs
606        from urlparse import urlparse
607        #from pdb import set_trace
608        wftool = self.portal_workflow
609        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
610        students_folder = self.portal_url.getPortalObject().campus.students
611        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
612        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
613        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
614        data = re.compile(s)
615        start = True
616        tr_count = 1
617        total = 0
618        #name = 'pume_results'
619        #name = 'epaymentsuccessful_z2log2'
620        name = filename
621        no_import = []
622        imported = []
623        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
624        try:
625            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
626        except:
627            logger.error('Error reading %s' % name)
628            return
629        tas = []
630        for line in transactions:
631            dict = {}
632            items = data.search(line)
633            dict['idict'] = idict = items.groupdict()
634            #print idict
635            #from pdb import set_trace;set_trace()
636            urlparsed = urlparse(idict['get'][4:])
637            #print urlparsed
638            path = urlparsed[2].split('/')
639            dict['student_id'] = student_id = path[8]
640            dict['payment_id'] = payment_id = path[10]
641            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
642            tas.append(dict)
643            tr_count += 1
644        return tas
645    ###)
646
647    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
648    def importOnlinePaymentTransactions(self):
649        """load Online Payment Transactions from CSV values"""
650        import transaction
651        import random
652        #from pdb import set_trace
653        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
654        opt = self.online_payments_import
655        students_folder = self.portal_url.getPortalObject().campus.students
656        start = True
657        tr_count = 1
658        total = 0
659        #name = 'pume_results'
660        name = 'OnlineTransactions'
661        no_import = []
662        imported = []
663        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
664        try:
665            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
666        except:
667            logger.error('Error reading %s.csv' % name)
668            return
669        for pay_transaction in transactions:
670            if start:
671                start = False
672                logger.info('Start loading from %s.csv' % name)
673                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
674                no_import.append('%s,"Error"' % s)
675                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
676                format_error = format + ',"%(Error)s"'
677            data = {}
678
679            # format of the first file sent by Tayo
680            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
681            #data['student_id'] = student_id = pay_transaction['Payer ID']
682            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
683            #data['response_code'] = response_code = pay_transaction['Resp Code']
684            #data['amount'] = amount = pay_transaction['Amount']
685
686            # format of the second file sent by Tayo
687            #data['datetime'] = date = 0
688            #data['student_id'] = student_id = pay_transaction['Payer ID']
689            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
690            #data['response_code'] = response_code = '00'
691            #data['amount'] = amount = pay_transaction['Amount']
692
693            # format of the third file sent by Kehinde
694            data['datetime'] = date = 0
695            data['student_id'] = student_id = pay_transaction['customer_id']
696            data['order_id'] = order_id = pay_transaction['merchant_reference']
697            data['response_code'] = response_code = '00'
698            data['amount'] = amount = pay_transaction['Amount']
699
700            dup = False
701            if response_code == "12":
702                continue
703            try:
704                opt.addRecord(**data)
705            except ValueError:
706                dup = True
707            #from pdb import set_trace;set_trace()
708            if dup:
709                if response_code == "00":
710                    try:
711                        opt.modifyRecord(**data)
712                    except:
713                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
714                        continue
715                else:
716                    pay_transaction['Error'] = "Duplicate order_id"
717                    no_import.append( format_error % pay_transaction)
718                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
719                    continue
720            tr_count += 1
721            if tr_count > 1000:
722                if len(no_import) > 0:
723                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
724                             '\n'.join(no_import) + '\n')
725                    no_import = []
726                em = '%d transactions committed\n' % (tr_count)
727                transaction.commit()
728                regs = []
729                logger.info(em)
730                total += tr_count
731                tr_count = 0
732        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
733                                                '\n'.join(no_import))
734        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
735    ###)
736
737
738    # all workflow transitions refer to students in state returning
739    student_field2types = {   ###(
740                      'StudentApplication':
741                          {'id': 'application',
742                           'title': 'Application Data',
743                           'wf_transition_returning': 'close',
744                           'wf_transition_admitted': 'remain',
745                           'fields':
746                             ('jamb_reg_no',
747                              'entry_mode',
748                              'entry_session',
749                              'jamb_score',
750                              'app_email',
751                              )
752                              },
753                      #'StudentPume':
754                      #    {'id': 'pume',
755                      #     'title': 'Pume Data',
756                      #     'wf_transition_returning': 'close',
757                      #     'wf_transition_admitted': 'close',
758                      #     'fields':
759                      #       ('pume_score',
760                      #        )
761                      #        },
762                      'StudentClearance':
763                          {'id': 'clearance',
764                           'title': 'Clearance Data',
765                           'wf_transition_returning': 'close',
766                           'wf_transition_admitted': 'remain',
767                           'fields':
768                             ('matric_no',
769                              'nationality',
770                              'lga',
771                              'birthday',
772                              )
773                              },
774                         'StudentPersonal':
775                          {'id': 'personal',
776                           'title': 'Personal Data',
777                           'wf_transition_returning': 'open',
778                           'wf_transition_admitted': 'remain',
779                           'fields':
780                             ('firstname',
781                              'middlename',
782                              'lastname',
783                              'sex',
784                              'email',
785                              'phone',
786                              'perm_address',
787                              )
788                              },
789                         'StudentStudyCourse':
790                          {'id': 'study_course',
791                           'title': 'Study Course',
792                           'wf_transition_returning': 'open',
793                           'wf_transition_admitted': 'remain',
794                           'fields':
795                             ('study_course',
796                              'current_level',
797                              'current_session',
798                              'current_mode',
799                              'current_verdict',
800                              )
801                              },
802
803                         'PaymentsFolder':
804                          {'id': 'payments',
805                           'title': 'Payments',
806                           'wf_transition_returning': 'open',
807                           'wf_transition_admitted': 'open',
808                           'fields':
809                             ()
810                              },
811                         }
812    ###)
813
814
815    security.declareProtected(ModifyPortalContent,'importStudent') ###(
816    def importStudent(self,mapping):
817        "create a students record due import"
818        logger = logging.getLogger('WAeUPTool.importStudent')
819        students_folder = self.portal_url.getPortalObject().campus.students
820        jamb_reg_no = mapping.get('jamb_reg_no',None)
821        if jamb_reg_no:
822            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
823            if res:
824                return '',"jamb_reg_no exists"
825        matric_no = mapping.get('matric_no',None)
826        if matric_no:
827            res = self.students_catalog(matric_no = matric_no)
828            if res:
829                return '',"matric_no exists"
830        sid = self.waeup_tool.generateStudentId('?')
831        students_folder.invokeFactory('Student', sid)
832        student_obj = getattr(students_folder,sid)
833        f2t = self.student_field2types
834        d = {}
835        d['jamb_sex']  = 'M'
836        if mapping.get('sex'):
837            d['jamb_sex']  = 'F'
838        for pt in f2t.keys():
839            student_obj.invokeFactory(pt,f2t[pt]['id'])
840            sub_obj = getattr(student_obj,f2t[pt]['id'])
841            sub_doc = sub_obj.getContent()
842            d['Title'] = f2t[pt]['title']
843            for field in f2t[pt]['fields']:
844                d[field] = mapping.get(field,'')
845            if pt == "StudyCourse":
846                for von,zu in (('entry_mode','current_mode'),
847                               ('entry_session','current_session')):
848                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
849                        d[zu] = mapping[von]
850            sub_doc.edit(mapping = d)
851
852            new_state = f2t[pt]['wf_transition_admitted']
853            if new_state != "remain":
854                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
855        wfaction = 'admit'
856        self.portal_workflow.doActionFor(student_obj,wfaction)
857        student_obj.manage_setLocalRoles(sid, ['Owner',])
858        return sid,''
859    ###)
860
861    security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
862    def importEditStudent(self,mapping):
863        "edit a students record due import"
864        logger = logging.getLogger('WAeUPTool.importEditStudent')
865        students_folder = self.portal_url.getPortalObject().campus.students
866        sid = mapping.get('id',None)
867        jamb_reg_no = mapping.get('jamb_reg_no',None)
868        matric_no = mapping.get('matric_no',None)
869        editable_keys = mapping.keys()
870        if sid:
871            res = self.students_catalog(id = sid)
872            if not res:
873                return '',"no student with id %s" % sid
874            elif matric_no and res[0].matric_no and\
875              matric_no != res[0].matric_no:
876                return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
877            elif jamb_reg_no and res[0].jamb_reg_no and\
878              jamb_reg_no != res[0].jamb_reg_no:
879                return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
880        elif jamb_reg_no:
881            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
882            if not res:
883                return '',"no student with jamb_reg_no %s" % jamb_reg_no
884            editable_keys.remove('jamb_reg_no')
885        elif matric_no:
886            res = self.students_catalog(matric_no = matric_no)
887            if not res:
888                return '',"no student with matric_no %s" % matric_no
889            editable_keys.remove('matric_no')
890        sid = res[0].id
891        student_obj = getattr(students_folder,sid)
892        f2t = self.student_field2types
893        d = {}
894        #import pdb;pdb.set_trace()
895        any_change = False
896        for pt in f2t.keys():
897            if pt == "student_application":
898                d['jamb_sex']  = 'M'
899                if mapping.get('sex'):
900                    d['jamb_sex']  = 'F'
901            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
902            if intersect:
903                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
904                if sub_obj is None:
905                    try:
906                        student_obj.invokeFactory(pt,f2t[pt]['id'])
907                    except:
908                        continue
909                    sub_obj = getattr(student_obj,f2t[pt]['id'])
910                    d['Title'] = f2t[pt]['title']
911                sub_doc = sub_obj.getContent()
912                for field in intersect:
913                    changed = False
914                    if getattr(sub_doc,field) != mapping.get(field,''):
915                        any_change = True
916                        changed = True
917                        d[field] = mapping.get(field,'')
918                    if changed:
919                        sub_doc.edit(mapping = d)
920
921
922        # included only to change wf state from admitted to returning
923            new_state = f2t[pt]['wf_transition_returning']
924            sub_obj = getattr(student_obj,f2t[pt]['id'],None)
925            if sub_obj and new_state != "remain":
926                try:
927                    self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
928                except:
929                    #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
930                    pass
931        wfaction = 'return'
932        try:
933            self.portal_workflow.doActionFor(student_obj,wfaction)
934            logger.info('%s, wf state changed' % sid)
935            any_change = True
936        except:
937            logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
938            pass
939        # end inclusion
940
941
942        if any_change:
943            return sid,''
944        else:
945            return sid,'not modified'
946    ###)
947
948
949    security.declareProtected(ModifyPortalContent,"importData")###(
950    def importData(self,filename,name,edit=False):
951        """load data from CSV values"""
952        import transaction
953        import random
954
955        pm = self.portal_membership
956        member = pm.getAuthenticatedMember()
957
958        logger = logging.getLogger('WAeUPTool.importData')
959        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
960        students_folder = self.portal_url.getPortalObject().campus.students
961        start = True
962        tr_count = 1
963        total_imported = 0
964        total_not_imported = 0
965        total = 0
966        iname = "import_%s" % name
967        stool = getToolByName(self, 'portal_schemas')
968        ltool = getToolByName(self, 'portal_layouts')
969        schema = stool._getOb(iname)
970        if schema is None:
971            em = 'No such schema %s' % iname
972            logger.error('No such schema %s' % iname)
973            return em
974        layout = ltool._getOb(iname)
975        if layout is None:
976            em = 'No such layout %s' % iname
977            logger.error(em)
978            return em
979        validators = {}
980        for widget in layout.keys():
981            validators[widget] = layout[widget].validate
982        if edit:
983            importer_name = "importEdit%s" % name.capitalize()
984        else:
985            importer_name = "import%s" % name.capitalize()
986        importer = getattr(self, '%s' % importer_name,None)
987        if importer is None:
988            em = 'No importer function %s' % importer_name
989            logger.error(em)
990            return em
991        not_imported = []
992        imported = []
993        try:
994            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
995        except:
996            em = 'Error reading %s.csv' % filename
997            logger.error(em)
998            return em
999        for item in items:
1000            if start:
1001                start = False
1002                adapters = [MappingStorageAdapter(schema, item)]
1003                dm = DataModel(item, adapters,context=self)
1004                logger.info('%s starts import from %s.csv' % (member,filename))
1005                import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1006                diff2schema = set(import_keys).difference(set(schema.keys()))
1007                diff2layout = set(import_keys).difference(set(layout.keys()))
1008                if diff2schema:
1009                    em = "not ignorable key(s) %s found in heading" % diff2schema
1010                    return em
1011                s = ','.join(['"%s"' % fn for fn in import_keys])
1012                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1013                s = '"id",' + s
1014                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1015                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1016                format_error = format + ',"%(Error)s"'
1017                format = '"%(id)s",'+ format
1018            ds = DataStructure(data=item,datamodel=dm)
1019            error_string = ""
1020            for k in import_keys:
1021                if not validators[k](ds):
1022                    error_string += " %s : %s" % (k,ds.getError(k))
1023            if not error_string:
1024                item.update(dm)
1025                item['id'],error = importer(item)
1026                if error:
1027                    error_string += error
1028            if error_string:
1029                item['Error'] = error_string
1030                not_imported.append(format_error % item)
1031                total_not_imported += 1
1032            else:
1033                em = format % item
1034                imported.append(em)
1035                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1036                tr_count += 1
1037                total_imported += 1
1038            total += 1
1039            if total_imported and not total_imported % 100:
1040                transaction.commit()
1041                if len(not_imported) > 0:
1042                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1043                             '\n'.join(not_imported) + '\n')
1044                    not_imported = []
1045                if len(imported) > 0:
1046                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1047                             '\n'.join(imported) + '\n')
1048                    imported = []
1049                em = '%d transactions committed\n' % (tr_count)
1050                regs = []
1051                logger.info(em)
1052                tr_count = 0
1053        if len(imported) > 0:
1054            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1055                                                '\n'.join(imported))
1056        if len(not_imported) > 0:
1057            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1058                                                '\n'.join(not_imported))
1059        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1060        logger.info(em)
1061        return em
1062    ###)
1063
1064InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.