source: WAeUP_SRP/trunk/WAeUPTool.py @ 2188

Last change on this file since 2188 was 2188, checked in by Henrik Bettermann, 17 years ago

remove jamb_lga and jamb_state from import schema

  • Property svn:keywords set to Id
File size: 50.4 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2188 2007-09-11 09:02:18Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29
30from Products.CMFCore.utils import getToolByName
31from Products.CPSSchemas.DataStructure import DataStructure
32from Products.CPSSchemas.DataModel import DataModel
33from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34from Products.CMFCore.ActionProviderBase import ActionProviderBase
35from Products.CMFCore.permissions import View
36from Products.ZCatalog.ZCatalog import ZCatalog
37from Products.CMFCore.permissions import ModifyPortalContent
38from Products.CMFCore.permissions import ManagePortal
39from Products.CMFCore.utils import UniqueObject
40from Products.CMFCore.URLTool import URLTool
41from Products.CMFCore.utils import getToolByName
42from Students import makeCertificateCode
43from Globals import package_home,INSTANCE_HOME
44p_home = package_home(globals())
45i_home = INSTANCE_HOME
46import DateTime,time
47import logging
48import transaction
49import csv,re,os
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52def getObject(object,name):
53    if object.hasObject(name):
54        return getattr(object,name)
55    return None
56
57class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58    """WAeUP tool"""
59
60    id = 'waeup_tool'
61    meta_type = 'WAeUP Tool'
62    _actions = ()
63
64    security = ClassSecurityInfo()
65    security.declareObjectProtected(View)
66
67    manage_options = ( ActionProviderBase.manage_options
68                     + SimpleItem.manage_options
69                     )
70
71    def rwrite(self,s): ###(
72        response = self.REQUEST.RESPONSE
73        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
74        response.write("%s<br />\r\n" % s)
75    ###)
76
77    def sleep(self,secs): ###(
78        "sleep"
79        import time
80        time.sleep(secs)
81        return
82
83###)
84
85    security.declareProtected(ModifyPortalContent,'openLog') ###(
86    def openLog(self,name):
87        """open a log file"""
88        version = 1
89        path = "%s/log/%s_%d.log" % (i_home,name,version)
90        while os.path.exists(path):
91            version += 1
92            path = "%s/log/%s_%d.log" % (i_home,name,version)
93        log = open(path,"w")
94        return log
95
96###)
97
98    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
99    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
100        """measure Object access Time"""
101        import random
102        if hasattr(self,'portal_catalog_real'):
103            aq_portal = self.portal_catalog_real.evalAdvancedQuery
104        else:
105            aq_portal = self.portal_catalog.evalAdvancedQuery
106        nr_pts = int(nr_pts)
107        probe = int(probe)
108        intervall = probe/10
109        objects = ("application","clearance","personal")
110        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
111        #i = random.randrange(num_objects)
112        count = 0
113        found = 0
114        not_found = 0
115        t_found = 0
116        t_not_found = 0
117        time_found = time_not_found = 0.0
118        t_time_found = t_time_not_found = 0.0
119        accessed = []
120        t_min = 1000
121        t_max = 0
122        #import pdb;pdb.set_trace()
123        students = self.portal_catalog(portal_type="Student")
124        num_students = len(students)
125        if method == "d":
126            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
127            res = aq_portal(query)
128            brains = {}
129            for r in res:
130                sid = r.relative_path.split('/')[-2]
131                if brains.has_key(sid):
132                    brains[sid][r.portal_type] = r
133                else:
134                    brains[sid] = {r.portal_type : r}
135            brains_list = brains.keys()
136            num_objects = len(brains_list)
137        else:
138            num_objects = num_students
139        print "="*40
140        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
141                                                                        probe,
142                                                                        nr_pts,
143                                                                        num_objects)
144        print "nr found/not time found/not min/max"
145        elapse = time.time()
146        i_elapse = time.time()
147        c_elapse = time.clock()
148        for c in range(1,probe + 1):
149            i = random.randrange(num_objects)
150            if method in ('a','b','c'):
151                student_brain = students[i]
152            elif method == "d":
153                #import pdb;pdb.set_trace()
154                student_brain = brains[brains_list[i]]
155            if method == "c":
156                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
157                res = aq_portal(query)
158                this_portal_types = [r.portal_type for r in res]
159            for i in range(nr_pts):
160                oid = objects[i]
161                if method == "a":
162                    try:
163                        student_path = student_brain.getPath()
164                        path = "%s/%s" % (student_path,oid)
165                        doc = self.unrestrictedTraverse(path).getContent()
166                        found += 1
167                        i_time = time.time() - i_elapse
168                        time_found += i_time
169                    except:
170                        not_found += 1
171                        i_time = time.time() - i_elapse
172                        time_not_found += i_time
173                        pass
174                elif method == "b":
175                    try:
176                        student_object = student_brain.getObject()
177                        doc = getattr(student_object,oid).getContent()
178                        found += 1
179                        i_time = time.time() - i_elapse
180                        time_found += i_time
181                    except:
182                        i_time = time.time() - i_elapse
183                        time_not_found += i_time
184                        not_found += 1
185                        pass
186                elif method == "c":
187                    if portal_types[i] in this_portal_types:
188                        found += 1
189                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
190                        i_time = time.time() - i_elapse
191                        time_found += i_time
192                    else:
193                        not_found += 1
194                        i_time = time.time() - i_elapse
195                        time_not_found += i_time
196                elif method == "d":
197                    if student_brain.has_key(portal_types[i]):
198                        found += 1
199                        doc = student_brain[portal_types[i]].getObject().getContent()
200                        i_time = time.time() - i_elapse
201                        time_found += i_time
202                    else:
203                        not_found += 1
204                        i_time = time.time() - i_elapse
205                        time_not_found += i_time
206                i_elapse = time.time()
207            if c and (c % intervall == 0):
208                #i_time = time.time() - i_elapse
209                t_per = 0.0
210                if found:
211                    t_per = time_found/found
212                if t_per > t_max:
213                    t_max = t_per
214                if t_per > 0.0 and t_per < t_min:
215                    t_min = t_per
216                itf = 0.0
217                if found:
218                    itf = time_found/found
219                itnf = 0.0
220                if not_found :
221                    itnf = time_not_found / not_found
222                interval_time = time_found + time_not_found
223                s = "%(c)d: %(found)d/%(not_found)d " % vars()
224                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
225                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
226                print s
227                t_found += found
228                t_not_found += not_found
229                t_time_found += time_found
230                t_time_not_found += time_not_found
231                time_found = time_not_found = 0.0
232                found = not_found = 0
233        # t_found += found
234        # t_not_found += not_found
235        elapse = time.time() - elapse
236        itf = 0.0
237        if t_found:
238            itf = t_time_found/t_found
239        itnf = 0.0
240        if t_not_found:
241            itnf = t_time_not_found / t_not_found
242        #c_elapse = time.clock() - c_elapse
243        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
244        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
245        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
246        print "-"*40
247        print s
248        rel_found = float(t_found)/probe
249        rel_not_found = float(t_not_found)/probe
250        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
251        print estimated_total_time
252    ###)
253
254    security.declareProtected(ModifyPortalContent,'writeLog') ###(
255    def writeLog(self,logfile,s):
256        """write to the log file"""
257        logfile.write(s)
258
259###)
260
261    def generateStudentId(self,letter): ###(
262        import random
263        r = random
264        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
265        if letter == '?':
266            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
267        sid = "%c%d" % (letter,r.randint(99999,1000000))
268        students = self.portal_url.getPortalObject().campus.students
269##        while hasattr(students, sid):
270##            sid = "%c%d" % (letter,r.randint(99999,1000000))
271        while self.students_catalog(id = sid):
272            sid = "%c%d" % (letter,r.randint(99999,1000000))
273        return sid
274    ###)
275
276    def generatePassword(self,s=None): ###(
277        import random
278        r = random
279        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
280        if s is None:
281            s = 'abcdefghklmnpqrstuvwxy23456789'
282        pw = ''
283        while len(pw) < 6:
284            pw += r.choice(s)
285        return pw
286    ###)
287
288    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
289    def removeDeletedDocIds(self, max=1000):
290        """
291        remove deleted docids from repository commit after max
292        """
293        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
294        repository = getToolByName(self, 'portal_repository')
295        pxtool = getToolByName(self, 'portal_proxies')
296        pxtool_infos = pxtool.getRevisionsUsed()
297
298        nb_revs = 0
299        docids_d = {} # all docids
300        unused_docids_d = {} # all docids that are unused
301        ids_unused_revs_docids = [] # ids for revs of unused docids
302        ids_unused_revs = [] # ids for unused revs
303        total = 0
304        idlist = repository.objectIds()
305        for id in idlist:
306            docid, rev = repository._splitId(id)
307            if docid is None:
308                logger.info("invalid doc_id %s" % docid)
309                continue
310            nb_revs += 1
311            docids_d[docid] = None
312            if not pxtool_infos.has_key(docid):
313                unused_docids_d[docid] = None
314                ids_unused_revs_docids.append(id)
315                ids_unused_revs.append(id)
316            elif not pxtool_infos[docid].has_key(rev):
317                ids_unused_revs.append(id)
318            if len(ids_unused_revs) >= max:
319                repository.manage_delObjects(ids_unused_revs)
320                #import pdb;pdb.set_trace()
321                transaction.commit()
322                total += max
323                logger.info('removed %d total %d unused docids ' % (max,total))
324        anz = len(ids_unused_revs)
325        if anz > 0:
326            repository.manage_delObjects(ids_unused_revs)
327            transaction.commit()
328            total += anz
329            logger.info('finished removing %d unused docids ' % (total))
330
331
332###)
333
334    security.declareProtected(ModifyPortalContent,'getCredential') ###(
335    def getCredential(self,student_id):
336        "return a student password"
337        student_entry = getattr(self.portal_directories.students,student_id,None)
338        if student_entry is None:
339            return None
340        return getattr(student_entry,"password","not set")
341    ###)
342
343    security.declarePublic('checkPassword') ###(
344    def checkPassword(self,student_id,password):
345        "return a student password"
346        student_entry = getattr(self.portal_directories.students,student_id,None)
347        if student_entry is None:
348            return False
349        return getattr(student_entry,"password","not set") == password
350    ###)
351
352    security.declarePublic('editPassword') ###(
353    def editPassword(self,student_id,password):
354        "edit a student password"
355        student_entry = getattr(self.portal_directories.students,student_id,None)
356        if student_entry is None:
357            return
358        setattr(student_entry,'password',password)
359    ###)
360
361    security.declareProtected(View,'doCommit') ###(
362    def doCommit(self,logger=None):
363        "commit some transactions"
364        transaction.commit()
365    ###)
366
367    security.declarePublic('loadStudentFoto') ###(
368    def loadStudentFoto(self,student,filename,folder):
369        "return a student passport picture"
370        app = student.application
371        app_doc = app.getContent()
372        #clear = student.clearance
373        #clear_doc = clear.getContent()
374        #matric_no = clear_doc.matric_no.upper()
375        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
376        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
377        #import pdb;pdb.set_trace()
378        if os.path.exists(picture1):
379            file = open(picture1)
380        elif os.path.exists(picture2):
381            file = open(picture2)
382        else:
383            return "passport picture not found %s" % picture1
384        reopened = False
385        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
386            self.portal_workflow.doActionFor(app,'open')
387            reopened = True
388        outfile = file.read()
389        app_doc.manage_addFile('passport',
390                               file=outfile,
391                               title="%s.jpg" % filename)
392        if reopened:
393            self.portal_workflow.doActionFor(app,'close')
394        return "successfully loaded passport picture"
395    ###)
396
397    security.declareProtected(ModifyPortalContent,'createOne') ###(
398    def createOne(self,students_folder,student_brain,letter,commit=False):
399        sid = self.waeup_tool.generateStudentId(letter)
400        students_folder.invokeFactory('Student', sid)
401        student = getattr(students_folder,sid)
402        self.portal_workflow.doActionFor(student,'return')
403        student.manage_setLocalRoles(sid, ['Owner',])
404        matric_no = student_brain.matric_no
405        jamb_reg_no = student_brain.Entryregno
406        self.students_catalog.addRecord(id = sid,
407                                           matric_no = matric_no,
408                                           jamb_reg_no = jamb_reg_no,
409                                           sex = student_brain.Sex == "F",
410                                           name = "%s %s %s" % (student_brain.Firstname,
411                                                                student_brain.Middlename,
412                                                                student_brain.Lastname)
413                                        )
414        if commit:
415            transaction.commit()
416        return sid,jamb_reg_no
417    ###)
418
419    security.declareProtected(ModifyPortalContent,'addStudent') ###(
420    def addStudent(self,dict):
421        students_folder = self.portal_url.getPortalObject().campus.students
422        sid = self.waeup_tool.generateStudentId('?')
423        students_folder.invokeFactory('Student', sid)
424        student_obj = getattr(students_folder,sid)
425        f2t = self.student_field2types
426        #from pdb import set_trace; set_trace()
427        d = {}
428        d['jamb_sex']  = 'M'
429        if dict.get('sex'):
430            d['jamb_sex']  = 'F'
431
432        entry_session = dict.get('entry_session')
433        if entry_session == self.getSessionId()[-2:]:
434            wfaction = 'admit'
435            wft = 'wf_transition_admit'
436            password = None
437        else:
438            wfaction = 'return'
439            wft = 'wf_transition_return'
440            password = self.generatePassword()
441            self.makeStudentMember(sid,password)
442
443        for pt in f2t.keys():
444            student_obj.invokeFactory(pt,f2t[pt]['id'])
445            sub_obj = getattr(student_obj,f2t[pt]['id'])
446            sub_doc = sub_obj.getContent()
447            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
448            d['Title'] = f2t[pt]['title']
449            for field in f2t[pt]['fields']:
450                d[field] = dict.get(field,'')
451            sub_doc.edit(mapping = d)
452            new_state = f2t[pt][wft]
453            if new_state != "remain":
454                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
455        self.portal_workflow.doActionFor(student_obj,wfaction)
456        student_obj.manage_setLocalRoles(sid, ['Owner',])
457        return sid,password
458    ###)
459
460    security.declarePublic('getCertificateBrain') ###(
461    def getCertificateBrain(self,cert_id):
462        "do it"
463        res = ZCatalog.searchResults(self.portal_catalog_real,
464                                {'portal_type':"Certificate",
465                                      'id': cert_id})
466        if res:
467            return res[0]
468        return None
469    ###)
470
471    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
472    def get_csv_filenames(self):
473        "do it"
474        files = [file for file in os.listdir("%s/import/" % (i_home))
475                 if file.endswith('.csv') and file.find('imported') == -1]
476        return files
477    ###)
478
479    security.declarePublic('findStudentByMatricelNo') ###(
480    def findStudentByMatricelNo(self,matric_no):
481        "do it"
482        res = ZCatalog.searchResults(self.portal_catalog_real,
483                                {'portal_type':"StudentClearance",
484                                 'SearchableText': matric_no})
485        if res:
486            return res[0]
487        return None
488    ###)
489
490    security.declarePublic('makeStudentMember') ###(
491    def makeStudentMember(self,sid,password='uNsEt'):
492        """make the student a member"""
493        membership = self.portal_membership
494        membership.addMember(sid,
495                             password ,
496                             roles=('Member',
497                                     'Student',
498                                     ),
499                             domains='',
500                             properties = {'memberareaCreationFlag': False,
501                                           'homeless': True},)
502        member = membership.getMemberById(sid)
503        self.portal_registration.afterAdd(member, sid, password, None)
504        #self.manage_setLocalRoles(sid, ['Owner',])
505    ###)
506
507    security.declarePublic('makeStudentData') ###(
508    def makeStudentData(self,student_id,email=None,phone_nr=None):
509        "create Datastructure for a returning Student"
510        #import pdb;pdb.set_trace()
511        logger = logging.getLogger('WAeUPTool.makeStudentData')
512        students_folder = self.portal_url.getPortalObject().campus.students
513        #res = self.students_catalog(id=student_id)
514        #if res:
515        #    st = res[0]
516        #res = self.returning_import(matric_no = st.matric_no)
517        res = self.returning_import(id = student_id)
518        if res:
519            student = res[0]
520        else:
521            logger.info('Id %s not found in returning_import' % student_id)
522            return
523        logger.info('%s creates data structure' % student_id)
524        s_results = self.results_import(matric_no = student.matric_no)
525        if s_results:
526            lnr = self.getLevelFromResultsCosCode(s_results)
527            level = "%d00" % lnr
528            verdict,eligible = self.getVerdict(s_results[0].Verdict)
529            if eligible:
530                level = "%d00" % (lnr + 1)
531        else:
532            logger.info('matric_no %s not found in results_import' % student.matric_no)
533            level = 0
534            verdict = 'N/A'
535        #student should not be allowed to perform this transition
536        #wftool = self.portal_workflow
537        #wftool.doActionFor(student,'return')
538        certcode_org = student.Coursemajorcode
539        certcode = makeCertificateCode(certcode_org)
540        certificate_brain = self.getCertificateBrain(certcode)
541        if not certificate_brain:
542            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
543            logger.info(em)
544        matric_no = student.matric_no
545        sid = student_id
546        student_obj = getattr(students_folder,sid)
547        student_obj.invokeFactory('StudentApplication','application')
548        application = student_obj.application
549        self.portal_workflow.doActionFor(application,'open',dest_container=application)
550        da = {'Title': 'Application Data'}
551        student_obj.invokeFactory('StudentPersonal','personal')
552        da['jamb_reg_no'] = student.Entryregno
553        em = self.getEntryMode(student.Entryregno)
554##        em = student.Mode_of_Entry
555##        if em in ('DIRECT', 'DIRECT ENTRY',):
556##            em = 'DE'
557##        elif em in ('U.M.E', 'UNE',):
558##            em = 'UME'
559##        elif not em:
560##            em = "unknown"
561        da['entry_mode'] = em
562        personal = student_obj.personal
563        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
564        dp = {'Title': 'Personal Data'}
565        student_obj.invokeFactory('StudentClearance','clearance')
566        clearance = student_obj.clearance
567        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
568        dc = {'Title': 'Clearance/Eligibility Record'}
569        dc['matric_no'] = matric_no
570        state = student.State
571        lga = student.LGA
572        if state and lga:
573            lga =  state + ' / ' + lga
574        else:
575            lga = "None"
576        da['jamb_lga'] = dc['lga'] = lga
577        da['app_email'] = dp['email'] = email
578        da['app_mobile'] = dp['phone'] = phone_nr
579        dp['firstname'] = student.Firstname
580        dp['middlename'] = student.Middlename
581        dp['lastname'] = student.Lastname
582        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
583        da['jamb_sex'] = student.Sex
584        dp['sex'] = student.Sex == 'F'
585        dp['perm_address'] = student.Permanent_Address
586        application.getContent().edit(mapping=da)
587        self.portal_workflow.doActionFor(application,'close',dest_container=application)
588        personal.getContent().edit(mapping=dp)
589        clearance.getContent().edit(mapping=dc)
590        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
591##        catd = {}
592##        catd['id'] = sid
593##        catd['entry_mode']= da['entry_mode']
594##        catd['matric_no'] = matric_no
595##        catd['jamb_reg_no'] = da['jamb_reg_no']
596##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
597##        catd['sex'] = dp['sex']
598##        catd['level'] = level
599##        catd['verdict'] = verdict
600##        if certificate_brain:
601##            cpath = certificate_brain.getPath().split('/')
602##            catd['faculty'] = cpath[-4]
603##            catd['department'] = cpath[-3]
604##            catd['course'] = certcode
605##        self.students_catalog.modifyRecord(**catd)
606        #
607        # Study Course
608        #
609        student_obj.invokeFactory('StudentStudyCourse','study_course')
610        studycourse = student_obj.study_course
611        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
612        dsc = {}
613        dsc['study_course'] = certcode
614        dsc['current_level'] = level
615        dsc['current_verdict'] = verdict
616        dsc['current_mode'] = em
617        dsc['current_session'] = '06'
618        studycourse.getContent().edit(mapping=dsc)
619        #
620        # Level
621        #
622##        l = getattr(studycourse,level,None)
623##        if l is None:
624##            studycourse.invokeFactory('StudentStudyLevel', level)
625##            l = getattr(studycourse, level)
626##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
627##            l.getContent().edit(mapping={'Title': "Level %s" % level})
628###)
629
630    security.declarePublic('makeStudentLevel') ###(
631    def makeStudentLevel(self,student_id):
632        "create the StudyLevel for a returning Student"
633        #import pdb;pdb.set_trace()
634        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
635        students_folder = self.portal_url.getPortalObject().campus.students
636        res = self.students_catalog(id=student_id)
637        if res:
638            st = res[0]
639        course = st.course
640        matric_no = st.matric_no
641        level = st.level
642        res = self.results_import(matric_no = matric_no)
643        if res:
644            results = res
645        logger.info('%s creating Level %s' % (student_id,level))
646        #
647        # Level
648        #
649        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
650        studycourse = getattr(student_obj,"study_course",None)
651        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
652        l = getattr(studycourse,level,None)
653        if l is None:
654            studycourse.invokeFactory('StudentStudyLevel', level)
655            l = getattr(studycourse, level)
656            self.portal_workflow.doActionFor(l,'open',dest_container=l)
657            l.getContent().edit(mapping={'Title': "Level %s" % level})
658        ###)
659
660    security.declarePublic('getHallInfo') ###(
661    def getHallInfo(self,bed):
662        """return Hall Info"""
663        info = {}
664        hall,block,room,letter = bed.split('_')
665        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
666        if res and len(res) == 1:
667            hall_brain = res[0]
668            hall_doc = hall_brain.getObject().getContent()
669        else:
670            return info
671        info['hall_title'] = hall_brain.Title
672        info['maintenance_code'] = hall_doc.maintenance_code
673        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
674        batch_doc = None
675        for brain in res:
676            if brain.id.startswith(info['maintenance_code']):
677                batch_doc = brain.getObject().getContent()
678                break
679        if batch_doc is None:
680            info['maintenance_fee'] = None
681        else:
682            info['maintenance_fee'] = batch_doc.cost
683        return info
684    ###)
685
686    security.declarePublic('showFsPicture') ###(
687    def showFsPicture(self,path):
688        """return a picture from the filesystem"""
689        picture_path = "%s/import/%s" % (i_home,path)
690        if os.path.exists(picture_path):
691            return open(picture_path).read()
692
693    ###)
694
695    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
696    def deleteAllCourses(self,department="All"):
697        ''' delete the courses'''
698        pm = self.portal_membership
699        member = pm.getAuthenticatedMember()
700
701        if str(member) not in ("henrik","joachim"):
702            return "not possible"
703        if department == "All":
704            res = self.portal_catalog({'meta_type': 'Department'})
705        if len(res) < 1:
706            return "No Departments found"
707
708        deleted = []
709        for dep in res:
710            cf = dep.getObject().courses
711            if cf:
712                cf.manage_delObjects(ids=cf.objectIds())
713                deleted.append("deleted Courses in %s" % dep.getId)
714        return "\r".join(deleted)
715    ###)
716
717    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
718    def getLogfileLines(self,filename="event.log",numlines=20):
719        """Get last NUMLINES lines of logfile FILENAME.
720
721        Return last lines' of a file in the instances logfile directory as
722        a list. The number of returned lines equals `numlines' or less. If
723        less than `numlines' lines are available, the whole file ist
724        returned. If the file can not be opened or some other error
725        occurs, empty list is returend.
726        """
727        result = []
728        lines_hit = 0
729
730        # We only handle files in instances' log directory...
731        logpath = os.path.join(i_home, "log")
732        filename = str(os.path.abspath( os.path.join( logpath, filename )))
733        if not filename.startswith( logpath ):
734            # Attempt to access file outside log-dir...
735            return []
736
737        try:
738            fd = file( filename, "rb" )
739        except IOError:
740            return []
741        if not fd:
742            return []
743
744        if os.linesep == None:
745            linesep = '\n'
746        else:
747            linesep = os.linesep
748
749        # Try to find 'numlines' times a lineseparator, searching from end
750        # and moving to the beginning of file...
751        fd.seek( 0, 2) # Move to end of file...
752        while lines_hit < numlines:
753            if fd.read(1) == linesep[-1]: # This moves filedescriptor
754                                          # one step forward...
755                lines_hit += 1
756            try:
757                fd.seek( -2, 1) # Go two bytes back from current pos...
758            except IOError:
759                # We cannot go back two bytes. Maybe the file is too small...
760                break
761        fd.seek(2,1)
762
763        # Read all lines from current position...
764        result = fd.readlines()
765        # Remove line endings...
766        result = [x.strip() for x in result]
767        fd.close()
768        return result
769    ###)
770
771    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
772    def getCallbacksFromLog(self,filename):
773        """fix Online Payment Transactions from Z2.log entries"""
774        import transaction
775        import random
776        from cgi import parse_qs
777        from urlparse import urlparse
778        #from pdb import set_trace
779        wftool = self.portal_workflow
780        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
781        students_folder = self.portal_url.getPortalObject().campus.students
782        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
783        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
784        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
785        data = re.compile(s)
786        start = True
787        tr_count = 1
788        total = 0
789        #name = 'pume_results'
790        #name = 'epaymentsuccessful_z2log2'
791        name = filename
792        no_import = []
793        imported = []
794        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
795        try:
796            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
797        except:
798            logger.error('Error reading %s' % name)
799            return
800        tas = []
801        for line in transactions:
802            dict = {}
803            items = data.search(line)
804            dict['idict'] = idict = items.groupdict()
805            #print idict
806            #from pdb import set_trace;set_trace()
807            urlparsed = urlparse(idict['get'][4:])
808            #print urlparsed
809            path = urlparsed[2].split('/')
810            dict['student_id'] = student_id = path[8]
811            dict['payment_id'] = payment_id = path[10]
812            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
813            tas.append(dict)
814            tr_count += 1
815        return tas
816    ###)
817
818    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
819    def importOnlinePaymentTransactions(self):
820        """load Online Payment Transactions from CSV values"""
821        import transaction
822        import random
823        #from pdb import set_trace
824        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
825        opt = self.online_payments_import
826        students_folder = self.portal_url.getPortalObject().campus.students
827        start = True
828        tr_count = 1
829        total = 0
830        #name = 'pume_results'
831        name = 'OnlineTransactions'
832        no_import = []
833        imported = []
834        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
835        try:
836            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
837        except:
838            logger.error('Error reading %s.csv' % name)
839            return
840        for pay_transaction in transactions:
841            if start:
842                start = False
843                logger.info('Start loading from %s.csv' % name)
844                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
845                no_import.append('%s,"Error"' % s)
846                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
847                format_error = format + ',"%(Error)s"'
848            data = {}
849
850            # format of the first file sent by Tayo
851            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
852            #data['student_id'] = student_id = pay_transaction['Payer ID']
853            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
854            #data['response_code'] = response_code = pay_transaction['Resp Code']
855            #data['amount'] = amount = pay_transaction['Amount']
856
857            # format of the second file sent by Tayo
858            #data['datetime'] = date = 0
859            #data['student_id'] = student_id = pay_transaction['Payer ID']
860            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
861            #data['response_code'] = response_code = '00'
862            #data['amount'] = amount = pay_transaction['Amount']
863
864            # format of the third file sent by Kehinde
865            data['datetime'] = date = 0
866            data['student_id'] = student_id = pay_transaction['customer_id']
867            data['order_id'] = order_id = pay_transaction['merchant_reference']
868            data['response_code'] = response_code = '00'
869            data['amount'] = amount = pay_transaction['Amount']
870
871            dup = False
872            if response_code == "12":
873                continue
874            try:
875                opt.addRecord(**data)
876            except ValueError:
877                dup = True
878            #from pdb import set_trace;set_trace()
879            if dup:
880                if response_code == "00":
881                    try:
882                        opt.modifyRecord(**data)
883                    except:
884                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
885                        continue
886                else:
887                    pay_transaction['Error'] = "Duplicate order_id"
888                    no_import.append( format_error % pay_transaction)
889                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
890                    continue
891            tr_count += 1
892            if tr_count > 1000:
893                if len(no_import) > 0:
894                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
895                             '\n'.join(no_import) + '\n')
896                    no_import = []
897                em = '%d transactions committed\n' % (tr_count)
898                transaction.commit()
899                regs = []
900                logger.info(em)
901                total += tr_count
902                tr_count = 0
903        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
904                                                '\n'.join(no_import))
905        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
906    ###)
907
908
909    student_field2types = {   ###(
910                      'StudentApplication':
911                          {'id': 'application',
912                           'title': 'Application Data',
913                           'wf_transition_return': 'close',
914                           'wf_transition_admit': 'remain',
915                           'fields':
916                             ('jamb_reg_no',
917                              'entry_mode',
918                              'entry_session',
919                              'jamb_score',
920                              'app_email',
921                              'jamb_age',
922                              #'jamb_state',
923                              #'jamb_lga',
924                              )
925                              },
926                      #'StudentPume':
927                      #    {'id': 'pume',
928                      #     'title': 'Pume Data',
929                      #     'wf_transition_return': 'close',
930                      #     'wf_transition_admit': 'close',
931                      #     'fields':
932                      #       ('pume_score',
933                      #        )
934                      #        },
935                      'StudentClearance':
936                          {'id': 'clearance',
937                           'title': 'Clearance/Eligibility Record',
938                           'wf_transition_return': 'close',
939                           'wf_transition_admit': 'remain',
940                           'fields':
941                             ('matric_no',
942                              'nationality',
943                              'lga',
944                              'birthday',
945                              )
946                              },
947                         'StudentPersonal':
948                          {'id': 'personal',
949                           'title': 'Personal Data',
950                           'wf_transition_return': 'open',
951                           'wf_transition_admit': 'remain',
952                           'fields':
953                             ('firstname',
954                              'middlename',
955                              'lastname',
956                              'sex',
957                              'email',
958                              'phone',
959                              'perm_address',
960                              )
961                              },
962                         'StudentStudyCourse':
963                          {'id': 'study_course',
964                           'title': 'Study Course',
965                           'wf_transition_return': 'open',
966                           'wf_transition_admit': 'remain',
967                           'fields':
968                             ('study_course',
969                              'current_level',
970                              'current_session',
971                              'current_mode',
972                              'current_verdict',
973                              )
974                              },
975
976                         'PaymentsFolder':
977                          {'id': 'payments',
978                           'title': 'Payments',
979                           'wf_transition_return': 'open',
980                           'wf_transition_admit': 'open',
981                           'fields':
982                             ()
983                              },
984                         }
985    ###)
986
987
988    security.declareProtected(ModifyPortalContent,'importStudent') ###(
989    def importStudent(self,mapping):
990        "create a students record due import"
991        logger = logging.getLogger('WAeUPTool.importStudent')
992        students_folder = self.portal_url.getPortalObject().campus.students
993        jamb_reg_no = mapping.get('jamb_reg_no',None)
994        if jamb_reg_no:
995            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
996            if res:
997                return '',"jamb_reg_no exists"
998        matric_no = mapping.get('matric_no',None)
999        if matric_no:
1000            res = self.students_catalog(matric_no = matric_no)
1001            if res:
1002                return '',"matric_no exists"
1003        sid = self.waeup_tool.generateStudentId('?')
1004        students_folder.invokeFactory('Student', sid)
1005        student_obj = getattr(students_folder,sid)
1006        f2t = self.student_field2types
1007        d = {}
1008        d['jamb_sex']  = 'M'
1009        if mapping.get('sex'):
1010            d['jamb_sex']  = 'F'
1011        transition = mapping.get('reg_transition','admit')
1012        if transition not in ('admit','return'):
1013            return '',"no valid transition provided"
1014        for pt in f2t.keys():
1015            student_obj.invokeFactory(pt,f2t[pt]['id'])
1016            sub_obj = getattr(student_obj,f2t[pt]['id'])
1017            sub_doc = sub_obj.getContent()
1018            d['Title'] = f2t[pt]['title']
1019            for field in f2t[pt]['fields']:
1020                d[field] = mapping.get(field,'')
1021            if pt == "StudyCourse":
1022                for von,zu in (('entry_mode','current_mode'),
1023                               ('entry_session','current_session')):
1024                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1025                        d[zu] = mapping[von]
1026            sub_doc.edit(mapping = d)
1027
1028            #import pdb;pdb.set_trace()
1029            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1030            if new_state != "remain":
1031                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1032        self.portal_workflow.doActionFor(student_obj,transition)
1033        student_obj.manage_setLocalRoles(sid, ['Owner',])
1034        return sid,''
1035    ###)
1036
1037    security.declareProtected(ModifyPortalContent,'importEditStudent') ###(
1038    def importEditStudent(self,mapping):
1039        "edit a students record due import"
1040        logger = logging.getLogger('WAeUPTool.importEditStudent')
1041        students_folder = self.portal_url.getPortalObject().campus.students
1042        sid = mapping.get('id',None)
1043        jamb_reg_no = mapping.get('jamb_reg_no',None)
1044        matric_no = mapping.get('matric_no',None)
1045        editable_keys = mapping.keys()
1046        if sid:
1047            res = self.students_catalog(id = sid)
1048            if not res:
1049                return '',"no student with id %s" % sid
1050            elif matric_no and res[0].matric_no and\
1051              matric_no != res[0].matric_no:
1052                return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
1053            elif jamb_reg_no and res[0].jamb_reg_no and\
1054              jamb_reg_no != res[0].jamb_reg_no:
1055                return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
1056        elif jamb_reg_no:
1057            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1058            if not res:
1059                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1060            editable_keys.remove('jamb_reg_no')
1061        elif matric_no:
1062            res = self.students_catalog(matric_no = matric_no)
1063            if not res:
1064                return '',"no student with matric_no %s" % matric_no
1065            editable_keys.remove('matric_no')
1066
1067        # included only to change wf state from admitted to returning
1068        if res[0].review_state not in ('admitted','objection_raised'):
1069            return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1070        # end inclusion
1071
1072        sid = res[0].id
1073        student_obj = getattr(students_folder,sid)
1074        f2t = self.student_field2types
1075        d = {}
1076        #import pdb;pdb.set_trace()
1077        any_change = False
1078        for pt in f2t.keys():
1079            if pt == "student_application":
1080                d['jamb_sex']  = 'M'
1081                if mapping.get('sex'):
1082                    d['jamb_sex']  = 'F'
1083            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1084            if intersect:
1085                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1086                if sub_obj is None:
1087                    try:
1088                        student_obj.invokeFactory(pt,f2t[pt]['id'])
1089                    except:
1090                        continue
1091                    sub_obj = getattr(student_obj,f2t[pt]['id'])
1092                    d['Title'] = f2t[pt]['title']
1093                sub_doc = sub_obj.getContent()
1094                for field in intersect:
1095                    changed = False
1096                    if getattr(sub_doc,field) != mapping.get(field,''):
1097                        any_change = True
1098                        changed = True
1099                        d[field] = mapping.get(field,'')
1100                    if changed:
1101                        sub_doc.edit(mapping = d)
1102
1103
1104        # included only to change wf state from admitted to returning
1105            if res[0].review_state in ('admitted','objection_raised'):
1106                new_state = f2t[pt]['wf_transition_return']
1107                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1108                if sub_obj and new_state != "remain":
1109                    try:
1110                        self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1111                    except:
1112                        #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1113                        pass
1114        if res[0].review_state in ('admitted','objection_raised'):
1115            wfaction = 'return'
1116            try:
1117                self.portal_workflow.doActionFor(student_obj,wfaction)
1118                logger.info('%s, wf state changed' % sid)
1119                any_change = True
1120            except:
1121                logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1122                pass
1123        # end inclusion
1124
1125
1126        if any_change:
1127            return sid,''
1128        else:
1129            return sid,'not modified'
1130    ###)
1131
1132
1133    security.declareProtected(ModifyPortalContent,"importData")###(
1134    def importData(self,filename,name,edit=False):
1135        """load data from CSV values"""
1136        import transaction
1137        import random
1138
1139        pm = self.portal_membership
1140        member = pm.getAuthenticatedMember()
1141
1142        logger = logging.getLogger('WAeUPTool.importData')
1143        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1144        students_folder = self.portal_url.getPortalObject().campus.students
1145        start = True
1146        tr_count = 1
1147        total_imported = 0
1148        total_not_imported = 0
1149        total = 0
1150        iname = "import_%s" % name
1151        stool = getToolByName(self, 'portal_schemas')
1152        ltool = getToolByName(self, 'portal_layouts')
1153        schema = stool._getOb(iname)
1154        if schema is None:
1155            em = 'No such schema %s' % iname
1156            logger.error('No such schema %s' % iname)
1157            return em
1158        layout = ltool._getOb(iname)
1159        if layout is None:
1160            em = 'No such layout %s' % iname
1161            logger.error(em)
1162            return em
1163        validators = {}
1164        for widget in layout.keys():
1165            validators[widget] = layout[widget].validate
1166        if edit:
1167            importer_name = "importEdit%s" % name.capitalize()
1168        else:
1169            importer_name = "import%s" % name.capitalize()
1170        importer = getattr(self, '%s' % importer_name,None)
1171        if importer is None:
1172            em = 'No importer function %s' % importer_name
1173            logger.error(em)
1174            return em
1175        not_imported = []
1176        imported = []
1177        try:
1178            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1179        except:
1180            em = 'Error reading %s.csv' % filename
1181            logger.error(em)
1182            return em
1183        for item in items:
1184            if start:
1185                start = False
1186                adapters = [MappingStorageAdapter(schema, item)]
1187                logger.info('%s starts import from %s.csv' % (member,filename))
1188                #import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1189                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1190                #import pdb;pdb.set_trace()
1191                import_keys = [k for k in attrs if not k.startswith('ignore')]
1192                diff2schema = set(import_keys).difference(set(schema.keys()))
1193                diff2layout = set(import_keys).difference(set(layout.keys()))
1194                if diff2schema:
1195                    em = "not ignorable key(s) %s found in heading" % diff2schema
1196                    return em
1197                s = ','.join(['"%s"' % fn for fn in import_keys])
1198                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1199                s = '"id",' + s
1200                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1201                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1202                format_error = format + ',"%(Error)s"'
1203                format = '"%(id)s",'+ format
1204            dm = DataModel(item, adapters,context=self)
1205            ds = DataStructure(data=item,datamodel=dm)
1206            error_string = ""
1207            for k in import_keys:
1208                if not validators[k](ds):
1209                    error_string += " %s : %s" % (k,ds.getError(k))
1210            if not error_string:
1211                item.update(dm)
1212                item['id'],error = importer(item)
1213                if error:
1214                    error_string += error
1215            if error_string:
1216                item['Error'] = error_string
1217                not_imported.append(format_error % item)
1218                total_not_imported += 1
1219            else:
1220                em = format % item
1221                imported.append(em)
1222                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1223                tr_count += 1
1224                total_imported += 1
1225            total += 1
1226            if total and not total % 100:
1227                transaction.commit()
1228                if len(not_imported) > 0:
1229                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1230                             '\n'.join(not_imported) + '\n')
1231                    not_imported = []
1232                if len(imported) > 0:
1233                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1234                             '\n'.join(imported) + '\n')
1235                    imported = []
1236                em = '%d transactions committed\n' % (tr_count)
1237                regs = []
1238                logger.info(em)
1239                tr_count = 0
1240        if len(imported) > 0:
1241            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1242                                                '\n'.join(imported))
1243        if len(not_imported) > 0:
1244            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1245                                                '\n'.join(not_imported))
1246        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1247        logger.info(em)
1248        return em
1249    ###)
1250
1251InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.