source: WAeUP_SRP/base/WAeUPTool.py @ 2635

Last change on this file since 2635 was 2632, checked in by joachim, 17 years ago

add student_levels, fix for #397,cleanup removeUnusedDocIds
  • Property svn:keywords set to Id
File size: 75.3 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2632 2007-11-12 17:23:46Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56
57def getObject(object,name):
58    if object.hasObject(name):
59        return getattr(object,name)
60    return None
61
62class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
63    """WAeUP tool"""
64
65    id = 'waeup_tool'
66    meta_type = 'WAeUP Tool'
67    _actions = ()
68    security = ClassSecurityInfo()
69    security.declareObjectProtected(View)
70    manage_options = ( ActionProviderBase.manage_options
71                     + SimpleItem.manage_options
72                     )
73
74    def rwrite(self,s): ###(
75        response = self.REQUEST.RESPONSE
76        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
77        response.write("%s<br />\r\n" % s)
78    ###)
79
80    def sleep(self,secs): ###(
81        "sleep"
82        import time
83        time.sleep(secs)
84        return
85
86###)
87
88    security.declareProtected(ModifyPortalContent,'openLog') ###(
89    def openLog(self,name):
90        """open a log file"""
91        version = 1
92        path = "%s/log/%s_%d.log" % (i_home,name,version)
93        while os.path.exists(path):
94            version += 1
95            path = "%s/log/%s_%d.log" % (i_home,name,version)
96        log = open(path,"w")
97        return log
98    ###)
99
100    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
101    def bypassQueueCatalog(self,enable=True):
102        """bypass the QueueCatalog by setting all indexes to process imediate,
103        if enable is True (default) the old settings are restored
104        """
105
106    ###)
107
108    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
109    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
110        """measure Object access Time"""
111        import random
112        if hasattr(self,'portal_catalog_real'):
113            aq_portal = self.portal_catalog_real.evalAdvancedQuery
114        else:
115            aq_portal = self.portal_catalog.evalAdvancedQuery
116        nr_pts = int(nr_pts)
117        probe = int(probe)
118        intervall = probe/10
119        objects = ("application","clearance","personal")
120        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
121        #i = random.randrange(num_objects)
122        count = 0
123        found = 0
124        not_found = 0
125        t_found = 0
126        t_not_found = 0
127        time_found = time_not_found = 0.0
128        t_time_found = t_time_not_found = 0.0
129        accessed = []
130        t_min = 1000
131        t_max = 0
132        #import pdb;pdb.set_trace()
133        students = self.portal_catalog(portal_type="Student")
134        num_students = len(students)
135        if method == "d":
136            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
137            res = aq_portal(query)
138            brains = {}
139            for r in res:
140                sid = r.relative_path.split('/')[-2]
141                if brains.has_key(sid):
142                    brains[sid][r.portal_type] = r
143                else:
144                    brains[sid] = {r.portal_type : r}
145            brains_list = brains.keys()
146            num_objects = len(brains_list)
147        else:
148            num_objects = num_students
149        print "="*40
150        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
151                                                                        probe,
152                                                                        nr_pts,
153                                                                        num_objects)
154        print "nr found/not time found/not min/max"
155        elapse = time.time()
156        i_elapse = time.time()
157        c_elapse = time.clock()
158        for c in range(1,probe + 1):
159            i = random.randrange(num_objects)
160            if method in ('a','b','c'):
161                student_brain = students[i]
162            elif method == "d":
163                #import pdb;pdb.set_trace()
164                student_brain = brains[brains_list[i]]
165            if method == "c":
166                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
167                res = aq_portal(query)
168                this_portal_types = [r.portal_type for r in res]
169            for i in range(nr_pts):
170                oid = objects[i]
171                if method == "a":
172                    try:
173                        student_path = student_brain.getPath()
174                        path = "%s/%s" % (student_path,oid)
175                        doc = self.unrestrictedTraverse(path).getContent()
176                        found += 1
177                        i_time = time.time() - i_elapse
178                        time_found += i_time
179                    except:
180                        not_found += 1
181                        i_time = time.time() - i_elapse
182                        time_not_found += i_time
183                        pass
184                elif method == "b":
185                    try:
186                        student_object = student_brain.getObject()
187                        doc = getattr(student_object,oid).getContent()
188                        found += 1
189                        i_time = time.time() - i_elapse
190                        time_found += i_time
191                    except:
192                        i_time = time.time() - i_elapse
193                        time_not_found += i_time
194                        not_found += 1
195                        pass
196                elif method == "c":
197                    if portal_types[i] in this_portal_types:
198                        found += 1
199                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
200                        i_time = time.time() - i_elapse
201                        time_found += i_time
202                    else:
203                        not_found += 1
204                        i_time = time.time() - i_elapse
205                        time_not_found += i_time
206                elif method == "d":
207                    if student_brain.has_key(portal_types[i]):
208                        found += 1
209                        doc = student_brain[portal_types[i]].getObject().getContent()
210                        i_time = time.time() - i_elapse
211                        time_found += i_time
212                    else:
213                        not_found += 1
214                        i_time = time.time() - i_elapse
215                        time_not_found += i_time
216                i_elapse = time.time()
217            if c and (c % intervall == 0):
218                #i_time = time.time() - i_elapse
219                t_per = 0.0
220                if found:
221                    t_per = time_found/found
222                if t_per > t_max:
223                    t_max = t_per
224                if t_per > 0.0 and t_per < t_min:
225                    t_min = t_per
226                itf = 0.0
227                if found:
228                    itf = time_found/found
229                itnf = 0.0
230                if not_found :
231                    itnf = time_not_found / not_found
232                interval_time = time_found + time_not_found
233                s = "%(c)d: %(found)d/%(not_found)d " % vars()
234                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
235                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
236                print s
237                t_found += found
238                t_not_found += not_found
239                t_time_found += time_found
240                t_time_not_found += time_not_found
241                time_found = time_not_found = 0.0
242                found = not_found = 0
243        # t_found += found
244        # t_not_found += not_found
245        elapse = time.time() - elapse
246        itf = 0.0
247        if t_found:
248            itf = t_time_found/t_found
249        itnf = 0.0
250        if t_not_found:
251            itnf = t_time_not_found / t_not_found
252        #c_elapse = time.clock() - c_elapse
253        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
254        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
255        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
256        print "-"*40
257        print s
258        rel_found = float(t_found)/probe
259        rel_not_found = float(t_not_found)/probe
260        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
261        print estimated_total_time
262    ###)
263
264    security.declareProtected(ModifyPortalContent,'writeLog') ###(
265    def writeLog(self,logfile,s):
266        """write to the log file"""
267        logfile.write(s)
268
269###)
270
271    def generateStudentId(self,letter): ###(
272        import random
273        r = random
274        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
275        if letter == '?':
276            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
277        sid = "%c%d" % (letter,r.randint(99999,1000000))
278        students = self.portal_url.getPortalObject().campus.students
279##        while hasattr(students, sid):
280##            sid = "%c%d" % (letter,r.randint(99999,1000000))
281        while self.students_catalog(id = sid):
282            sid = "%c%d" % (letter,r.randint(99999,1000000))
283        return sid
284    ###)
285
286    def generatePassword(self,s=None): ###(
287        import random
288        r = random
289        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
290        if s is None:
291            s = 'abcdefghklmnpqrstuvwxy23456789'
292        pw = ''
293        while len(pw) < 6:
294            pw += r.choice(s)
295        return pw
296    ###)
297
298    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
299    def removeDeletedDocIds(self, max=1000):
300        """
301        remove deleted docids from repository commit after max
302        """
303        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
304        repository = getToolByName(self, 'portal_repository')
305        pxtool = getToolByName(self, 'portal_proxies')
306        logger.info('start')
307        pxtool_infos = pxtool.getRevisionsUsed()
308        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
309
310        nb_revs = 0
311        docids_d = {} # all docids
312        unused_docids_d = {} # all docids that are unused
313        ids_unused_revs_docids = [] # ids for revs of unused docids
314        unused_ids = [] # ids for unused revs
315        total = 0
316        idlist = repository.objectIds()
317        to_delete = 0
318        found = False
319        for id in idlist:
320            docid, rev = repository._splitId(id)
321            if docid is None:
322                logger.info("invalid doc_id %s" % docid)
323                continue
324            nb_revs += 1
325            if not pxtool_infos.has_key(docid):
326                found = True
327                to_delete += 1
328                unused_ids.append(id)
329            elif not pxtool_infos[docid].has_key(rev):
330                found = True
331                to_delete += 1
332                unused_ids.append(id)
333            if found and not to_delete % max:
334                found = False
335                #import pdb;pdb.set_trace()
336                repository.manage_delObjects(unused_ids)
337                transaction.commit()
338                logger.info('removed %d total %d unused docids ' % (max,to_delete))
339        else:
340            if unused_ids:
341                repository.manage_delObjects(unused_ids)
342                transaction.commit()
343        logger.info('finished removing %d unused docids ' % (to_delete))
344
345
346###)
347
348    security.declareProtected(View,'getCredential') ###(
349    def getCredential(self,student_id):
350        "return a student password"
351        student_entry = getattr(self.portal_directories.students,student_id,None)
352        if not self.isStaff():
353            mtool = self.portal_membership
354            member = mtool.getAuthenticatedMember()
355            logger = logging.getLogger('WAeUPTool.getCredential')
356            logger.info('%s tried to access password of %s' % (member,student_id))
357            return None
358        if student_entry is None:
359            return None
360        return getattr(student_entry,"password","not set")
361    ###)
362
363    security.declarePublic('checkPassword') ###(
364    def checkPassword(self,student_id,password):
365        "return a student password"
366        student_entry = getattr(self.portal_directories.students,student_id,None)
367        if student_entry is None:
368            return False
369        return getattr(student_entry,"password","not set") == password
370    ###)
371
372    security.declareProtected(ModifyPortalContent,'editPassword') ###(
373    def editPassword(self,student_id,password):
374        "edit a student password"
375        student_entry = getattr(self.portal_directories.students,student_id,None)
376        if student_entry is None:
377            return
378        setattr(student_entry,'password',password)
379    ###)
380
381    security.declareProtected(ModifyPortalContent,'doCommit') ###(
382    def doCommit(self,logger=None):
383        "commit some transactions"
384        transaction.commit()
385    ###)
386
387    security.declarePublic('loadStudentFoto') ###(
388    def loadStudentFoto(self,student,filename,folder):
389        "return a student passport picture"
390        #import pdb;pdb.set_trace()
391        picture ="%s/import/%s/%s" % (i_home,folder,filename)
392        student_id = student.getId()
393        images_dir = os.path.join("%s" % images_base,student_id)
394        if not os.path.exists(images_dir):
395            os.mkdir(images_dir)
396        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
397        for extension in ('.jpg','.JPG'):
398            fullname = "%(picture)s%(extension)s" % vars()
399            if os.path.exists(fullname):
400                copy2(fullname,image_name)
401                return "successfully copied passport picture"
402        return "passport picture not found: %s.jpg or .JPG" % picture
403    ###)
404
405    def old____loadStudentFoto(self,student,filename,folder): ###(
406        "return a student passport picture"
407        app = student.application
408        app_doc = app.getContent()
409        #clear = student.clearance
410        #clear_doc = clear.getContent()
411        #matric_no = clear_doc.matric_no.upper()
412        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
413        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
414        #import pdb;pdb.set_trace()
415        if os.path.exists(picture1):
416            file = open(picture1)
417        elif os.path.exists(picture2):
418            file = open(picture2)
419        else:
420            return "passport picture not found %s" % picture1
421        reopened = False
422        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
423            self.portal_workflow.doActionFor(app,'open')
424            reopened = True
425        outfile = file.read()
426        app_doc.manage_addFile('passport',
427                               file=outfile,
428                               title="%s.jpg" % filename)
429        if reopened:
430            self.portal_workflow.doActionFor(app,'close')
431        return "successfully loaded passport picture"
432    ###)
433
434    security.declareProtected(ModifyPortalContent,'createOne') ###(
435    def createOne(self,students_folder,student_brain,letter,commit=False):
436        sid = self.waeup_tool.generateStudentId(letter)
437        students_folder.invokeFactory('Student', sid)
438        student = getattr(students_folder,sid)
439        self.portal_workflow.doActionFor(student,'return')
440        student.manage_setLocalRoles(sid, ['Owner',])
441        matric_no = student_brain.matric_no
442        jamb_reg_no = student_brain.Entryregno
443        self.students_catalog.addRecord(id = sid,
444                                           matric_no = matric_no,
445                                           jamb_reg_no = jamb_reg_no,
446                                           sex = student_brain.Sex == "F",
447                                           name = "%s %s %s" % (student_brain.Firstname,
448                                                                student_brain.Middlename,
449                                                                student_brain.Lastname)
450                                        )
451        if commit:
452            transaction.commit()
453        return sid,jamb_reg_no
454    ###)
455
456    security.declareProtected(ModifyPortalContent,'addStudent') ###(
457    def addStudent(self,dict):
458        students_folder = self.portal_url.getPortalObject().campus.students
459        sid = self.waeup_tool.generateStudentId('?')
460        students_folder.invokeFactory('Student', sid)
461        student_obj = getattr(students_folder,sid)
462        f2t = self.student_field2types
463        #from pdb import set_trace; set_trace()
464        d = {}
465        #d['jamb_sex']  = 'M'
466        #if dict.get('sex'):
467        #    d['jamb_sex']  = 'F'
468
469        entry_session = dict.get('entry_session')
470        if entry_session == self.getSessionId()[0]:
471            wfaction = 'admit'
472            wft = 'wf_transition_admit'
473            password = None
474        else:
475            wfaction = 'return'
476            wft = 'wf_transition_return'
477            password = self.generatePassword()
478            self.makeStudentMember(sid,password)
479
480        for pt in f2t.keys():
481            student_obj.invokeFactory(pt,f2t[pt]['id'])
482            sub_obj = getattr(student_obj,f2t[pt]['id'])
483            sub_doc = sub_obj.getContent()
484            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
485            d['Title'] = f2t[pt]['title']
486            for field in f2t[pt]['fields']:
487                d[field] = dict.get(field,'')
488            sub_doc.edit(mapping = d)
489            new_state = f2t[pt][wft]
490            if new_state != "remain":
491                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
492        self.portal_workflow.doActionFor(student_obj,wfaction)
493        student_obj.manage_setLocalRoles(sid, ['Owner',])
494        return sid,password
495    ###)
496
497    security.declarePublic('getCertificateBrain') ###(
498    def getCertificateBrain(self,cert_id):
499        "do it"
500        res = ZCatalog.searchResults(self.portal_catalog_real,
501                                {'portal_type':"Certificate",
502                                      'id': cert_id})
503        if res:
504            return res[0]
505        return None
506    ###)
507
508    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
509    def get_csv_filenames(self):
510        "do it"
511        files = [file for file in os.listdir("%s/import/" % (i_home))
512                 if file.endswith('.csv') and file.find('imported') == -1]
513        return files
514    ###)
515
516    security.declarePublic('findStudentByMatricelNo') ###(
517    def findStudentByMatricelNo(self,matric_no):
518        "do it"
519        res = ZCatalog.searchResults(self.portal_catalog_real,
520                                {'portal_type':"StudentClearance",
521                                 'SearchableText': matric_no})
522        if res:
523            return res[0]
524        return None
525    ###)
526
527    security.declarePublic('makeStudentMember') ###(
528    def makeStudentMember(self,sid,password='uNsEt'):
529        """make the student a member"""
530        membership = self.portal_membership
531        membership.addMember(sid,
532                             password ,
533                             roles=('Member',
534                                     'Student',
535                                     ),
536                             domains='',
537                             properties = {'memberareaCreationFlag': False,
538                                           'homeless': True},)
539        member = membership.getMemberById(sid)
540        self.portal_registration.afterAdd(member, sid, password, None)
541        #self.manage_setLocalRoles(sid, ['Owner',])
542    ###)
543
544    security.declareProtected(View,'makeStudentData') ###(
545    def makeStudentData(self,student_id,email=None,phone_nr=None):
546        "create Datastructure for a returning Student"
547        #import pdb;pdb.set_trace()
548        logger = logging.getLogger('WAeUPTool.makeStudentData')
549        students_folder = self.portal_url.getPortalObject().campus.students
550        #res = self.students_catalog(id=student_id)
551        #if res:
552        #    st = res[0]
553        #res = self.returning_import(matric_no = st.matric_no)
554        res = self.returning_import(id = student_id)
555        if res:
556            student = res[0]
557        else:
558            logger.info('Id %s not found in returning_import' % student_id)
559            return
560        logger.info('%s creates data structure' % student_id)
561        s_results = self.results_import(matric_no = student.matric_no)
562        if s_results:
563            lnr = self.getLevelFromResultsCosCode(s_results)
564            level = "%d00" % lnr
565            verdict,eligible = self.getVerdict(s_results[0].Verdict)
566            #if eligible:
567            #    level = "%d00" % (lnr + 1)
568        else:
569            logger.info('matric_no %s not found in results_import' % student.matric_no)
570            level = 0
571            verdict = 'N/A'
572        #student should not be allowed to perform this transition
573        #wftool = self.portal_workflow
574        #wftool.doActionFor(student,'return')
575        certcode_org = student.Coursemajorcode
576        certcode = makeCertificateCode(certcode_org)
577        certificate_brain = self.getCertificateBrain(certcode)
578        if not certificate_brain:
579            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
580            logger.info(em)
581        matric_no = student.matric_no
582        sid = student_id
583        student_obj = getattr(students_folder,sid)
584        student_obj.invokeFactory('StudentApplication','application')
585        application = student_obj.application
586        self.portal_workflow.doActionFor(application,'open',dest_container=application)
587        da = {'Title': 'Application Data'}
588        student_obj.invokeFactory('StudentPersonal','personal')
589        da['jamb_reg_no'] = student.Entryregno
590        em = self.getEntryMode(student.Entryregno)
591        da['entry_mode'] = em
592        personal = student_obj.personal
593        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
594        dp = {'Title': 'Personal Data'}
595        student_obj.invokeFactory('StudentClearance','clearance')
596        clearance = student_obj.clearance
597        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
598        dc = {'Title': 'Clearance/Eligibility Record'}
599        dc['matric_no'] = matric_no
600        state = student.State
601        lga = student.LGA
602        if state and lga:
603            lga =  state + ' / ' + lga
604        else:
605            lga = "None"
606        da['jamb_lga'] = dc['lga'] = lga
607        da['app_email'] = dp['email'] = email
608        da['app_mobile'] = dp['phone'] = phone_nr
609        dp['firstname'] = student.Firstname
610        dp['middlename'] = student.Middlename
611        dp['lastname'] = student.Lastname
612        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
613        da['jamb_sex'] = student.Sex
614        dp['sex'] = student.Sex == 'F'
615        dp['perm_address'] = student.Permanent_Address
616        application.getContent().edit(mapping=da)
617        self.portal_workflow.doActionFor(application,'close',dest_container=application)
618        personal.getContent().edit(mapping=dp)
619        clearance.getContent().edit(mapping=dc)
620        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
621        #
622        # Study Course
623        #
624        student_obj.invokeFactory('StudentStudyCourse','study_course')
625        studycourse = student_obj.study_course
626        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
627        dsc = {}
628        dsc['study_course'] = certcode
629        dsc['current_level'] = level
630        dsc['current_verdict'] = verdict
631        dsc['current_mode'] = em
632        dsc['current_session'] = '05'
633        studycourse.getContent().edit(mapping=dsc)
634        #
635        # Level
636        #
637##        l = getattr(studycourse,level,None)
638##        if l is None:
639##            studycourse.invokeFactory('StudentStudyLevel', level)
640##            l = getattr(studycourse, level)
641##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
642##            l.getContent().edit(mapping={'Title': "Level %s" % level})
643###)
644
645    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
646    def admitOneStudent(self,brain,entry_session,pin_password):
647        "create Datastructure for an admitted Student"
648        #import pdb;pdb.set_trace()
649        students_folder = self.portal_url.getPortalObject().campus.students
650        logger = logging.getLogger('WAeUPTool.admitStudent')
651        if brain.status != "admitted":
652            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
653            return
654        pin_parts = brain.pin.split('-')
655        if len(pin_parts) != 3:
656            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
657            return
658        student_id = self.generateStudentId('?')
659        students_folder.invokeFactory('Student', student_id)
660        student_object = getattr(students_folder,student_id)
661        if pin_password:
662            password = pin_parts[2]
663            self.makeStudentMember(student_id,password = password)
664        student_object.manage_setLocalRoles(student_id, ['Owner',])
665        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
666        #
667        # application
668        #
669        student_object.invokeFactory('StudentApplication','application')
670        application = student_object.application
671        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
672        da = {'Title': 'Application Data'}
673        da['jamb_reg_no'] = brain.reg_no
674
675        sex = 'M'
676        if brain.sex:
677            sex = 'F'
678        da['jamb_sex'] = sex
679        #da['app_ac_pin'] = brain.pin
680        #if brain.lga:
681        #    state_lga = brain.lga.split('_')
682        #    da['state_lga'] = state_lga[0]
683        #    da['jamb_lga'] = state_lga[-1]
684        da['state_lga'] = brain.jamb_lga
685        da['jamb_lga'] = brain.jamb_state
686        da['jamb_score'] = brain.aggregate
687        da['app_email'] = brain.email
688        da['app_mobile'] = brain.phone
689        if brain.entry_mode:
690            da['entry_mode'] = brain.entry_mode
691        elif brain.screening_type == 'pume':
692            da['entry_mode'] = 'ume_ft'
693        elif brain.screening_type == 'pde':
694            da['entry_mode'] = 'de_ft'
695        else:
696            da['entry_mode'] = brain.screening_type
697        da['entry_session'] = entry_session
698        da['jamb_lastname'] = brain.lastname
699        da['jamb_middlename'] = brain.middlenames   # different field names!
700        da['firstname'] = brain.firstname
701        da['screening_application_date'] = brain.application_date
702        da['date_of_birth'] = brain.date_of_birth
703        da['jamb_first_cos'] = brain.course1
704        da['jamb_second_cos'] = brain.course2
705        da['course3'] = brain.course3
706        da['screening_type'] = brain.screening_type
707        da['screening_score'] = brain.screening_score
708        da['screening_date'] = brain.screening_date
709        da['hq_type'] = brain.hq_type
710        da['hq_grade'] = brain.hq_grade
711        da['aos'] = brain.aos
712
713        application.getContent().edit(mapping=da)
714        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
715        #
716        # personal
717        #
718        student_object.invokeFactory('StudentPersonal','personal')
719        personal = student_object.personal
720        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
721        dp = {'Title': 'Personal Data'}
722        dp['sex'] = brain.sex
723        dp['email'] = brain.email
724        dp['phone'] = brain.phone
725        dp['lastname'] = brain.lastname
726        dp['middlename'] = brain.middlenames   # different field names!
727        dp['firstname'] = brain.firstname
728        personal.getContent().edit(mapping=dp)
729        #
730        # clearance
731        #
732        student_object.invokeFactory('StudentClearance','clearance')
733        clearance = student_object.clearance
734        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
735        dc = {'Title': 'Clearance/Eligibility Record'}
736        dc['lga'] = brain.lga
737        clearance.getContent().edit(mapping=dc)
738        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
739        #
740        # study Course
741        #
742        student_object.invokeFactory('StudentStudyCourse','study_course')
743        studycourse = student_object.study_course
744        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
745        dsc = {}
746        dsc['study_course'] = brain.course_admitted
747        dsc['current_verdict'] = ''
748        dsc['current_mode'] = da['entry_mode']
749        if da['entry_mode'].startswith('de'):
750            dsc['current_level'] = '200'
751        else:
752            dsc['current_level'] = '100'
753        dsc['current_session'] = entry_session
754        studycourse.getContent().edit(mapping=dsc)
755        #
756        # payments folder
757        student_object.invokeFactory('PaymentsFolder','payments')
758        payments = getattr(student_object,'payments')
759        dpay = {}
760        dpay['Title'] = 'Payments'
761        payments.getContent().edit(mapping=dpay)
762        self.portal_workflow.doActionFor(payments,'open')
763        #
764        # passport foto
765        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
766                                                              brain.screening_type,
767                                                              brain.reg_no)
768        images_dir = os.path.join("%s" % images_base,student_id)
769        if not os.path.exists(images_dir):
770            os.mkdir(images_dir)
771        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
772        if os.path.exists(app_picture):
773            copy2(app_picture,image_name)
774        else:
775            logger.info('passport of %s/%s not found: %s' % (student_id,
776                                                             brain.reg_no,
777                                                             app_picture))
778           
779        return student_id
780    ###)
781
782    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
783    def makeStudentLevel(self,student_id):
784        "create the StudyLevel for a returning Student"
785        #import pdb;pdb.set_trace()
786        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
787        students_folder = self.portal_url.getPortalObject().campus.students
788        res = self.students_catalog(id=student_id)
789        if res:
790            st = res[0]
791        course = st.course
792        matric_no = st.matric_no
793        level = st.level
794        res = self.results_import(matric_no = matric_no)
795        if res:
796            results = res
797        logger.info('%s creating Level %s' % (student_id,level))
798        #
799        # Level
800        #
801        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
802        studycourse = getattr(student_obj,"study_course",None)
803        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
804        l = getattr(studycourse,level,None)
805        if l is None:
806            studycourse.invokeFactory('StudentStudyLevel', level)
807            l = getattr(studycourse, level)
808            self.portal_workflow.doActionFor(l,'open',dest_container=l)
809            l.getContent().edit(mapping={'Title': "Level %s" % level})
810        ###)
811
812    security.declarePublic('getHallInfo') ###(
813    def getHallInfo(self,bed):
814        """return Hall Info"""
815        info = {}
816        hall,block,room,letter = bed.split('_')
817        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
818        if res and len(res) == 1:
819            hall_brain = res[0]
820            hall_doc = hall_brain.getObject().getContent()
821        else:
822            return info
823        info['hall_title'] = hall_brain.Title
824        info['maintenance_code'] = hall_doc.maintenance_code
825        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
826        batch_doc = None
827        for brain in res:
828            if brain.id.startswith(info['maintenance_code']):
829                batch_doc = brain.getObject().getContent()
830                break
831        if batch_doc is None:
832            info['maintenance_fee'] = None
833        else:
834            info['maintenance_fee'] = batch_doc.cost
835        return info
836    ###)
837
838    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
839    def removePictureFolder(self,student_id):
840        """remove picture_folder by renaming it"""
841        path = 'images'
842        picture_path = os.path.join(i_home,path,student_id)
843        if not os.path.exists(picture_path):
844            return False
845        os.rename(picture_path,picture_path + "_removed")
846        return True
847    ###)
848
849    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
850    def restorePictureFolder(self,student_id):
851        """restore picture_folder by renaming it"""
852        path = 'images'
853        picture_path = os.path.join(i_home,path,student_id)
854        if not os.path.exists(picture_path + "_removed"):
855            return False
856        os.rename(picture_path + "_removed",picture_path)
857        return True
858    ###)
859
860    security.declarePublic('picturesExist') ###(
861    def picturesExist(self, ids,student_id=None):
862        """check if pictures exist in the filesystem"""
863        if student_id is None:
864            student_id = self.getStudentId()
865        if student_id is None:
866            return False
867        picture_path = os.path.join(images_base,student_id)
868        if not os.path.exists(picture_path):
869            return False
870        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
871        return set(ids).issubset(set(pictures))
872    ###)
873
874    security.declarePublic('picturesList') ###(
875    def picturesList(self):
876        """check if pictures exist in the filesystem"""
877        path = 'images'
878        student_id = self.getStudentId()
879        picture_path = os.path.join(i_home,path,student_id)
880        if not os.path.exists(picture_path):
881            return []
882        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
883    ###)
884
885    security.declarePublic('showFsPicture') ###(
886    def showFsPicture(self,path):
887        """return a picture from the filesystem"""
888        picture_path = os.path.join(i_home,path)
889        response = self.REQUEST.RESPONSE
890        #import pdb;pdb.set_trace()
891        registry = getToolByName(self, 'mimetypes_registry')
892        mimetype = str(registry.lookupExtension(path.lower()) or
893                    registry.lookupExtension('file.bin'))
894        if os.path.exists(picture_path):
895            response.setHeader('Content-type',mimetype)
896            return open(picture_path).read()
897        picture_path = os.path.join(i_home,'import',path)
898        if os.path.exists(picture_path):
899            return open(picture_path).read()
900    ###)
901
902    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
903    def deleteAllCourses(self,department="All"):
904        ''' delete the courses'''
905        pm = self.portal_membership
906        member = pm.getAuthenticatedMember()
907
908        if str(member) not in ("henrik","joachim"):
909            return "not possible"
910        if department == "All":
911            res = self.portal_catalog({'meta_type': 'Department'})
912        if len(res) < 1:
913            return "No Departments found"
914
915        deleted = []
916        for dep in res:
917            cf = dep.getObject().courses
918            if cf:
919                cf.manage_delObjects(ids=cf.objectIds())
920                deleted.append("deleted Courses in %s" % dep.getId)
921        return "\r".join(deleted)
922    ###)
923
924    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
925    def getLogfileLines(self,filename="event.log",numlines=20):
926        """Get last NUMLINES lines of logfile FILENAME.
927
928        Return last lines' of a file in the instances logfile directory as
929        a list. The number of returned lines equals `numlines' or less. If
930        less than `numlines' lines are available, the whole file ist
931        returned. If the file can not be opened or some other error
932        occurs, empty list is returend.
933        """
934        result = []
935        lines_hit = 0
936
937        # We only handle files in instances' log directory...
938        logpath = os.path.join(i_home, "log")
939        filename = str(os.path.abspath( os.path.join( logpath, filename )))
940        if not filename.startswith( logpath ):
941            # Attempt to access file outside log-dir...
942            return []
943
944        try:
945            fd = file( filename, "rb" )
946        except IOError:
947            return []
948        if not fd:
949            return []
950
951        if os.linesep == None:
952            linesep = '\n'
953        else:
954            linesep = os.linesep
955
956        # Try to find 'numlines' times a lineseparator, searching from end
957        # and moving to the beginning of file...
958        fd.seek( 0, 2) # Move to end of file...
959        while lines_hit < numlines:
960            if fd.read(1) == linesep[-1]: # This moves filedescriptor
961                                          # one step forward...
962                lines_hit += 1
963            try:
964                fd.seek( -2, 1) # Go two bytes back from current pos...
965            except IOError:
966                # We cannot go back two bytes. Maybe the file is too small...
967                break
968        fd.seek(2,1)
969
970        # Read all lines from current position...
971        result = fd.readlines()
972        # Remove line endings...
973        result = [x.strip() for x in result]
974        fd.close()
975        return result
976    ###)
977
978    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
979    def getCallbacksFromLog(self,filename):
980        """fix Online Payment Transactions from Z2.log entries"""
981        import transaction
982        import random
983        from cgi import parse_qs
984        from urlparse import urlparse
985        #from pdb import set_trace
986        wftool = self.portal_workflow
987        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
988        students_folder = self.portal_url.getPortalObject().campus.students
989        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
990        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
991        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
992        data = re.compile(s)
993        start = True
994        tr_count = 1
995        total = 0
996        #name = 'pume_results'
997        #name = 'epaymentsuccessful_z2log2'
998        name = filename
999        no_import = []
1000        imported = []
1001        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1002        try:
1003            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1004        except:
1005            logger.error('Error reading %s' % name)
1006            return
1007        tas = []
1008        for line in transactions:
1009            dict = {}
1010            items = data.search(line)
1011            dict['idict'] = idict = items.groupdict()
1012            #print idict
1013            #from pdb import set_trace;set_trace()
1014            urlparsed = urlparse(idict['get'][4:])
1015            #print urlparsed
1016            path = urlparsed[2].split('/')
1017            dict['student_id'] = student_id = path[8]
1018            dict['payment_id'] = payment_id = path[10]
1019            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1020            tas.append(dict)
1021            tr_count += 1
1022        return tas
1023    ###)
1024
1025    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1026    def importOnlinePaymentTransactions(self):
1027        """load Online Payment Transactions from CSV values"""
1028        import transaction
1029        import random
1030        #from pdb import set_trace
1031        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1032        opt = self.online_payments_import
1033        students_folder = self.portal_url.getPortalObject().campus.students
1034        start = True
1035        tr_count = 1
1036        total = 0
1037        #name = 'pume_results'
1038        name = 'OnlineTransactions'
1039        no_import = []
1040        imported = []
1041        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1042        try:
1043            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1044        except:
1045            logger.error('Error reading %s.csv' % name)
1046            return
1047        for pay_transaction in transactions:
1048            if start:
1049                start = False
1050                logger.info('Start loading from %s.csv' % name)
1051                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1052                no_import.append('%s,"Error"' % s)
1053                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1054                format_error = format + ',"%(Error)s"'
1055            data = {}
1056
1057            # format of the first file sent by Tayo
1058            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1059            #data['student_id'] = student_id = pay_transaction['Payer ID']
1060            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1061            #data['response_code'] = response_code = pay_transaction['Resp Code']
1062            #data['amount'] = amount = pay_transaction['Amount']
1063
1064            # format of the second file sent by Tayo
1065            #data['datetime'] = date = 0
1066            #data['student_id'] = student_id = pay_transaction['Payer ID']
1067            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1068            #data['response_code'] = response_code = '00'
1069            #data['amount'] = amount = pay_transaction['Amount']
1070
1071            # format of the third file sent by Kehinde
1072            data['datetime'] = date = 0
1073            data['student_id'] = student_id = pay_transaction['customer_id']
1074            data['order_id'] = order_id = pay_transaction['merchant_reference']
1075            data['response_code'] = response_code = '00'
1076            data['amount'] = amount = pay_transaction['Amount']
1077
1078            dup = False
1079            if response_code == "12":
1080                continue
1081            try:
1082                opt.addRecord(**data)
1083            except ValueError:
1084                dup = True
1085            #from pdb import set_trace;set_trace()
1086            if dup:
1087                if response_code == "00":
1088                    try:
1089                        opt.modifyRecord(**data)
1090                    except:
1091                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1092                        continue
1093                else:
1094                    pay_transaction['Error'] = "Duplicate order_id"
1095                    no_import.append( format_error % pay_transaction)
1096                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1097                    continue
1098            tr_count += 1
1099            if tr_count > 1000:
1100                if len(no_import) > 0:
1101                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1102                             '\n'.join(no_import) + '\n')
1103                    no_import = []
1104                em = '%d transactions committed\n' % (tr_count)
1105                transaction.commit()
1106                regs = []
1107                logger.info(em)
1108                total += tr_count
1109                tr_count = 0
1110        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1111                                                '\n'.join(no_import))
1112        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1113    ###)
1114
1115    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1116    def mass_create_faculty(self,mapping):
1117        "create a faculty"
1118        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1119        academics_folder = self.portal_url.getPortalObject().campus.academics
1120        fid = mapping['code']
1121        if getattr(academics_folder,fid,None) is not None:
1122            return '', "Faculty with ID: %s exists" % fid
1123        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1124        try:
1125            academics_folder.invokeFactory('Faculty', fid)
1126        except BadRequest,E:
1127            return '', "%s" % E
1128        f = getattr(academics_folder,fid,None)
1129        f.getContent().edit(mapping=mapping)
1130        return fid,''
1131    ###)
1132
1133    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1134    def mass_edit_faculty(self,mapping):
1135        "edit a faculty"
1136        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1137        academics_folder = self.portal_url.getPortalObject().campus.academics
1138        fid = mapping['code']
1139        f = getattr(academics_folder,fid,None)
1140        if f is None:
1141            return '', "Faculty with ID: %s does not exist" % fid
1142        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1143        f.getContent().edit(mapping=mapping)
1144        return fid,''
1145    ###)
1146
1147    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1148    def mass_create_department(self,mapping):
1149        "create a department in the correct faculty"
1150        logger = logging.getLogger('WAeUPTool.mass_create_department')
1151        fid = mapping['faculty_code']
1152        if getattr(self,'_v_faculties',None) is None:
1153            res = self.portal_catalog(portal_type = "Faculty")
1154            self._v_faculties = {}
1155            for f in res:
1156                self._v_faculties[f.getId] = f.getObject()
1157        f = self._v_faculties.get(fid,None)
1158        if f is None:
1159            return '', "No Faculty with ID: %s" % fid
1160        else:
1161            did = mapping.get('code')
1162            d = getattr(f,did,None)
1163            if d is None or d.portal_type == "Faculty":
1164                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1165                try:
1166                    f.invokeFactory('Department', did)
1167                except BadRequest,E:
1168                    return '', "%s" % E
1169                d = getattr(f,did)
1170                d.invokeFactory('CoursesFolder','courses')
1171                courses = getattr(d,'courses')
1172                dict = {'Title': 'Courses'}
1173                courses.getContent().edit(mapping=dict)
1174                d.invokeFactory('CertificatesFolder','certificates')
1175                certificates = getattr(d,'certificates')
1176                dict = {'Title': 'Certificates'}
1177                certificates.getContent().edit(mapping=dict)
1178            d.getContent().edit(mapping=mapping)
1179        return did,''
1180    ###)
1181
1182    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1183    def mass_edit_department(self,mapping):
1184        "create a department in the correct faculty"
1185        logger = logging.getLogger('WAeUPTool.mass_create_department')
1186        academics_folder = self.portal_url.getPortalObject().campus.academics
1187        fid = mapping['faculty_code']
1188        did = mapping.get('code')
1189        try:
1190            d = getattr(getattr(academics_folder,fid),did,None)
1191        except KeyError:
1192            return '', "Department %s or Faculty %s wrong" % (did,fid)
1193        else:
1194            if d is None or d.portal_type == "Faculty":
1195                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1196            d.getContent().edit(mapping=mapping)
1197        return did,''
1198    ###)
1199
1200    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1201    def mass_create_course(self,mapping):
1202        #import pdb;pdb.set_trace()
1203        if getattr(self,'_v_course_list',None) is None:
1204            self._v_course_list = []
1205        if getattr(self,'_v_departments',None) is None:
1206            res = self.portal_catalog(portal_type = "Department")
1207            self._v_department_courses = {}
1208            for d in res:
1209                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1210        did = mapping['department_code']
1211        d = self._v_department_courses.get(did,None)
1212        if d is None:
1213            return '', "No Department with ID: %s" % did
1214        course_id = mapping.get('code')
1215        if course_id in self._v_course_list:
1216            return '', "Duplicate Course ID: %s" % did
1217        c = getattr(d,course_id,None)
1218        if c is not None:
1219            return '', "Duplicate Course ID: %s" % did
1220        try:
1221            d.invokeFactory('Course', course_id)
1222        except BadRequest,E:
1223            return '', "%s" % E
1224        self._v_course_list.append(course_id)
1225        c = getattr(d,course_id)
1226        c.getContent().edit(mapping=mapping)
1227        return course_id,''
1228    ###)
1229
1230    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1231    def mass_edit_course(self,mapping):
1232        #import pdb;pdb.set_trace()
1233        course_id = mapping.get('code')
1234        res = self.portal_catalog(id=course_id)
1235        if not res:
1236            return '', "No Course with ID: %s" % course_id
1237        c = res[0].getObject()
1238        c.getContent().edit(mapping=mapping)
1239        return course_id,''
1240    ###)
1241
1242    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1243    def mass_create_certificate(self,mapping):
1244        if getattr(self,'_v_certificate_list',None) is None:
1245            self._v_certificate_list = []
1246        if getattr(self,'_v_department_certificates',None) is None:
1247            res = self.portal_catalog(portal_type = "Department")
1248            self._v_department_certificates = {}
1249            for d in res:
1250                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1251        did = mapping['department_code']
1252        d = self._v_department_certificates.get(did,None)
1253        if d is None:
1254            return '', "No Department with ID: %s" % did
1255        certificate_id = mapping.get('code')
1256        if certificate_id in self._v_certificate_list:
1257            return '', "Duplicate Certificate ID: %s" % did
1258        c = getattr(d,certificate_id,None)
1259        if c is not None:
1260            return '', "Duplicate Certificate ID: %s" % did
1261        try:
1262            d.invokeFactory('Certificate', certificate_id)
1263        except BadRequest,E:
1264            return '', "%s" % E
1265        self._v_certificate_list.append(certificate_id)
1266        c = getattr(d,certificate_id)
1267        c.getContent().edit(mapping=mapping)
1268        return certificate_id,''
1269    ###)
1270
1271    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1272    def mass_edit_certificate(self,mapping):
1273        #import pdb;pdb.set_trace()
1274        certificate_id = mapping.get('code')
1275        res = self.portal_catalog(id=certificate_id)
1276        if not res:
1277            return '', "No Certificate with ID: %s" % did
1278        c = res[0].getObject()
1279        c.getContent().edit(mapping=mapping)
1280        return certificate_id,''
1281    ###)
1282
1283    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1284    def mass_create_application(self,mapping):
1285        #import pdb;pdb.set_trace()
1286        reg_no = mapping.get('reg_no')
1287        try:
1288            self.applicants_catalog.addRecord(**mapping)
1289        except ValueError:
1290            return '', "applicant record with reg_no %s already exists" % reg_no
1291        return reg_no,''
1292    ###)
1293
1294    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1295    def mass_edit_application(self,mapping):
1296        #import pdb;pdb.set_trace()
1297        reg_no = mapping.get('reg_no')
1298        try:
1299            self.applicants_catalog.modifyRecord(**mapping)
1300        except KeyError:
1301            return '', "applicant record with reg_no %s does not exist" % reg_no
1302        return reg_no,''
1303    ###)
1304
1305    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1306    def mass_create_course_result(self,mapping):
1307        #import pdb;pdb.set_trace()
1308        if getattr(self,'_v_courses',None) is None:
1309            res = self.courses_catalog()
1310            self._v_courses = {}
1311            for brain in res:
1312                self._v_courses[brain.code] = brain
1313        course_id = mapping.get('code')
1314        if course_id not in self._v_courses.keys():
1315            return '', "No course with ID: %s" % did
1316        id_key = ''
1317        for id_key in ('student_id','matric_no'):
1318            id_field = mapping.get(id_key,None)
1319            if id_field is not None:
1320                student_id = id_field
1321                break
1322        query = Eq(id_key,id_field)
1323        res = self.students_catalog.evalAdvancedQuery(query)
1324        if not res:
1325            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1326        if id_field != "student_id":
1327            mapping['student_id'] = res[0].id
1328        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping
1329        for k in ('semester',):
1330            mapping[k] = getattr(self._v_courses[course_id],k)
1331        try:
1332            self.course_results.addRecord(**mapping)
1333        except ValueError:
1334            return '', "course result already exists: %s" % key
1335        return key,''
1336    ###)
1337
1338    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1339    def mass_edit_course_result(self,mapping):
1340        #import pdb;pdb.set_trace()
1341        # if getattr(self,'_v_courses',None) is None:
1342        #     res = self.courses_catalog()
1343        #     self._v_courses = {}
1344        #     for brain in res:
1345        #         self._v_courses[brain.code] = brain
1346        # course_id = mapping.get('code')
1347        # if course_id not in self._v_courses.keys():
1348        #     return '', "No course with ID: %s" % did
1349        id_key = ''
1350        for id_key in ('student_id','matric_no'):
1351            id_field = mapping.get(id_key,None)
1352            if id_field is not None:
1353                student_id = id_field
1354                break
1355        query = Eq(id_key,id_field)
1356        res = self.students_catalog.evalAdvancedQuery(query)
1357        if not res:
1358            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1359        if id_field != "student_id":
1360            mapping['student_id'] = res[0].id
1361        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping
1362        # for k in ('semester',):
1363        #     mapping[k] = getattr(self._v_courses[course_id],k)
1364        try:
1365            self.course_results.modifyRecord(**mapping)
1366        except KeyError:
1367            return '', "No course result to edit: %s" % key
1368        return key,''
1369    ###)
1370
1371    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1372    def mass_create_certificate_course(self,mapping):
1373        if getattr(self,'_v_courses',None) is None:
1374            res = self.courses_catalog()
1375            self._v_courses= [course.code for course in res]
1376        if getattr(self,'_v_ceritficates',None) is None:
1377            res = self.portal_catalog(portal_type = "Certificate")
1378            self._v_certificates = {}
1379            for cert in res:
1380                self._v_certificates[cert.getId] = cert.getObject()
1381        certificate_course_id = mapping.get('code')
1382        if certificate_course_id not in self._v_courses:
1383            return '', "No Course with ID: %s" % certificate_course_id
1384        cert_id = mapping['certificate_code']
1385        cert = self._v_certificates.get(cert_id,None)
1386        if cert is None:
1387            return '', "No Certificate with ID: %s" % cert_id
1388        level_id = mapping.get('level')
1389        level = getattr(cert,level_id,None)
1390        if level is None:
1391            cert.invokeFactory('StudyLevel', level_id)
1392            level = getattr(cert,level_id,None)
1393        elif hasattr(level,certificate_course_id):
1394            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1395            (certificate_course_id,cert_id,level_id)
1396        level.invokeFactory('CertificateCourse', certificate_course_id)
1397        c = getattr(level,certificate_course_id)
1398        c.getContent().edit(mapping=mapping)
1399        return certificate_course_id,''
1400    ###)
1401
1402    field2types_student = {   ###(
1403                      'StudentApplication':
1404                          {'id': 'application',
1405                           'title': 'Application Data',
1406                           'wf_transition_return': 'close',
1407                           'wf_transition_admit': 'remain',
1408                           'fields':
1409                             ('jamb_reg_no',
1410                              'entry_mode',
1411                              'entry_session',
1412                              'jamb_score',
1413                              'app_email',
1414                              'jamb_age',
1415                              'jamb_state',
1416                              'jamb_lga',
1417                              )
1418                              },
1419                      #'StudentPume':
1420                      #    {'id': 'pume',
1421                      #     'title': 'Pume Data',
1422                      #     'wf_transition_return': 'close',
1423                      #     'wf_transition_admit': 'close',
1424                      #     'fields':
1425                      #       ('pume_score',
1426                      #        )
1427                      #        },
1428                      'StudentClearance':
1429                          {'id': 'clearance',
1430                           'title': 'Clearance/Eligibility Record',
1431                           'wf_transition_return': 'close',
1432                           'wf_transition_admit': 'remain',
1433                           'fields':
1434                             ('matric_no',
1435                              'nationality',
1436                              'lga',
1437                              'birthday',
1438                              )
1439                              },
1440                         'StudentPersonal':
1441                          {'id': 'personal',
1442                           'title': 'Personal Data',
1443                           'wf_transition_return': 'open',
1444                           'wf_transition_admit': 'remain',
1445                           'fields':
1446                             ('firstname',
1447                              'middlename',
1448                              'lastname',
1449                              'sex',
1450                              'email',
1451                              'phone',
1452                              'perm_address',
1453                              )
1454                              },
1455                         'StudentStudyCourse':
1456                          {'id': 'study_course',
1457                           'title': 'Study Course',
1458                           'wf_transition_return': 'open',
1459                           'wf_transition_admit': 'remain',
1460                           'fields':
1461                             ('study_course',
1462                              'current_level',
1463                              'current_session',
1464                              'current_mode',
1465                              'current_verdict',
1466                              'previous_verdict',
1467                              )
1468                              },
1469
1470                         'PaymentsFolder':
1471                          {'id': 'payments',
1472                           'title': 'Payments',
1473                           'wf_transition_return': 'open',
1474                           'wf_transition_admit': 'open',
1475                           'fields':
1476                             ()
1477                              },
1478                         }
1479    ###)
1480
1481    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1482    def mass_create_student(self,mapping):
1483        "create a students record due import"
1484        logger = logging.getLogger('WAeUPTool.mass_create_student')
1485        students_folder = self.portal_url.getPortalObject().campus.students
1486        jamb_reg_no = mapping.get('jamb_reg_no',None)
1487        if jamb_reg_no:
1488            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1489            if res:
1490                return '',"jamb_reg_no exists"
1491        matric_no = mapping.get('matric_no',None)
1492        if matric_no:
1493            res = self.students_catalog(matric_no = matric_no)
1494            if res:
1495                return '',"matric_no exists"
1496        sid = self.waeup_tool.generateStudentId('?')
1497        students_folder.invokeFactory('Student', sid)
1498        student_obj = getattr(students_folder,sid)
1499        f2t = self.field2types_student
1500        d = {}
1501        transition = mapping.get('reg_transition','admit')
1502        if transition not in ('admit','return'):
1503            return '',"no valid transition provided"
1504        for pt in f2t.keys():
1505            student_obj.invokeFactory(pt,f2t[pt]['id'])
1506            sub_obj = getattr(student_obj,f2t[pt]['id'])
1507            sub_doc = sub_obj.getContent()
1508            d['Title'] = f2t[pt]['title']
1509            for field in f2t[pt]['fields']:
1510                d[field] = mapping.get(field,'')
1511
1512            if pt == "StudentApplication":
1513                d['jamb_sex']  = 'M'
1514                if mapping.get('sex'):
1515                    d['jamb_sex']  = 'F'
1516                d['jamb_firstname'] = mapping.get('firstname',None)
1517                d['jamb_middlename'] = mapping.get('middlename',None)
1518                d['jamb_lastname'] = mapping.get('lastname',None)
1519
1520            # if pt == "StudyCourse":
1521            #     for von,zu in (('entry_mode','current_mode'),
1522            #                    ('entry_session','current_session')):
1523            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1524            #             d[zu] = mapping[von]
1525            sub_doc.edit(mapping = d)
1526
1527            #import pdb;pdb.set_trace()
1528            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1529            if new_state != "remain":
1530                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1531        self.portal_workflow.doActionFor(student_obj,transition)
1532        student_obj.manage_setLocalRoles(sid, ['Owner',])
1533        return sid,''
1534    ###)
1535
1536    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1537    def mass_edit_student(self,mapping):
1538        "edit a students record due import"
1539        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1540        students_folder = self.portal_url.getPortalObject().campus.students
1541        sid = mapping.get('id',None)
1542        jamb_reg_no = mapping.get('jamb_reg_no',None)
1543        matric_no = mapping.get('matric_no',None)
1544        editable_keys = mapping.keys()
1545        if sid:
1546            res = self.students_catalog(id = sid)
1547            if not res:
1548                return '',"no student with id %s" % sid
1549            if matric_no and res[0].matric_no and\
1550              matric_no != res[0].matric_no:
1551                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1552            if jamb_reg_no and res[0].jamb_reg_no and\
1553              jamb_reg_no != res[0].jamb_reg_no:
1554                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1555        elif jamb_reg_no:
1556            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1557            if not res:
1558                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1559            editable_keys.remove('jamb_reg_no')
1560        elif matric_no:
1561            res = self.students_catalog(matric_no = matric_no)
1562            if not res:
1563                return '',"no student with matric_no %s" % matric_no
1564            editable_keys.remove('matric_no')
1565
1566        ## included only to change wf state from admitted to returning
1567        #if res[0].review_state not in ('admitted','objection_raised'):
1568        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1569        ## end inclusion
1570
1571        sid = res[0].id
1572        student_obj = getattr(students_folder,sid)
1573        f2t = self.field2types_student
1574        d = {}
1575        #import pdb;pdb.set_trace()
1576        any_change = False
1577        for pt in f2t.keys():
1578            if pt == "student_application":
1579                d['jamb_sex']  = 'M'
1580                if mapping.get('sex'):
1581                    d['jamb_sex']  = 'F'
1582            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1583            if intersect:
1584                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1585                if sub_obj is None:
1586                    try:
1587                        student_obj.invokeFactory(pt,f2t[pt]['id'])
1588                    except:
1589                        continue
1590                    sub_obj = getattr(student_obj,f2t[pt]['id'])
1591                    d['Title'] = f2t[pt]['title']
1592                sub_doc = sub_obj.getContent()
1593                for field in intersect:
1594                    changed = False
1595                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1596                        any_change = True
1597                        changed = True
1598                        d[field] = mapping.get(field,'')
1599                    if changed:
1600                        sub_doc.edit(mapping = d)
1601
1602
1603        ## included only to change wf state from admitted to returning
1604        #    if res[0].review_state in ('admitted','objection_raised'):
1605        #        new_state = f2t[pt]['wf_transition_return']
1606        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1607        #        if sub_obj and new_state != "remain":
1608        #            try:
1609        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1610        #            except:
1611        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1612        #                pass
1613        #if res[0].review_state in ('admitted','objection_raised'):
1614        #    wfaction = 'return'
1615        #    try:
1616        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1617        #        logger.info('%s, wf state changed' % sid)
1618        #        any_change = True
1619        #    except:
1620        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1621        #        pass
1622        ## end inclusion
1623
1624
1625        if any_change:
1626            return sid,''
1627        else:
1628            return sid,'not modified'
1629    ###)
1630
1631    security.declareProtected(ModifyPortalContent,"importData")###(
1632    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1633        """load data from CSV values"""
1634        import transaction
1635        import random
1636
1637        pm = self.portal_membership
1638        member = pm.getAuthenticatedMember()
1639
1640        logger = logging.getLogger('WAeUPTool.importData')
1641        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1642        students_folder = self.portal_url.getPortalObject().campus.students
1643        start = True
1644        tr_count = 0
1645        total_imported = 0
1646        total_not_imported = 0
1647        total = 0
1648        iname = "import_%s" % name
1649        if iname == 'import_application':
1650            commit_after = 2000
1651        else:
1652            commit_after = 100
1653        stool = getToolByName(self, 'portal_schemas')
1654        ltool = getToolByName(self, 'portal_layouts')
1655        schema = stool._getOb(iname)
1656        if schema is None:
1657            em = 'No such schema %s' % iname
1658            logger.error('No such schema %s' % iname)
1659            return em
1660        layout = ltool._getOb(iname)
1661        if layout is None:
1662            em = 'No such layout %s' % iname
1663            logger.error(em)
1664            return em
1665        validators = {}
1666        for widget in layout.keys():
1667            validators[widget] = layout[widget].validate
1668        #import pdb;pdb.set_trace()
1669        mode = "create"
1670        if edit:
1671            mode = "edit"
1672        importer_name = "mass_%(mode)s_%(name)s" % vars()
1673        importer = getattr(self, '%s' % importer_name,None)
1674        if importer is None:
1675            em = 'No importer function %s' % importer_name
1676            logger.error(em)
1677            return em
1678        not_imported = []
1679        imported = []
1680        try:
1681            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1682        except:
1683            em = 'Error reading %s.csv' % filename
1684            logger.error(em)
1685            return em
1686        for item in items:
1687            if start:
1688                start = False
1689                adapters = [MappingStorageAdapter(schema, item)]
1690                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1691                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1692                import_keys = [k for k in attrs if not k.startswith('ignore')]
1693                diff2schema = set(import_keys).difference(set(schema.keys()))
1694                diff2layout = set(import_keys).difference(set(layout.keys()))
1695                if diff2schema:
1696                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1697                    return em
1698                if mode == "create":
1699                    required_keys = [layout.getIdUnprefixed(id)
1700                                     for id,widget in layout.objectItems()
1701                                     if widget.is_required]
1702                    if not set(required_keys).issubset(set(import_keys)):
1703                        diff2import = set(required_keys).difference(set(import_keys))
1704                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1705                        return em
1706                s = ','.join(['"%s"' % fn for fn in import_keys])
1707                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1708                s = '"id",' + s
1709                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1710                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1711                format_error = format + ',"%(Error)s"'
1712                format = '"%(id)s",'+ format
1713
1714            dm = DataModel(item, adapters,context=self)
1715            ds = DataStructure(data=item,datamodel=dm)
1716            error_string = ""
1717            total += 1
1718            for k in import_keys:
1719                if not validators[k](ds,mode=mode):
1720                    error_string += " %s : %s" % (k,
1721                                                  self.translation_service(ds.getError(k),
1722                                                                           ds.getErrorMapping(k)))
1723            if not error_string:
1724                temp_item = item.copy()
1725                temp_item.update(dm)
1726                temp_item['id'],error = importer(temp_item)
1727                if error:
1728                    error_string += error
1729                else:
1730                    item = temp_item
1731            if error_string:
1732                item['Error'] = error_string
1733                not_imported.append(format_error % item)
1734                total_not_imported += 1
1735            else:
1736                em = format % item
1737                imported.append(em)
1738                tr_count += 1
1739                total_imported += 1
1740                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1741
1742            if total and not total % commit_after:
1743                transaction.commit()
1744                if len(not_imported) > 0:
1745                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1746                             '\n'.join(not_imported) + '\n')
1747                    not_imported = []
1748                if len(imported) > 0:
1749                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1750                             '\n'.join(imported) + '\n')
1751                    imported = []
1752                em = '%d transactions committed\n' % (tr_count)
1753                regs = []
1754                logger.info(em)
1755                tr_count = 0
1756                #if total > 100:
1757                #    return
1758        if len(imported) > 0:
1759            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1760                                                '\n'.join(imported))
1761        if len(not_imported) > 0:
1762            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1763                                                '\n'.join(not_imported))
1764        em = "Finished: %d imported, %d not imported (of total %d)" % (total_imported,total_not_imported,total)
1765        logger.info(em)
1766        return em
1767    ###)
1768
1769    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1770    def moveImagesToFS(self,student_id="O738726"):
1771        "move the images to the filesystem"
1772        images_dir = os.path.join("%s" % images_base,student_id)
1773        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1774        stool = getToolByName(self, 'portal_schemas')
1775        schemas = ['student_application',
1776                   'student_clearance',
1777                   ]
1778        created = False
1779        for schema_id in schemas:
1780            schema = stool._getOb(schema_id)
1781            object = getattr(student_folder,schema_id[len('student_'):],None)
1782            if object is None:
1783                continue
1784            doc = object.getContent()
1785            for key in schema.keys():
1786                if schema[key].meta_type != "CPS Image Field":
1787                    continue
1788                #import pdb;pdb.set_trace()
1789                image = getattr(doc,key,None)
1790                if not image or not hasattr(image,"data"):
1791                    continue
1792                if not created:
1793                    if not os.path.exists(images_dir):
1794                        os.mkdir(images_dir)
1795                    created = True
1796                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1797                open(filename,"wb").write(str(image.data))
1798    ###)
1799
1800    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1801    def movePassportToFS(self,student_id="O738726"):
1802        "move the passports to the filesystem"
1803        images_dir = os.path.join("%s" % i_home,'passports')
1804        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1805        stool = getToolByName(self, 'portal_schemas')
1806        schemas = ['student_application',
1807                   #'student_clearance',
1808                   ]
1809        created = False
1810        for schema_id in schemas:
1811            schema = stool._getOb(schema_id)
1812            object = getattr(student_folder,schema_id[len('student_'):],None)
1813            if object is None:
1814                continue
1815            doc = object.getContent()
1816            for key in schema.keys():
1817                if schema[key].meta_type != "CPS Image Field":
1818                    continue
1819                #import pdb;pdb.set_trace()
1820                image = getattr(doc,key)
1821                if not hasattr(image,"data"):
1822                    continue
1823                if not created:
1824                    if not os.path.exists(images_dir):
1825                        os.mkdir(images_dir)
1826                    created = True
1827                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1828                open(filename,"wb").write(str(image.data))
1829    ###)
1830
1831InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.