source: WAeUP_SRP/base/WAeUPTool.py @ 2766

Last change on this file since 2766 was 2766, checked in by joachim, 17 years ago

M skins/waeup_student/create_level.py
use course_results for carry_overs

M skins/waeup_pins/pins_view.pt
M skins/waeup_pins/search_pins.py
sort batches in select-box

M Widgets.py
M WAeUPTables.py
M WAeUPTool.py
mass_(create,edit)_courseresults edits verdict in level.

M profiles/default/layouts/application_cest.xml
M profiles/default/applicants_catalog.xml
new meta_data field "serial", index must be added manually
  • Property svn:keywords set to Id
File size: 85.4 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2766 2007-11-26 19:18:09Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
65    """WAeUP tool"""
66
67    id = 'waeup_tool'
68    meta_type = 'WAeUP Tool'
69    _actions = ()
70    security = ClassSecurityInfo()
71    security.declareObjectProtected(View)
72    manage_options = ( ActionProviderBase.manage_options
73                     + SimpleItem.manage_options
74                     )
75
76    def rwrite(self,s): ###(
77        response = self.REQUEST.RESPONSE
78        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
79        response.write("%s<br />\r\n" % s)
80    ###)
81
82    def addtodict(self,d,key,item): ###(
83        d[key].append(item)
84        return d[key]
85    ###)
86
87    def sleep(self,secs): ###(
88        "sleep"
89        import time
90        time.sleep(secs)
91        return
92
93###)
94
95    security.declareProtected(View,'updateRoleMappingsFor') ###(
96    def updateRoleMappingsFor(self,wf_definition,ob):
97        "do so for public"
98        wf_def = getattr(self.portal_workflow,wf_definition)
99        wf_def.updateRoleMappingsFor(ob)
100    ###)
101
102    security.declareProtected(ModifyPortalContent,'openLog') ###(
103    def openLog(self,name):
104        """open a log file"""
105        version = 1
106        path = "%s/log/%s_%d.log" % (i_home,name,version)
107        while os.path.exists(path):
108            version += 1
109            path = "%s/log/%s_%d.log" % (i_home,name,version)
110        log = open(path,"w")
111        return log
112    ###)
113
114    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
115    def bypassQueueCatalog(self,enable=True):
116        """bypass the QueueCatalog by setting all indexes to process imediate,
117        if enable is True (default) the old settings are restored
118        """
119
120    ###)
121
122    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
123    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
124        """measure Object access Time"""
125        import random
126        if hasattr(self,'portal_catalog_real'):
127            aq_portal = self.portal_catalog_real.evalAdvancedQuery
128        else:
129            aq_portal = self.portal_catalog.evalAdvancedQuery
130        nr_pts = int(nr_pts)
131        probe = int(probe)
132        intervall = probe/10
133        objects = ("application","clearance","personal")
134        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
135        #i = random.randrange(num_objects)
136        count = 0
137        found = 0
138        not_found = 0
139        t_found = 0
140        t_not_found = 0
141        time_found = time_not_found = 0.0
142        t_time_found = t_time_not_found = 0.0
143        accessed = []
144        t_min = 1000
145        t_max = 0
146        #import pdb;pdb.set_trace()
147        students = self.portal_catalog(portal_type="Student")
148        num_students = len(students)
149        if method == "d":
150            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
151            res = aq_portal(query)
152            brains = {}
153            for r in res:
154                sid = r.relative_path.split('/')[-2]
155                if brains.has_key(sid):
156                    brains[sid][r.portal_type] = r
157                else:
158                    brains[sid] = {r.portal_type : r}
159            brains_list = brains.keys()
160            num_objects = len(brains_list)
161        else:
162            num_objects = num_students
163        print "="*40
164        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
165                                                                        probe,
166                                                                        nr_pts,
167                                                                        num_objects)
168        print "nr found/not time found/not min/max"
169        elapse = time.time()
170        i_elapse = time.time()
171        c_elapse = time.clock()
172        for c in range(1,probe + 1):
173            i = random.randrange(num_objects)
174            if method in ('a','b','c'):
175                student_brain = students[i]
176            elif method == "d":
177                #import pdb;pdb.set_trace()
178                student_brain = brains[brains_list[i]]
179            if method == "c":
180                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
181                res = aq_portal(query)
182                this_portal_types = [r.portal_type for r in res]
183            for i in range(nr_pts):
184                oid = objects[i]
185                if method == "a":
186                    try:
187                        student_path = student_brain.getPath()
188                        path = "%s/%s" % (student_path,oid)
189                        doc = self.unrestrictedTraverse(path).getContent()
190                        found += 1
191                        i_time = time.time() - i_elapse
192                        time_found += i_time
193                    except:
194                        not_found += 1
195                        i_time = time.time() - i_elapse
196                        time_not_found += i_time
197                        pass
198                elif method == "b":
199                    try:
200                        student_object = student_brain.getObject()
201                        doc = getattr(student_object,oid).getContent()
202                        found += 1
203                        i_time = time.time() - i_elapse
204                        time_found += i_time
205                    except:
206                        i_time = time.time() - i_elapse
207                        time_not_found += i_time
208                        not_found += 1
209                        pass
210                elif method == "c":
211                    if portal_types[i] in this_portal_types:
212                        found += 1
213                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
214                        i_time = time.time() - i_elapse
215                        time_found += i_time
216                    else:
217                        not_found += 1
218                        i_time = time.time() - i_elapse
219                        time_not_found += i_time
220                elif method == "d":
221                    if student_brain.has_key(portal_types[i]):
222                        found += 1
223                        doc = student_brain[portal_types[i]].getObject().getContent()
224                        i_time = time.time() - i_elapse
225                        time_found += i_time
226                    else:
227                        not_found += 1
228                        i_time = time.time() - i_elapse
229                        time_not_found += i_time
230                i_elapse = time.time()
231            if c and (c % intervall == 0):
232                #i_time = time.time() - i_elapse
233                t_per = 0.0
234                if found:
235                    t_per = time_found/found
236                if t_per > t_max:
237                    t_max = t_per
238                if t_per > 0.0 and t_per < t_min:
239                    t_min = t_per
240                itf = 0.0
241                if found:
242                    itf = time_found/found
243                itnf = 0.0
244                if not_found :
245                    itnf = time_not_found / not_found
246                interval_time = time_found + time_not_found
247                s = "%(c)d: %(found)d/%(not_found)d " % vars()
248                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
249                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
250                print s
251                t_found += found
252                t_not_found += not_found
253                t_time_found += time_found
254                t_time_not_found += time_not_found
255                time_found = time_not_found = 0.0
256                found = not_found = 0
257        # t_found += found
258        # t_not_found += not_found
259        elapse = time.time() - elapse
260        itf = 0.0
261        if t_found:
262            itf = t_time_found/t_found
263        itnf = 0.0
264        if t_not_found:
265            itnf = t_time_not_found / t_not_found
266        #c_elapse = time.clock() - c_elapse
267        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
268        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
269        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
270        print "-"*40
271        print s
272        rel_found = float(t_found)/probe
273        rel_not_found = float(t_not_found)/probe
274        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
275        print estimated_total_time
276    ###)
277
278    security.declareProtected(ModifyPortalContent,'writeLog') ###(
279    def writeLog(self,logfile,s):
280        """write to the log file"""
281        logfile.write(s)
282
283###)
284
285    def generateStudentId(self,letter): ###(
286        import random
287        r = random
288        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
289        if letter == '?':
290            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
291        sid = "%c%d" % (letter,r.randint(99999,1000000))
292        students = self.portal_url.getPortalObject().campus.students
293##        while hasattr(students, sid):
294##            sid = "%c%d" % (letter,r.randint(99999,1000000))
295        while self.students_catalog(id = sid):
296            sid = "%c%d" % (letter,r.randint(99999,1000000))
297        return sid
298    ###)
299
300    def generatePassword(self,s=None): ###(
301        import random
302        r = random
303        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
304        if s is None:
305            s = 'abcdefghklmnpqrstuvwxy23456789'
306        pw = ''
307        while len(pw) < 6:
308            pw += r.choice(s)
309        return pw
310    ###)
311
312    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
313    def dumpSchoolfeePayments(self):
314        "dump paid schoolfees"
315        mtool = self.portal_membership
316        member = mtool.getAuthenticatedMember()
317        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
318        aq_student = self.students_catalog.evalAdvancedQuery
319        query = In('review_state',('schoolfee_paid',
320                                   'courses_registered',
321                                   'courses_validated',
322                                   ))
323        res = aq_student(query)
324        #import pdb;pdb.set_trace()
325        l = []
326        logger.info("start for %d" % len(res))
327        count = 1
328        log_after = 100
329        for student in res:
330            if not count % log_after:
331                logger.info("processed %d total %d" % (log_after,count))
332            count += 1
333            fee_dict =self.getSchoolFee(student)
334            fulltime = student.mode.endswith('_ft')
335            d = {}
336            d['student_id'] = student.id
337            d['name'] = student.name
338            d['amount'] = fee_dict.get(new_returning)
339            l += d,
340        csv_name = self.dumpListToCSV(l,'payments')
341        logger.info('%s dumped payments to %s' % (member,export_file))
342    ###)
343
344    security.declarePublic('dumpListToCSV') ###(
345    def dumpListToCSV(self,l,filename,fields=None):
346        """dump a list of dicts to a CSV file"""
347        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
348        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
349        if fields is None:
350            fields = l[0].keys()
351        headline = ','.join(fields)
352        out = open(export_file,"wb")
353        out.write(headline +'\n')
354        out.close()
355        out = open(export_file,"a")
356        csv_writer = csv.DictWriter(out,fields,)
357        csv_writer.writerows(l)
358        return export_file
359    ###)
360
361    security.declareProtected(ManagePortal, 'listMembers') ###(
362    def listMembers(self):
363        "list all members"
364        mtool = self.portal_membership
365        member = mtool.getAuthenticatedMember()
366        logger = logging.getLogger('WAeUPTool.listMembers')
367        if str(member) not in ('admin','joachim'):
368            logger.info('%s tried to list members' % (member))
369            return None
370        members = self.portal_directories.members
371        all = members.listEntryIdsAndTitles()
372        l = []
373        for user_id,name in all:
374            d = {}
375            d['user_id'] = user_id
376            d['name'] = name
377            d['pw'] = getattr(getattr(members,user_id),'password')
378            d['email'] = getattr(getattr(members,user_id),'email')
379            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
380            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
381            l += d,
382        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
383        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
384        logger.info('%s dumped memberlist to %s' % (member,export_file))
385        fields = l[0].keys()
386        headline = ','.join(fields)
387        out = open(export_file,"wb")
388        out.write(headline +'\n')
389        out.close()
390        out = open(export_file,"a")
391        csv_writer = csv.DictWriter(out,fields,)
392        csv_writer.writerows(l)
393    ###)
394
395    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
396    def removeDeletedDocIds(self, max=1000):
397        """
398        remove deleted docids from repository commit after max
399        """
400        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
401        repository = getToolByName(self, 'portal_repository')
402        pxtool = getToolByName(self, 'portal_proxies')
403        logger.info('start')
404        pxtool_infos = pxtool.getRevisionsUsed()
405        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
406
407        nb_revs = 0
408        docids_d = {} # all docids
409        unused_docids_d = {} # all docids that are unused
410        ids_unused_revs_docids = [] # ids for revs of unused docids
411        unused_ids = [] # ids for unused revs
412        total = 0
413        idlist = repository.objectIds()
414        to_delete = 0
415        found = False
416        for id in idlist:
417            docid, rev = repository._splitId(id)
418            if docid is None:
419                logger.info("invalid doc_id %s" % docid)
420                continue
421            nb_revs += 1
422            if not pxtool_infos.has_key(docid):
423                found = True
424                to_delete += 1
425                unused_ids.append(id)
426            elif not pxtool_infos[docid].has_key(rev):
427                found = True
428                to_delete += 1
429                unused_ids.append(id)
430            if found and not to_delete % max:
431                found = False
432                #import pdb;pdb.set_trace()
433                repository.manage_delObjects(unused_ids)
434                transaction.commit()
435                logger.info('removed %d total %d unused docids ' % (max,to_delete))
436        else:
437            if unused_ids:
438                repository.manage_delObjects(unused_ids)
439                transaction.commit()
440        logger.info('finished removing %d unused docids ' % (to_delete))
441
442
443###)
444
445    security.declareProtected(View,'getCredential') ###(
446    def getCredential(self,student_id):
447        student_entry = getattr(self.portal_directories.students,student_id,None)
448        if not self.isStaff():
449            mtool = self.portal_membership
450            member = mtool.getAuthenticatedMember()
451            logger = logging.getLogger('WAeUPTool.getCredential')
452            logger.info('%s tried to access password of %s' % (member,student_id))
453            return None
454        if student_entry is None:
455            return None
456        return getattr(student_entry,"password","not set")
457    ###)
458
459    security.declarePublic('checkPassword') ###(
460    def checkPassword(self,student_id,password):
461        student_entry = getattr(self.portal_directories.students,student_id,None)
462        if student_entry is None:
463            return False
464        return getattr(student_entry,"password","not set") == password
465    ###)
466
467    security.declarePublic('checkGenericPassword') ###(
468    def checkGenericPassword(self,member_id):
469        member_entry = getattr(self.portal_directories.members,member_id,None)
470        if member_entry is None:
471            return False
472        ltool = getToolByName(self, 'portal_layouts')
473        unsecure_words = ltool._getOb('members')['w__password'].check_words
474        password = getattr(member_entry,"password","not set")
475        is_unsecure = password in unsecure_words
476        if is_unsecure:
477            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
478            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
479        return is_unsecure
480    ###)
481
482    security.declareProtected(ModifyPortalContent,'editPassword') ###(
483    def editPassword(self,student_id,password):
484        "edit a student password"
485        student_entry = getattr(self.portal_directories.students,student_id,None)
486        if student_entry is None:
487            return
488        setattr(student_entry,'password',password)
489    ###)
490
491    security.declareProtected(ModifyPortalContent,'doCommit') ###(
492    def doCommit(self,logger=None):
493        "commit some transactions"
494        transaction.commit()
495    ###)
496
497    security.declarePublic('loadStudentFoto') ###(
498    def loadStudentFoto(self,student,filename,folder):
499        "return a student passport picture"
500        #import pdb;pdb.set_trace()
501        picture ="%s/import/%s/%s" % (i_home,folder,filename)
502        student_id = student.getId()
503        images_dir = getImagesDir(student_id)
504        if not os.path.exists(images_dir):
505            os.mkdir(images_dir)
506        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
507        for extension in ('.jpg','.JPG'):
508            fullname = "%(picture)s%(extension)s" % vars()
509            if os.path.exists(fullname):
510                copy2(fullname,image_name)
511                return "successfully copied passport picture"
512        return "passport picture not found: %s.jpg or .JPG" % picture
513    ###)
514
515    def old____loadStudentFoto(self,student,filename,folder): ###(
516        "return a student passport picture"
517        app = student.application
518        app_doc = app.getContent()
519        #clear = student.clearance
520        #clear_doc = clear.getContent()
521        #matric_no = clear_doc.matric_no.upper()
522        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
523        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
524        #import pdb;pdb.set_trace()
525        if os.path.exists(picture1):
526            file = open(picture1)
527        elif os.path.exists(picture2):
528            file = open(picture2)
529        else:
530            return "passport picture not found %s" % picture1
531        reopened = False
532        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
533            self.portal_workflow.doActionFor(app,'open')
534            reopened = True
535        outfile = file.read()
536        app_doc.manage_addFile('passport',
537                               file=outfile,
538                               title="%s.jpg" % filename)
539        if reopened:
540            self.portal_workflow.doActionFor(app,'close')
541        return "successfully loaded passport picture"
542    ###)
543
544    security.declareProtected(ModifyPortalContent,'createOne') ###(
545    def createOne(self,students_folder,student_brain,letter,commit=False):
546        sid = self.waeup_tool.generateStudentId(letter)
547        students_folder.invokeFactory('Student', sid)
548        student = getattr(students_folder,sid)
549        self.portal_workflow.doActionFor(student,'return')
550        student.manage_setLocalRoles(sid, ['Owner',])
551        matric_no = student_brain.matric_no
552        jamb_reg_no = student_brain.Entryregno
553        self.students_catalog.addRecord(id = sid,
554                                           matric_no = matric_no,
555                                           jamb_reg_no = jamb_reg_no,
556                                           sex = student_brain.Sex == "F",
557                                           name = "%s %s %s" % (student_brain.Firstname,
558                                                                student_brain.Middlename,
559                                                                student_brain.Lastname)
560                                        )
561        if commit:
562            transaction.commit()
563        return sid,jamb_reg_no
564    ###)
565
566    security.declareProtected(ModifyPortalContent,'addStudent') ###(
567    def addStudent(self,dict):
568        students_folder = self.portal_url.getPortalObject().campus.students
569        sid = self.waeup_tool.generateStudentId('?')
570        students_folder.invokeFactory('Student', sid)
571        student_obj = getattr(students_folder,sid)
572        f2t = self.student_field2types
573        #from pdb import set_trace; set_trace()
574        d = {}
575        #d['jamb_sex']  = 'M'
576        #if dict.get('sex'):
577        #    d['jamb_sex']  = 'F'
578
579        entry_session = dict.get('entry_session')
580        if entry_session == self.getSessionId()[0]:
581            wfaction = 'admit'
582            wft = 'wf_transition_admit'
583            password = None
584        else:
585            wfaction = 'return'
586            wft = 'wf_transition_return'
587            password = self.generatePassword()
588            self.makeStudentMember(sid,password)
589
590        for pt in f2t.keys():
591            student_obj.invokeFactory(pt,f2t[pt]['id'])
592            sub_obj = getattr(student_obj,f2t[pt]['id'])
593            sub_doc = sub_obj.getContent()
594            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
595            d['Title'] = f2t[pt]['title']
596            for field in f2t[pt]['fields']:
597                d[field] = dict.get(field,'')
598            sub_doc.edit(mapping = d)
599            new_state = f2t[pt][wft]
600            if new_state != "remain":
601                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
602        self.portal_workflow.doActionFor(student_obj,wfaction)
603        student_obj.manage_setLocalRoles(sid, ['Owner',])
604        return sid,password
605    ###)
606
607    security.declarePublic('getCertificateBrain') ###(
608    def getCertificateBrain(self,cert_id):
609        "do it"
610        res = ZCatalog.searchResults(self.portal_catalog_real,
611                                {'portal_type':"Certificate",
612                                      'id': cert_id})
613        if res:
614            return res[0]
615        return None
616    ###)
617
618    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
619    def get_csv_filenames(self):
620        "do it"
621        files = [file for file in os.listdir("%s/import/" % (i_home))
622                 if file.endswith('.csv') and file.find('imported') == -1]
623        return files
624    ###)
625
626    security.declarePublic('findStudentByMatricelNo') ###(
627    def findStudentByMatricelNo(self,matric_no):
628        "do it"
629        res = ZCatalog.searchResults(self.portal_catalog_real,
630                                {'portal_type':"StudentClearance",
631                                 'SearchableText': matric_no})
632        if res:
633            return res[0]
634        return None
635    ###)
636
637    security.declarePublic('makeStudentMember') ###(
638    def makeStudentMember(self,sid,password='uNsEt'):
639        """make the student a member"""
640        membership = self.portal_membership
641        membership.addMember(sid,
642                             password ,
643                             roles=('Member',
644                                     'Student',
645                                     ),
646                             domains='',
647                             properties = {'memberareaCreationFlag': False,
648                                           'homeless': True},)
649        member = membership.getMemberById(sid)
650        self.portal_registration.afterAdd(member, sid, password, None)
651        #self.manage_setLocalRoles(sid, ['Owner',])
652    ###)
653
654    security.declareProtected(View,'makeStudentData') ###(
655    def makeStudentData(self,student_id,email=None,phone_nr=None):
656        "create Datastructure for a returning Student"
657        #import pdb;pdb.set_trace()
658        logger = logging.getLogger('WAeUPTool.makeStudentData')
659        students_folder = self.portal_url.getPortalObject().campus.students
660        #res = self.students_catalog(id=student_id)
661        #if res:
662        #    st = res[0]
663        #res = self.returning_import(matric_no = st.matric_no)
664        res = self.returning_import(id = student_id)
665        if res:
666            student = res[0]
667        else:
668            logger.info('Id %s not found in returning_import' % student_id)
669            return
670        logger.info('%s creates data structure' % student_id)
671        s_results = self.results_import(matric_no = student.matric_no)
672        if s_results:
673            lnr = self.getLevelFromResultsCosCode(s_results)
674            level = "%d00" % lnr
675            verdict,eligible = self.getVerdict(s_results[0].Verdict)
676            #if eligible:
677            #    level = "%d00" % (lnr + 1)
678        else:
679            logger.info('matric_no %s not found in results_import' % student.matric_no)
680            level = ''
681            verdict = ''
682        #student should not be allowed to perform this transition
683        #wftool = self.portal_workflow
684        #wftool.doActionFor(student,'return')
685        certcode_org = student.Coursemajorcode
686        certcode = makeCertificateCode(certcode_org)
687        certificate_brain = self.getCertificateBrain(certcode)
688        if not certificate_brain:
689            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
690            logger.info(em)
691        matric_no = student.matric_no
692        sid = student_id
693        student_obj = getattr(students_folder,sid)
694        student_obj.invokeFactory('StudentApplication','application')
695        application = student_obj.application
696        self.portal_workflow.doActionFor(application,'open',dest_container=application)
697        da = {'Title': 'Application Data'}
698        student_obj.invokeFactory('StudentPersonal','personal')
699        da['jamb_reg_no'] = student.Entryregno
700        em = self.getEntryMode(student.Entryregno)
701        da['entry_mode'] = em
702        personal = student_obj.personal
703        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
704        dp = {'Title': 'Personal Data'}
705        student_obj.invokeFactory('StudentClearance','clearance')
706        clearance = student_obj.clearance
707        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
708        dc = {'Title': 'Clearance/Eligibility Record'}
709        dc['matric_no'] = matric_no
710        state = student.State
711        lga = student.LGA
712        if state and lga:
713            lga =  state + ' / ' + lga
714        else:
715            lga = "None"
716        da['jamb_lga'] = dc['lga'] = lga
717        da['app_email'] = dp['email'] = email
718        da['app_mobile'] = dp['phone'] = phone_nr
719        dp['firstname'] = student.Firstname
720        dp['middlename'] = student.Middlename
721        dp['lastname'] = student.Lastname
722        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
723        da['jamb_sex'] = student.Sex
724        dp['sex'] = student.Sex == 'F'
725        dp['perm_address'] = student.Permanent_Address
726        application.getContent().edit(mapping=da)
727        self.portal_workflow.doActionFor(application,'close',dest_container=application)
728        personal.getContent().edit(mapping=dp)
729        clearance.getContent().edit(mapping=dc)
730        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
731        #
732        # Study Course
733        #
734        student_obj.invokeFactory('StudentStudyCourse','study_course')
735        studycourse = student_obj.study_course
736        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
737        dsc = {}
738        dsc['study_course'] = certcode
739        dsc['current_level'] = level
740        dsc['current_verdict'] = verdict
741        dsc['current_mode'] = em
742        dsc['current_session'] = '05'
743        studycourse.getContent().edit(mapping=dsc)
744        #
745        # Level
746        #
747##        l = getattr(studycourse,level,None)
748##        if l is None:
749##            studycourse.invokeFactory('StudentStudyLevel', level)
750##            l = getattr(studycourse, level)
751##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
752##            l.getContent().edit(mapping={'Title': "Level %s" % level})
753###)
754
755    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
756    def admitOneStudent(self,brain,entry_session,pin_password):
757        "create Datastructure for an admitted Student"
758        #import pdb;pdb.set_trace()
759        if not hasattr(self,"_v_certificates"):
760            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
761        students_folder = self.portal_url.getPortalObject().campus.students
762        logger = logging.getLogger('WAeUPTool.admitOneStudent')
763        if brain.status != "admitted":
764            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
765            return
766        pin_parts = brain.pin.split('-')
767        if pin_parts and len(pin_parts) != 3:
768            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
769            return
770        if brain.course_admitted not in self._v_certificates:
771            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
772            return
773        student_id = self.generateStudentId('?')
774        students_folder.invokeFactory('Student', student_id)
775        student_object = getattr(students_folder,student_id)
776        if pin_password:
777            password = pin_parts[2]
778            self.makeStudentMember(student_id,password = password)
779        student_object.manage_setLocalRoles(student_id, ['Owner',])
780        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
781        #
782        # application
783        #
784        student_object.invokeFactory('StudentApplication','application')
785        application = student_object.application
786        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
787        da = {'Title': 'Application Data'}
788        da['jamb_reg_no'] = brain.reg_no
789
790        sex = 'M'
791        if brain.sex:
792            sex = 'F'
793        da['jamb_sex'] = sex
794        da['jamb_age'] = brain.jamb_age
795        #da['app_ac_pin'] = brain.pin
796        da['jamb_lga'] = brain.jamb_lga
797        da['jamb_state'] = brain.jamb_state
798        da['jamb_score'] = brain.aggregate
799        da['app_email'] = brain.email
800        da['app_mobile'] = brain.phone
801        if brain.entry_mode:
802            da['entry_mode'] = brain.entry_mode
803        elif brain.screening_type == 'pume':
804            da['entry_mode'] = 'ume_ft'
805        elif brain.screening_type == 'pde':
806            da['entry_mode'] = 'de_ft'
807        else:
808            da['entry_mode'] = brain.screening_type
809        da['entry_session'] = entry_session
810        da['jamb_lastname'] = brain.lastname
811        da['jamb_middlename'] = brain.middlenames   # different field names!
812        da['jamb_firstname'] = brain.firstname
813        da['screening_application_date'] = brain.application_date
814        da['date_of_birth'] = brain.date_of_birth
815        da['jamb_first_cos'] = brain.course1
816        da['jamb_second_cos'] = brain.course2
817        da['course3'] = brain.course3
818        da['screening_type'] = brain.screening_type
819        da['screening_score'] = brain.screening_score
820        da['screening_date'] = brain.screening_date
821        da['hq_type'] = brain.hq_type
822        da['hq_grade'] = brain.hq_grade
823        da['aos'] = brain.aos
824
825        application.getContent().edit(mapping=da)
826        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
827        #
828        # personal
829        #
830        student_object.invokeFactory('StudentPersonal','personal')
831        personal = student_object.personal
832        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
833        dp = {'Title': 'Personal Data'}
834        dp['sex'] = brain.sex
835        dp['email'] = brain.email
836        dp['phone'] = brain.phone
837        dp['lastname'] = brain.lastname
838        dp['middlename'] = brain.middlenames   # different field names!
839        dp['firstname'] = brain.firstname
840        personal.getContent().edit(mapping=dp)
841        #
842        # clearance
843        #
844        student_object.invokeFactory('StudentClearance','clearance')
845        clearance = student_object.clearance
846        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
847        dc = {'Title': 'Clearance/Eligibility Record'}
848        dc['lga'] = brain.lga
849        dc['birthday'] = brain.date_of_birth
850        clearance.getContent().edit(mapping=dc)
851        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
852        #
853        # study Course
854        #
855        student_object.invokeFactory('StudentStudyCourse','study_course')
856        studycourse = student_object.study_course
857        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
858        dsc = {}
859        dsc['study_course'] = brain.course_admitted
860        dsc['current_verdict'] = ''
861        dsc['current_mode'] = da['entry_mode']
862        if da['entry_mode'].startswith('de'):
863            dsc['current_level'] = '200'
864        else:
865            dsc['current_level'] = '100'
866        dsc['current_session'] = entry_session
867        studycourse.getContent().edit(mapping=dsc)
868        #
869        # payments folder
870        student_object.invokeFactory('PaymentsFolder','payments')
871        payments = getattr(student_object,'payments')
872        dpay = {}
873        dpay['Title'] = 'Payments'
874        payments.getContent().edit(mapping=dpay)
875        self.portal_workflow.doActionFor(payments,'open')
876        #
877        # passport foto
878        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
879                                                              brain.screening_type,
880                                                              brain.reg_no)
881        images_dir = getImagesDir(student_id)
882        #images_dir = os.path.join("%s" % images_base,student_id)
883        letter_dir,student_dir = os.path.split(images_dir)
884        if not os.path.exists(letter_dir):
885            os.mkdir(letter_dir)
886        if not os.path.exists(images_dir):
887            os.mkdir(images_dir)
888        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
889        if os.path.exists(app_picture):
890            copy2(app_picture,image_name)
891        else:
892            logger.info('passport of %s/%s not found: %s' % (student_id,
893                                                             brain.reg_no,
894                                                             app_picture))
895
896        return student_id
897    ###)
898
899    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
900    def makeStudentLevel(self,student_id):
901        "create the StudyLevel for a returning Student"
902        #import pdb;pdb.set_trace()
903        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
904        students_folder = self.portal_url.getPortalObject().campus.students
905        res = self.students_catalog(id=student_id)
906        if res:
907            st = res[0]
908        course = st.course
909        matric_no = st.matric_no
910        level = st.level
911        res = self.results_import(matric_no = matric_no)
912        if res:
913            results = res
914        logger.info('%s creating Level %s' % (student_id,level))
915        #
916        # Level
917        #
918        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
919        studycourse = getattr(student_obj,"study_course",None)
920        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
921        l = getattr(studycourse,level,None)
922        if l is None:
923            studycourse.invokeFactory('StudentStudyLevel', level)
924            l = getattr(studycourse, level)
925            self.portal_workflow.doActionFor(l,'open',dest_container=l)
926            l.getContent().edit(mapping={'Title': "Level %s" % level})
927        ###)
928
929    security.declarePublic('getHallInfo') ###(
930    def getHallInfo(self,bed):
931        """return Hall Info"""
932        info = {}
933        hall,block,room,letter = bed.split('_')
934        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
935        if res and len(res) == 1:
936            hall_brain = res[0]
937            hall_doc = hall_brain.getObject().getContent()
938        else:
939            return info
940        info['hall_title'] = hall_brain.Title
941        info['maintenance_code'] = hall_doc.maintenance_code
942        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
943        batch_doc = None
944        for brain in res:
945            if brain.id.startswith(info['maintenance_code']):
946                batch_doc = brain.getObject().getContent()
947                break
948        if batch_doc is None:
949            info['maintenance_fee'] = None
950        else:
951            info['maintenance_fee'] = batch_doc.cost
952        return info
953    ###)
954
955    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
956    def removePictureFolder(self,student_id):
957        """remove picture_folder by renaming it"""
958        path = 'images'
959        picture_path = os.path.join(i_home,path,student_id)
960        if not os.path.exists(picture_path):
961            return False
962        os.rename(picture_path,picture_path + "_removed")
963        return True
964    ###)
965
966    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
967    def restorePictureFolder(self,student_id):
968        """restore picture_folder by renaming it"""
969        path = 'images'
970        picture_path = os.path.join(i_home,path,student_id)
971        if not os.path.exists(picture_path + "_removed"):
972            return False
973        os.rename(picture_path + "_removed",picture_path)
974        return True
975    ###)
976
977    security.declarePublic('picturesExist') ###(
978    def picturesExist(self, ids,student_id=None):
979        """check if pictures exist in the filesystem"""
980        if student_id is None:
981            student_id = self.getStudentId()
982        if student_id is None:
983            return False
984        picture_path = getImagesDir(student_id)
985        #picture_path = os.path.join(images_base,student_id)
986        if not os.path.exists(picture_path):
987            return False
988        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
989        return set(ids).issubset(set(pictures))
990    ###)
991
992    security.declarePublic('picturesList') ###(
993    def picturesList(self):
994        """check if pictures exist in the filesystem"""
995        path = 'images'
996        student_id = self.getStudentId()
997        #picture_path = os.path.join(i_home,path,student_id)
998        picture_path = getImagesDir(student_id)
999        if not os.path.exists(picture_path):
1000            return []
1001        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1002    ###)
1003
1004    security.declarePublic('showFsPicture') ###(
1005    def showFsPicture(self,path):
1006        """return a picture from the filesystem"""
1007        #picture_path = os.path.join(i_home,path)
1008        picture_path = os.path.join(images_base,path)
1009        response = self.REQUEST.RESPONSE
1010        #import pdb;pdb.set_trace()
1011        registry = getToolByName(self, 'mimetypes_registry')
1012        mimetype = str(registry.lookupExtension(path.lower()) or
1013                    registry.lookupExtension('file.bin'))
1014        if os.path.exists(picture_path):
1015            response.setHeader('Content-type',mimetype)
1016            return open(picture_path).read()
1017        picture_path = os.path.join(i_home,'import',path)
1018        if os.path.exists(picture_path):
1019            return open(picture_path).read()
1020    ###)
1021
1022    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1023    def deleteAllCourses(self,department="All"):
1024        ''' delete the courses'''
1025        pm = self.portal_membership
1026        member = pm.getAuthenticatedMember()
1027
1028        if str(member) not in ("henrik","joachim"):
1029            return "not possible"
1030        if department == "All":
1031            res = self.portal_catalog({'meta_type': 'Department'})
1032        if len(res) < 1:
1033            return "No Departments found"
1034
1035        deleted = []
1036        for dep in res:
1037            cf = dep.getObject().courses
1038            if cf:
1039                cf.manage_delObjects(ids=cf.objectIds())
1040                deleted.append("deleted Courses in %s" % dep.getId)
1041        return "\r".join(deleted)
1042    ###)
1043
1044    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1045    def getLogfileLines(self,filename="event.log",numlines=20):
1046        """Get last NUMLINES lines of logfile FILENAME.
1047
1048        Return last lines' of a file in the instances logfile directory as
1049        a list. The number of returned lines equals `numlines' or less. If
1050        less than `numlines' lines are available, the whole file ist
1051        returned. If the file can not be opened or some other error
1052        occurs, empty list is returend.
1053        """
1054        result = []
1055        lines_hit = 0
1056
1057        # We only handle files in instances' log directory...
1058        logpath = os.path.join(i_home, "log")
1059        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1060        if not filename.startswith( logpath ):
1061            # Attempt to access file outside log-dir...
1062            return []
1063
1064        try:
1065            fd = file( filename, "rb" )
1066        except IOError:
1067            return []
1068        if not fd:
1069            return []
1070
1071        if os.linesep == None:
1072            linesep = '\n'
1073        else:
1074            linesep = os.linesep
1075
1076        # Try to find 'numlines' times a lineseparator, searching from end
1077        # and moving to the beginning of file...
1078        fd.seek( 0, 2) # Move to end of file...
1079        while lines_hit < numlines:
1080            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1081                                          # one step forward...
1082                lines_hit += 1
1083            try:
1084                fd.seek( -2, 1) # Go two bytes back from current pos...
1085            except IOError:
1086                # We cannot go back two bytes. Maybe the file is too small...
1087                break
1088        fd.seek(2,1)
1089
1090        # Read all lines from current position...
1091        result = fd.readlines()
1092        # Remove line endings...
1093        result = [x.strip() for x in result]
1094        fd.close()
1095        return result
1096    ###)
1097
1098    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1099    def getCallbacksFromLog(self,filename):
1100        """fix Online Payment Transactions from Z2.log entries"""
1101        import transaction
1102        import random
1103        from cgi import parse_qs
1104        from urlparse import urlparse
1105        #from pdb import set_trace
1106        wftool = self.portal_workflow
1107        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1108        students_folder = self.portal_url.getPortalObject().campus.students
1109        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1110        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1111        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1112        data = re.compile(s)
1113        start = True
1114        tr_count = 1
1115        total = 0
1116        #name = 'pume_results'
1117        #name = 'epaymentsuccessful_z2log2'
1118        name = filename
1119        no_import = []
1120        imported = []
1121        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1122        try:
1123            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1124        except:
1125            logger.error('Error reading %s' % name)
1126            return
1127        tas = []
1128        for line in transactions:
1129            dict = {}
1130            items = data.search(line)
1131            dict['idict'] = idict = items.groupdict()
1132            #print idict
1133            #from pdb import set_trace;set_trace()
1134            urlparsed = urlparse(idict['get'][4:])
1135            #print urlparsed
1136            path = urlparsed[2].split('/')
1137            dict['student_id'] = student_id = path[8]
1138            dict['payment_id'] = payment_id = path[10]
1139            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1140            tas.append(dict)
1141            tr_count += 1
1142        return tas
1143    ###)
1144
1145    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1146    def importOnlinePaymentTransactions(self):
1147        """load Online Payment Transactions from CSV values"""
1148        import transaction
1149        import random
1150        #from pdb import set_trace
1151        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1152        opt = self.online_payments_import
1153        students_folder = self.portal_url.getPortalObject().campus.students
1154        start = True
1155        tr_count = 1
1156        total = 0
1157        #name = 'pume_results'
1158        name = 'OnlineTransactions'
1159        no_import = []
1160        imported = []
1161        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1162        try:
1163            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1164        except:
1165            logger.error('Error reading %s.csv' % name)
1166            return
1167        for pay_transaction in transactions:
1168            if start:
1169                start = False
1170                logger.info('Start loading from %s.csv' % name)
1171                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1172                no_import.append('%s,"Error"' % s)
1173                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1174                format_error = format + ',"%(Error)s"'
1175            data = {}
1176
1177            # format of the first file sent by Tayo
1178            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1179            #data['student_id'] = student_id = pay_transaction['Payer ID']
1180            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1181            #data['response_code'] = response_code = pay_transaction['Resp Code']
1182            #data['amount'] = amount = pay_transaction['Amount']
1183
1184            # format of the second file sent by Tayo
1185            #data['datetime'] = date = 0
1186            #data['student_id'] = student_id = pay_transaction['Payer ID']
1187            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1188            #data['response_code'] = response_code = '00'
1189            #data['amount'] = amount = pay_transaction['Amount']
1190
1191            # format of the third file sent by Kehinde
1192            data['datetime'] = date = 0
1193            data['student_id'] = student_id = pay_transaction['customer_id']
1194            data['order_id'] = order_id = pay_transaction['merchant_reference']
1195            data['response_code'] = response_code = '00'
1196            data['amount'] = amount = pay_transaction['Amount']
1197
1198            dup = False
1199            if response_code == "12":
1200                continue
1201            try:
1202                opt.addRecord(**data)
1203            except ValueError:
1204                dup = True
1205            #from pdb import set_trace;set_trace()
1206            if dup:
1207                if response_code == "00":
1208                    try:
1209                        opt.modifyRecord(**data)
1210                    except:
1211                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1212                        continue
1213                else:
1214                    pay_transaction['Error'] = "Duplicate order_id"
1215                    no_import.append( format_error % pay_transaction)
1216                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1217                    continue
1218            tr_count += 1
1219            if tr_count > 1000:
1220                if len(no_import) > 0:
1221                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1222                             '\n'.join(no_import) + '\n')
1223                    no_import = []
1224                em = '%d transactions committed\n' % (tr_count)
1225                transaction.commit()
1226                regs = []
1227                logger.info(em)
1228                total += tr_count
1229                tr_count = 0
1230        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1231                                                '\n'.join(no_import))
1232        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1233    ###)
1234
1235    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1236    def mass_create_faculty(self,mapping):
1237        "create a faculty"
1238        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1239        academics_folder = self.portal_url.getPortalObject().campus.academics
1240        fid = mapping['code']
1241        if getattr(academics_folder,fid,None) is not None:
1242            return '', "Faculty with ID: %s exists" % fid
1243        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1244        try:
1245            academics_folder.invokeFactory('Faculty', fid)
1246        except BadRequest,E:
1247            return '', "%s" % E
1248        f = getattr(academics_folder,fid,None)
1249        f.getContent().edit(mapping=mapping)
1250        return fid,''
1251    ###)
1252
1253    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1254    def mass_edit_faculty(self,mapping):
1255        "edit a faculty"
1256        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1257        academics_folder = self.portal_url.getPortalObject().campus.academics
1258        fid = mapping['code']
1259        f = getattr(academics_folder,fid,None)
1260        if f is None:
1261            return '', "Faculty with ID: %s does not exist" % fid
1262        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1263        f.getContent().edit(mapping=mapping)
1264        return fid,''
1265    ###)
1266
1267    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1268    def mass_create_department(self,mapping):
1269        "create a department in the correct faculty"
1270        logger = logging.getLogger('WAeUPTool.mass_create_department')
1271        fid = mapping['faculty_code']
1272        if getattr(self,'_v_faculties',None) is None:
1273            res = self.portal_catalog(portal_type = "Faculty")
1274            self._v_faculties = {}
1275            for f in res:
1276                self._v_faculties[f.getId] = f.getObject()
1277        f = self._v_faculties.get(fid,None)
1278        if f is None:
1279            return '', "No Faculty with ID: %s" % fid
1280        else:
1281            did = mapping.get('code')
1282            d = getattr(f,did,None)
1283            if d is None or d.portal_type == "Faculty":
1284                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1285                try:
1286                    f.invokeFactory('Department', did)
1287                except BadRequest,E:
1288                    return '', "%s" % E
1289                d = getattr(f,did)
1290                d.invokeFactory('CoursesFolder','courses')
1291                courses = getattr(d,'courses')
1292                dict = {'Title': 'Courses'}
1293                courses.getContent().edit(mapping=dict)
1294                d.invokeFactory('CertificatesFolder','certificates')
1295                certificates = getattr(d,'certificates')
1296                dict = {'Title': 'Certificates'}
1297                certificates.getContent().edit(mapping=dict)
1298            d.getContent().edit(mapping=mapping)
1299        return did,''
1300    ###)
1301
1302    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1303    def mass_edit_department(self,mapping):
1304        "create a department in the correct faculty"
1305        logger = logging.getLogger('WAeUPTool.mass_create_department')
1306        academics_folder = self.portal_url.getPortalObject().campus.academics
1307        fid = mapping['faculty_code']
1308        did = mapping.get('code')
1309        try:
1310            d = getattr(getattr(academics_folder,fid),did,None)
1311        except KeyError:
1312            return '', "Department %s or Faculty %s wrong" % (did,fid)
1313        else:
1314            if d is None or d.portal_type == "Faculty":
1315                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1316            d.getContent().edit(mapping=mapping)
1317        return did,''
1318    ###)
1319
1320    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1321    def mass_create_course(self,mapping):
1322        #import pdb;pdb.set_trace()
1323        if getattr(self,'_v_course_list',None) is None:
1324            self._v_course_list = []
1325        if getattr(self,'_v_departments',None) is None:
1326            res = self.portal_catalog(portal_type = "Department")
1327            self._v_department_courses = {}
1328            for d in res:
1329                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1330        did = mapping['department_code']
1331        d = self._v_department_courses.get(did,None)
1332        if d is None:
1333            return '', "No Department with ID: %s" % did
1334        course_id = mapping.get('code')
1335        if course_id in self._v_course_list:
1336            return '', "Duplicate Course ID: %s" % did
1337        c = getattr(d,course_id,None)
1338        if c is not None:
1339            return '', "Duplicate Course ID: %s" % did
1340        try:
1341            d.invokeFactory('Course', course_id)
1342        except BadRequest,E:
1343            return '', "%s" % E
1344        self._v_course_list.append(course_id)
1345        c = getattr(d,course_id)
1346        c.getContent().edit(mapping=mapping)
1347        return course_id,''
1348    ###)
1349
1350    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1351    def mass_edit_course(self,mapping):
1352        #import pdb;pdb.set_trace()
1353        course_id = mapping.get('code')
1354        res = self.portal_catalog(id=course_id)
1355        if not res:
1356            return '', "No Course with ID: %s" % course_id
1357        c = res[0].getObject()
1358        c.getContent().edit(mapping=mapping)
1359        return course_id,''
1360    ###)
1361
1362    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1363    def mass_create_certificate(self,mapping):
1364        if getattr(self,'_v_certificate_list',None) is None:
1365            self._v_certificate_list = []
1366        if getattr(self,'_v_department_certificates',None) is None:
1367            res = self.portal_catalog(portal_type = "Department")
1368            self._v_department_certificates = {}
1369            for d in res:
1370                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1371        did = mapping['department_code']
1372        d = self._v_department_certificates.get(did,None)
1373        if d is None:
1374            return '', "No Department with ID: %s" % did
1375        certificate_id = mapping.get('code')
1376        if certificate_id in self._v_certificate_list:
1377            return '', "Duplicate Certificate ID: %s" % did
1378        c = getattr(d,certificate_id,None)
1379        if c is not None:
1380            return '', "Duplicate Certificate ID: %s" % did
1381        try:
1382            d.invokeFactory('Certificate', certificate_id)
1383        except BadRequest,E:
1384            return '', "%s" % E
1385        self._v_certificate_list.append(certificate_id)
1386        c = getattr(d,certificate_id)
1387        c.getContent().edit(mapping=mapping)
1388        return certificate_id,''
1389    ###)
1390
1391    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1392    def mass_edit_certificate(self,mapping):
1393        #import pdb;pdb.set_trace()
1394        certificate_id = mapping.get('code')
1395        res = self.portal_catalog(id=certificate_id)
1396        if not res:
1397            return '', "No Certificate with ID: %s" % did
1398        c = res[0].getObject()
1399        c.getContent().edit(mapping=mapping)
1400        return certificate_id,''
1401    ###)
1402
1403    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1404    def mass_create_application(self,mapping):
1405        #import pdb;pdb.set_trace()
1406        reg_no = mapping.get('reg_no')
1407        try:
1408            self.applicants_catalog.addRecord(**mapping)
1409        except ValueError:
1410            return '', "applicant record with reg_no %s already exists" % reg_no
1411        return reg_no,''
1412    ###)
1413
1414    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1415    def mass_edit_application(self,mapping):
1416        #import pdb;pdb.set_trace()
1417        reg_no = mapping.get('reg_no')
1418        status = mapping.get('status')
1419        res = self.applicants_catalog(reg_no = reg_no)
1420        if len(res):
1421            if res[0].status == 'created' and status != 'created':
1422                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1423            elif status == 'created' and res[0].status != 'created':
1424                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1425            else:
1426                self.applicants_catalog.modifyRecord(**mapping)
1427                return reg_no,''
1428        else:
1429            return '', "applicant record with reg_no %s does not exist" % reg_no
1430
1431    ###)
1432
1433    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1434    def mass_create_course_result(self,mapping):
1435        #import pdb;pdb.set_trace()
1436        students_folder = self.portal_url.getPortalObject().campus.students
1437        if getattr(self,'_v_courses',None) is None:
1438            res = self.courses_catalog()
1439            self._v_courses = {}
1440            for brain in res:
1441                self._v_courses[brain.code] = brain
1442        if getattr(self,'_v_level_created',None) is None:
1443            self._v_level_created = []
1444        course_id = mapping.get('code')
1445        if course_id not in self._v_courses.keys():
1446            return '', "No course with ID: %s" % course_id
1447        # id_key = ''
1448        # for id_key in ('student_id','matric_no'):
1449        #     id_field = mapping.get(id_key,None)
1450        #     if id_field is not None:
1451        #         student_id = id_field
1452        #         break
1453        # query = Eq(id_key,id_field)
1454        # res = self.students_catalog.evalAdvancedQuery(query)
1455        student_rec = self.students_catalog.getRecordByKey(mapping['student_id'])
1456        if student_rec is None:
1457            return '', "no student with %(id_key)s %(id_field)s" % vars()
1458        # if id_field != "student_id":
1459        #     mapping['student_id'] = res[0].id
1460        student_id = student_rec.id
1461        level_id = mapping['level_id']
1462        code = mapping['code']
1463        if mapping['student_id'] not in self._v_level_created:
1464            try:
1465                context = getattr(getattr(students_folder,
1466                                          "%(student_id)s" % vars()),
1467                                  'study_course')
1468            except:
1469                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1470            if level_id not in context.objectIds():
1471                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1472                level = getattr(context,"%(level_id)s" % vars())
1473                self.portal_workflow.doActionFor(level,'open')
1474                # the session string must not be copied into the level object
1475                if student_rec.current_verdict:
1476                    level.getContent().edit(mapping={'verdict': "%s" %
1477                                                     student_rec.current_verdict,
1478                                                     'session': "%s" %
1479                                                     student_rec.current_session,
1480                                                    })
1481                    self.portal_workflow.doActionFor(level,'close')
1482            self._v_level_created += student_id,
1483        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1484        for k in ('semester','credits',):
1485            mapping[k] = getattr(self._v_courses[course_id],k)
1486        try:
1487            self.course_results.addRecord(**mapping)
1488        except ValueError:
1489            return '', "course result already exists: %s" % key
1490        return key,''
1491    ###)
1492
1493    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1494    def mass_edit_course_result(self,mapping):
1495        #import pdb;pdb.set_trace()
1496        # if getattr(self,'_v_courses',None) is None:
1497        #     res = self.courses_catalog()
1498        #     self._v_courses = {}
1499        #     for brain in res:
1500        #         self._v_courses[brain.code] = brain
1501        # course_id = mapping.get('code')
1502        # if course_id not in self._v_courses.keys():
1503        #     return '', "No course with ID: %s" % did
1504        # id_key = ''
1505        # for id_key in ('student_id','matric_no'):
1506        #     id_field = mapping.get(id_key,None)
1507        #     if id_field is not None:
1508        #         student_id = id_field
1509        #         break
1510        # query = Eq(id_key,id_field)
1511        # res = self.students_catalog.evalAdvancedQuery(query)
1512        # if not res:
1513        #     return '', "No student with %(id_field)s: %(id_key)s" % vars()
1514        # if id_field != "student_id":
1515        #     mapping['student_id'] = res[0].id
1516        students_folder = self.portal_url.getPortalObject().campus.students
1517        student_rec = self.students_catalog.getRecordByKey(mapping['student_id'])
1518        if student_rec is None:
1519            return '', "no student with %(id_key)s %(id_field)s" % vars()
1520        student_id = student_rec.id
1521        level_id = mapping['level_id']
1522        code = mapping['code']
1523        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1524        # for k in ('semester',):
1525        #     mapping[k] = getattr(self._v_courses[course_id],k)
1526        try:
1527            self.course_results.modifyRecord(**mapping)
1528        except KeyError:
1529            return '', "No course result to edit: %s" % key
1530        if student_rec.current_verdict:
1531            #makes only sense if the course_results are imported for the current session
1532            try:
1533                level = getattr(getattr(getattr(students_folder,
1534                                                "%(student_id)s" % vars()),
1535                                        'study_course'),
1536                                "%(level_id)s" % vars())
1537            except:
1538                return '', "could not find level %(level_id)s for %(student_id)s" % vars()
1539            if self.portal_workflow.getInfoFor(level,'review_state',None) == "closed":
1540                self.portal_workflow.doActionFor(level,'open')
1541            level.getContent().edit(mapping={'verdict': "%s" %
1542                                                student_rec.current_verdict,
1543                                                'session': "%s" %
1544                                                student_rec.current_session,
1545                                            })
1546            self.portal_workflow.doActionFor(level,'close')
1547        return key,''
1548    ###)
1549
1550    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1551    def mass_create_certificate_course(self,mapping):
1552        if getattr(self,'_v_courses',None) is None:
1553            res = self.courses_catalog()
1554            self._v_courses= [course.code for course in res]
1555        if getattr(self,'_v_ceritficates',None) is None:
1556            res = self.portal_catalog(portal_type = "Certificate")
1557            self._v_certificates = {}
1558            for cert in res:
1559                self._v_certificates[cert.getId] = cert.getObject()
1560        certificate_course_id = mapping.get('code')
1561        if certificate_course_id not in self._v_courses:
1562            return '', "No Course with ID: %s" % certificate_course_id
1563        cert_id = mapping['certificate_code']
1564        cert = self._v_certificates.get(cert_id,None)
1565        if cert is None:
1566            return '', "No Certificate with ID: %s" % cert_id
1567        level_id = mapping.get('level')
1568        level = getattr(cert,level_id,None)
1569        if level is None:
1570            cert.invokeFactory('StudyLevel', level_id)
1571            level = getattr(cert,level_id,None)
1572        elif hasattr(level,certificate_course_id):
1573            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1574            (certificate_course_id,cert_id,level_id)
1575        level.invokeFactory('CertificateCourse', certificate_course_id)
1576        c = getattr(level,certificate_course_id)
1577        c.getContent().edit(mapping=mapping)
1578        return certificate_course_id,''
1579    ###)
1580
1581    field2types_student = {   ###(
1582                      'StudentApplication':
1583                          {'id': 'application',
1584                           'title': 'Application Data',
1585                           'wf_transition_return': 'close',
1586                           'wf_transition_admit': 'remain',
1587                           'fields':
1588                             ('jamb_reg_no',
1589                              'entry_mode',
1590                              'entry_session',
1591                              'jamb_score',
1592                              'app_email',
1593                              'jamb_age',
1594                              'jamb_state',
1595                              'jamb_lga',
1596                              )
1597                              },
1598                      #'StudentPume':
1599                      #    {'id': 'pume',
1600                      #     'title': 'Pume Data',
1601                      #     'wf_transition_return': 'close',
1602                      #     'wf_transition_admit': 'close',
1603                      #     'fields':
1604                      #       ('pume_score',
1605                      #        )
1606                      #        },
1607                      'StudentClearance':
1608                          {'id': 'clearance',
1609                           'title': 'Clearance/Eligibility Record',
1610                           'wf_transition_return': 'close',
1611                           'wf_transition_admit': 'remain',
1612                           'fields':
1613                             ('matric_no',
1614                              'nationality',
1615                              'lga',
1616                              'birthday',
1617                              )
1618                              },
1619                         'StudentPersonal':
1620                          {'id': 'personal',
1621                           'title': 'Personal Data',
1622                           'wf_transition_return': 'open',
1623                           'wf_transition_admit': 'remain',
1624                           'fields':
1625                             ('firstname',
1626                              'middlename',
1627                              'lastname',
1628                              'sex',
1629                              'email',
1630                              'phone',
1631                              'perm_address',
1632                              )
1633                              },
1634                         'StudentStudyCourse':
1635                          {'id': 'study_course',
1636                           'title': 'Study Course',
1637                           'wf_transition_return': 'open',
1638                           'wf_transition_admit': 'remain',
1639                           'fields':
1640                             ('study_course',
1641                              'current_level',
1642                              'current_session',
1643                              'current_mode',
1644                              'current_verdict',
1645                              'previous_verdict',
1646                              )
1647                              },
1648                         'StudentStudyLevel':
1649                          {'id': 'current_level',
1650                           'title': '',
1651                           'wf_transition_return': 'open',
1652                           'wf_transition_admit': 'remain',
1653                           'fields':
1654                             ('verdict',
1655                              'session',
1656                              )
1657                              },
1658                         'PaymentsFolder':
1659                          {'id': 'payments',
1660                           'title': 'Payments',
1661                           'wf_transition_return': 'open',
1662                           'wf_transition_admit': 'open',
1663                           'fields':
1664                             ()
1665                              },
1666                         }
1667    ###)
1668
1669    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1670    def mass_create_student(self,mapping):
1671        "create a students record due import"
1672        logger = logging.getLogger('WAeUPTool.mass_create_student')
1673        students_folder = self.portal_url.getPortalObject().campus.students
1674        jamb_reg_no = mapping.get('jamb_reg_no',None)
1675        if jamb_reg_no:
1676            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1677            if res:
1678                return '',"jamb_reg_no exists"
1679        matric_no = mapping.get('matric_no',None)
1680        if matric_no:
1681            res = self.students_catalog(matric_no = matric_no)
1682            if res:
1683                return '',"matric_no exists"
1684        sid = self.waeup_tool.generateStudentId('?')
1685        students_folder.invokeFactory('Student', sid)
1686        student_obj = getattr(students_folder,sid)
1687        f2t = self.field2types_student
1688        d = {}
1689        transition = mapping.get('reg_transition','admit')
1690        if transition not in ('admit','return'):
1691            return '',"no valid transition provided"
1692        for pt in f2t.keys():
1693            student_obj.invokeFactory(pt,f2t[pt]['id'])
1694            sub_obj = getattr(student_obj,f2t[pt]['id'])
1695            sub_doc = sub_obj.getContent()
1696            d['Title'] = f2t[pt]['title']
1697            for field in f2t[pt]['fields']:
1698                d[field] = mapping.get(field,'')
1699
1700            if pt == "StudentApplication":
1701                d['jamb_sex']  = 'M'
1702                if mapping.get('sex'):
1703                    d['jamb_sex']  = 'F'
1704                d['jamb_firstname'] = mapping.get('firstname',None)
1705                d['jamb_middlename'] = mapping.get('middlename',None)
1706                d['jamb_lastname'] = mapping.get('lastname',None)
1707
1708            # if pt == "StudyCourse":
1709            #     for von,zu in (('entry_mode','current_mode'),
1710            #                    ('entry_session','current_session')):
1711            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1712            #             d[zu] = mapping[von]
1713            sub_doc.edit(mapping = d)
1714
1715            #import pdb;pdb.set_trace()
1716            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1717            if new_state != "remain":
1718                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1719        self.portal_workflow.doActionFor(student_obj,transition)
1720        student_obj.manage_setLocalRoles(sid, ['Owner',])
1721        return sid,''
1722    ###)
1723
1724    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1725    def mass_edit_student(self,mapping):
1726        wftool = self.portal_workflow
1727        "edit a students record due import"
1728        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1729        students_folder = self.portal_url.getPortalObject().campus.students
1730        sid = mapping.get('id',None)
1731        jamb_reg_no = mapping.get('jamb_reg_no',None)
1732        matric_no = mapping.get('matric_no',None)
1733        editable_keys = mapping.keys()
1734        if sid:
1735            res = self.students_catalog(id = sid)
1736            if not res:
1737                return '',"no student with id %s" % sid
1738            if matric_no and res[0].matric_no and\
1739              matric_no != res[0].matric_no:
1740                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1741            if jamb_reg_no and res[0].jamb_reg_no and\
1742              jamb_reg_no != res[0].jamb_reg_no:
1743                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1744        elif jamb_reg_no:
1745            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1746            if not res:
1747                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1748            editable_keys.remove('jamb_reg_no')
1749        elif matric_no:
1750            res = self.students_catalog(matric_no = matric_no)
1751            if not res:
1752                return '',"no student with matric_no %s" % matric_no
1753            editable_keys.remove('matric_no')
1754
1755        ## included only to change wf state from admitted to returning
1756        #if res[0].review_state not in ('admitted','objection_raised'):
1757        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1758        ## end inclusion
1759
1760        sid = res[0].id
1761        student_obj = getattr(students_folder,sid)
1762        f2t = self.field2types_student
1763        d = {}
1764        #import pdb;pdb.set_trace()
1765        any_change = False
1766        for pt in f2t.keys():
1767            if pt == "StudentApplication":
1768                d['jamb_sex']  = 'M'
1769                if mapping.get('sex'):
1770                    d['jamb_sex']  = 'F'
1771            #special treatment for StudentStudyLevel
1772            elif pt == "StudentStudyLevel":
1773                d['verdict']  = mapping.get('current_verdict','')
1774                d['session']  = mapping.get('current_session','')
1775                while d['session'] or d['verdict']:
1776                    sub_obj = getattr(student_obj,'study_course',None)
1777                    if sub_obj is None:
1778                        break
1779                    level = getattr(sub_obj,mapping.get('current_level',None))
1780                    if level is None:
1781                        break
1782                    level.getContent().edit(mapping = d)
1783                    try:
1784                        wftool.doActionFor(level,'close')
1785                    except:
1786                        pass
1787                    break
1788            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1789            if intersect and pt not in ('StudentStudyLevel',):
1790                object_id = f2t[pt]['id']
1791                sub_obj = getattr(student_obj,object_id,None)
1792                if sub_obj is None:
1793                    try:
1794                        student_obj.invokeFactory(pt,object_id)
1795                    except:
1796                        continue
1797                    sub_obj = getattr(student_obj,object_id)
1798                    if f2t[pt]['title'] != '':
1799                        d['Title'] = f2t[pt]['title']
1800                sub_doc = sub_obj.getContent()
1801                for field in intersect:
1802                    changed = False
1803                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1804                        any_change = True
1805                        changed = True
1806                        d[field] = mapping.get(field,'')
1807                    if changed:
1808                        sub_doc.edit(mapping = d)
1809
1810
1811        ## included only to change wf state from admitted to returning
1812        #    if res[0].review_state in ('admitted','objection_raised'):
1813        #        new_state = f2t[pt]['wf_transition_return']
1814        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1815        #        if sub_obj and new_state != "remain":
1816        #            try:
1817        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1818        #            except:
1819        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1820        #                pass
1821        #if res[0].review_state in ('admitted','objection_raised'):
1822        #    wfaction = 'return'
1823        #    try:
1824        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1825        #        logger.info('%s, wf state changed' % sid)
1826        #        any_change = True
1827        #    except:
1828        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1829        #        pass
1830        ## end inclusion
1831
1832
1833        if any_change:
1834            return sid,''
1835        else:
1836            return sid,'not modified'
1837    ###)
1838
1839    security.declareProtected(ModifyPortalContent,"importData")###(
1840    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1841        """load data from CSV values"""
1842        import transaction
1843        import random
1844
1845        pm = self.portal_membership
1846        member = pm.getAuthenticatedMember()
1847
1848        logger = logging.getLogger('WAeUPTool.importData')
1849        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1850        students_folder = self.portal_url.getPortalObject().campus.students
1851        start = True
1852        tr_count = 0
1853        total_imported = 0
1854        total_not_imported = 0
1855        total = 0
1856        iname = "import_%s" % name
1857        if iname in ('import_application','import_course_result',):
1858            commit_after = 2000
1859        else:
1860            commit_after = 100
1861        stool = getToolByName(self, 'portal_schemas')
1862        ltool = getToolByName(self, 'portal_layouts')
1863        schema = stool._getOb(iname)
1864        if schema is None:
1865            em = 'No such schema %s' % iname
1866            logger.error('No such schema %s' % iname)
1867            return em
1868        layout = ltool._getOb(iname)
1869        if layout is None:
1870            em = 'No such layout %s' % iname
1871            logger.error(em)
1872            return em
1873        validators = {}
1874        for widget in layout.keys():
1875            validators[widget] = layout[widget].validate
1876        mode = "create"
1877        if edit:
1878            mode = "edit"
1879        importer_name = "mass_%(mode)s_%(name)s" % vars()
1880        importer = getattr(self, '%s' % importer_name,None)
1881        if importer is None:
1882            em = 'No importer function %s' % importer_name
1883            logger.error(em)
1884            return em
1885        not_imported = []
1886        imported = []
1887        try:
1888            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1889        except:
1890            em = 'Error reading %s.csv' % filename
1891            logger.error(em)
1892            return em
1893        for item in items:
1894            #import pdb;pdb.set_trace()
1895            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1896            if start:
1897                start = False
1898                adapters = [MappingStorageAdapter(schema, item)]
1899                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1900                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1901                import_keys = [k.strip() for k in attrs if not k.startswith('ignore')]
1902                diff2schema = set(import_keys).difference(set(schema.keys()))
1903                diff2layout = set(import_keys).difference(set(layout.keys()))
1904                if diff2schema:
1905                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1906                    return em
1907                if mode == "create":
1908                    required_keys = [layout.getIdUnprefixed(id)
1909                                     for id,widget in layout.objectItems()
1910                                     if widget.is_required]
1911                    if not set(required_keys).issubset(set(import_keys)):
1912                        diff2import = set(required_keys).difference(set(import_keys))
1913                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1914                        return em
1915                s = ','.join(['"%s"' % fn for fn in import_keys])
1916                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1917                s = '"id",' + s
1918                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1919                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1920                format_error = format + ',"%(Error)s"'
1921                format = '"%(id)s",'+ format
1922
1923            dm = DataModel(item, adapters,context=self)
1924            ds = DataStructure(data=item,datamodel=dm)
1925            error_string = ""
1926            total += 1
1927            error_count = 0
1928            for k in import_keys:
1929                if not validators[k](ds,mode=mode):
1930                    if error_count:
1931                        error_string += ' ++ '
1932                    error_string += "%s: %s" % (k,
1933                                                  self.translation_service(ds.getError(k),
1934                                                                           ds.getErrorMapping(k)))
1935                    error_count += 1
1936            if not error_string:
1937                temp_item = item.copy()
1938                temp_item.update(dm)
1939                temp_item['id'],error = importer(temp_item)
1940                if error:
1941                    error_string += error
1942                else:
1943                    item = temp_item
1944            if error_string:
1945                item['Error'] = error_string
1946                not_imported.append(format_error % item)
1947                total_not_imported += 1
1948            else:
1949                em = format % item
1950                imported.append(em)
1951                tr_count += 1
1952                total_imported += 1
1953                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1954
1955            if total and not total % commit_after:
1956                transaction.commit()
1957                if len(not_imported) > 0:
1958                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1959                             '\n'.join(not_imported) + '\n')
1960                    not_imported = []
1961                if len(imported) > 0:
1962                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1963                             '\n'.join(imported) + '\n')
1964                    imported = []
1965                em = '%d transactions committed\n' % (tr_count)
1966                regs = []
1967                logger.info(em)
1968                tr_count = 0
1969                #if total > 100:
1970                #    return
1971        if len(imported) > 0:
1972            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1973                                                '\n'.join(imported))
1974        if len(not_imported) > 0:
1975            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1976                                                '\n'.join(not_imported))
1977        em = "Finished: %d imported, %d not imported (of total %d)" % (total_imported,total_not_imported,total)
1978        logger.info(em)
1979        return em
1980    ###)
1981
1982    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1983    def moveImagesToFS(self,student_id="O738726"):
1984        "move the images to the filesystem"
1985        images_dir = getImagesDir(student_id)
1986        #images_dir = os.path.join("%s" % images_base,student_id)
1987        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1988        stool = getToolByName(self, 'portal_schemas')
1989        schemas = ['student_application',
1990                   'student_clearance',
1991                   ]
1992        created = False
1993        for schema_id in schemas:
1994            schema = stool._getOb(schema_id)
1995            object = getattr(student_folder,schema_id[len('student_'):],None)
1996            if object is None:
1997                continue
1998            doc = object.getContent()
1999            for key in schema.keys():
2000                if schema[key].meta_type != "CPS Image Field":
2001                    continue
2002                #import pdb;pdb.set_trace()
2003                image = getattr(doc,key,None)
2004                if not image or not hasattr(image,"data"):
2005                    continue
2006                if not created:
2007                    if not os.path.exists(images_dir):
2008                        os.mkdir(images_dir)
2009                    created = True
2010                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2011                open(filename,"wb").write(str(image.data))
2012    ###)
2013
2014    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2015    def movePassportToFS(self,student_id="O738726"):
2016        "move the passports to the filesystem"
2017        images_dir = os.path.join("%s" % i_home,'passports')
2018        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2019        stool = getToolByName(self, 'portal_schemas')
2020        schemas = ['student_application',
2021                   #'student_clearance',
2022                   ]
2023        created = False
2024        for schema_id in schemas:
2025            schema = stool._getOb(schema_id)
2026            object = getattr(student_folder,schema_id[len('student_'):],None)
2027            if object is None:
2028                continue
2029            doc = object.getContent()
2030            for key in schema.keys():
2031                if schema[key].meta_type != "CPS Image Field":
2032                    continue
2033                #import pdb;pdb.set_trace()
2034                image = getattr(doc,key)
2035                if not hasattr(image,"data"):
2036                    continue
2037                if not created:
2038                    if not os.path.exists(images_dir):
2039                        os.mkdir(images_dir)
2040                    created = True
2041                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2042                open(filename,"wb").write(str(image.data))
2043    ###)
2044
2045InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.