source: WAeUP_SRP/base/WAeUPTool.py @ 2750

Last change on this file since 2750 was 2750, checked in by Henrik Bettermann, 17 years ago

'int' object has no attribute 'isdigit'
  • Property svn:keywords set to Id
File size: 82.0 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2750 2007-11-23 21:27:49Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
65    """WAeUP tool"""
66
67    id = 'waeup_tool'
68    meta_type = 'WAeUP Tool'
69    _actions = ()
70    security = ClassSecurityInfo()
71    security.declareObjectProtected(View)
72    manage_options = ( ActionProviderBase.manage_options
73                     + SimpleItem.manage_options
74                     )
75
76    def rwrite(self,s): ###(
77        response = self.REQUEST.RESPONSE
78        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
79        response.write("%s<br />\r\n" % s)
80    ###)
81
82    def addtodict(self,d,key,item): ###(
83        d[key].append(item)
84        return d[key]
85    ###)
86
87    def sleep(self,secs): ###(
88        "sleep"
89        import time
90        time.sleep(secs)
91        return
92
93###)
94
95    security.declareProtected(View,'updateRoleMappingsFor') ###(
96    def updateRoleMappingsFor(self,wf_definition,ob):
97        "do so for public"
98        wf_def = getattr(self.portal_workflow,wf_definition)
99        wf_def.updateRoleMappingsFor(ob)
100    ###)
101
102    security.declareProtected(ModifyPortalContent,'openLog') ###(
103    def openLog(self,name):
104        """open a log file"""
105        version = 1
106        path = "%s/log/%s_%d.log" % (i_home,name,version)
107        while os.path.exists(path):
108            version += 1
109            path = "%s/log/%s_%d.log" % (i_home,name,version)
110        log = open(path,"w")
111        return log
112    ###)
113
114    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
115    def bypassQueueCatalog(self,enable=True):
116        """bypass the QueueCatalog by setting all indexes to process imediate,
117        if enable is True (default) the old settings are restored
118        """
119
120    ###)
121
122    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
123    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
124        """measure Object access Time"""
125        import random
126        if hasattr(self,'portal_catalog_real'):
127            aq_portal = self.portal_catalog_real.evalAdvancedQuery
128        else:
129            aq_portal = self.portal_catalog.evalAdvancedQuery
130        nr_pts = int(nr_pts)
131        probe = int(probe)
132        intervall = probe/10
133        objects = ("application","clearance","personal")
134        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
135        #i = random.randrange(num_objects)
136        count = 0
137        found = 0
138        not_found = 0
139        t_found = 0
140        t_not_found = 0
141        time_found = time_not_found = 0.0
142        t_time_found = t_time_not_found = 0.0
143        accessed = []
144        t_min = 1000
145        t_max = 0
146        #import pdb;pdb.set_trace()
147        students = self.portal_catalog(portal_type="Student")
148        num_students = len(students)
149        if method == "d":
150            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
151            res = aq_portal(query)
152            brains = {}
153            for r in res:
154                sid = r.relative_path.split('/')[-2]
155                if brains.has_key(sid):
156                    brains[sid][r.portal_type] = r
157                else:
158                    brains[sid] = {r.portal_type : r}
159            brains_list = brains.keys()
160            num_objects = len(brains_list)
161        else:
162            num_objects = num_students
163        print "="*40
164        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
165                                                                        probe,
166                                                                        nr_pts,
167                                                                        num_objects)
168        print "nr found/not time found/not min/max"
169        elapse = time.time()
170        i_elapse = time.time()
171        c_elapse = time.clock()
172        for c in range(1,probe + 1):
173            i = random.randrange(num_objects)
174            if method in ('a','b','c'):
175                student_brain = students[i]
176            elif method == "d":
177                #import pdb;pdb.set_trace()
178                student_brain = brains[brains_list[i]]
179            if method == "c":
180                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
181                res = aq_portal(query)
182                this_portal_types = [r.portal_type for r in res]
183            for i in range(nr_pts):
184                oid = objects[i]
185                if method == "a":
186                    try:
187                        student_path = student_brain.getPath()
188                        path = "%s/%s" % (student_path,oid)
189                        doc = self.unrestrictedTraverse(path).getContent()
190                        found += 1
191                        i_time = time.time() - i_elapse
192                        time_found += i_time
193                    except:
194                        not_found += 1
195                        i_time = time.time() - i_elapse
196                        time_not_found += i_time
197                        pass
198                elif method == "b":
199                    try:
200                        student_object = student_brain.getObject()
201                        doc = getattr(student_object,oid).getContent()
202                        found += 1
203                        i_time = time.time() - i_elapse
204                        time_found += i_time
205                    except:
206                        i_time = time.time() - i_elapse
207                        time_not_found += i_time
208                        not_found += 1
209                        pass
210                elif method == "c":
211                    if portal_types[i] in this_portal_types:
212                        found += 1
213                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
214                        i_time = time.time() - i_elapse
215                        time_found += i_time
216                    else:
217                        not_found += 1
218                        i_time = time.time() - i_elapse
219                        time_not_found += i_time
220                elif method == "d":
221                    if student_brain.has_key(portal_types[i]):
222                        found += 1
223                        doc = student_brain[portal_types[i]].getObject().getContent()
224                        i_time = time.time() - i_elapse
225                        time_found += i_time
226                    else:
227                        not_found += 1
228                        i_time = time.time() - i_elapse
229                        time_not_found += i_time
230                i_elapse = time.time()
231            if c and (c % intervall == 0):
232                #i_time = time.time() - i_elapse
233                t_per = 0.0
234                if found:
235                    t_per = time_found/found
236                if t_per > t_max:
237                    t_max = t_per
238                if t_per > 0.0 and t_per < t_min:
239                    t_min = t_per
240                itf = 0.0
241                if found:
242                    itf = time_found/found
243                itnf = 0.0
244                if not_found :
245                    itnf = time_not_found / not_found
246                interval_time = time_found + time_not_found
247                s = "%(c)d: %(found)d/%(not_found)d " % vars()
248                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
249                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
250                print s
251                t_found += found
252                t_not_found += not_found
253                t_time_found += time_found
254                t_time_not_found += time_not_found
255                time_found = time_not_found = 0.0
256                found = not_found = 0
257        # t_found += found
258        # t_not_found += not_found
259        elapse = time.time() - elapse
260        itf = 0.0
261        if t_found:
262            itf = t_time_found/t_found
263        itnf = 0.0
264        if t_not_found:
265            itnf = t_time_not_found / t_not_found
266        #c_elapse = time.clock() - c_elapse
267        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
268        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
269        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
270        print "-"*40
271        print s
272        rel_found = float(t_found)/probe
273        rel_not_found = float(t_not_found)/probe
274        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
275        print estimated_total_time
276    ###)
277
278    security.declareProtected(ModifyPortalContent,'writeLog') ###(
279    def writeLog(self,logfile,s):
280        """write to the log file"""
281        logfile.write(s)
282
283###)
284
285    def generateStudentId(self,letter): ###(
286        import random
287        r = random
288        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
289        if letter == '?':
290            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
291        sid = "%c%d" % (letter,r.randint(99999,1000000))
292        students = self.portal_url.getPortalObject().campus.students
293##        while hasattr(students, sid):
294##            sid = "%c%d" % (letter,r.randint(99999,1000000))
295        while self.students_catalog(id = sid):
296            sid = "%c%d" % (letter,r.randint(99999,1000000))
297        return sid
298    ###)
299
300    def generatePassword(self,s=None): ###(
301        import random
302        r = random
303        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
304        if s is None:
305            s = 'abcdefghklmnpqrstuvwxy23456789'
306        pw = ''
307        while len(pw) < 6:
308            pw += r.choice(s)
309        return pw
310    ###)
311
312    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
313    def dumpSchoolfeePayments(self):
314        "dump paid schoolfees"
315        mtool = self.portal_membership
316        member = mtool.getAuthenticatedMember()
317        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
318        aq_student = self.students_catalog.evalAdvancedQuery
319        query = In('review_state',('schoolfee_paid',
320                                   'courses_registered',
321                                   'courses_validated',
322                                   ))
323        res = aq_student(query)
324        #import pdb;pdb.set_trace()
325        l = []
326        logger.info("start for %d" % len(res))
327        count = 1
328        log_after = 100
329        for student in res:
330            if not count % log_after:
331                logger.info("processed %d total %d" % (log_after,count))
332            count += 1
333            fee_dict =self.getSchoolFee(student)
334            fulltime = student.mode.endswith('_ft')
335            d = {}
336            d['student_id'] = student.id
337            d['name'] = student.name
338            d['amount'] = fee_dict.get(new_returning)
339            l += d,
340        csv_name = self.dumpListToCSV(l,'payments')
341        logger.info('%s dumped payments to %s' % (member,export_file))
342    ###)
343
344    security.declarePublic('dumpListToCSV') ###(
345    def dumpListToCSV(self,l,filename,fields=None):
346        """dump a list of dicts to a CSV file"""
347        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
348        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
349        if fields is None:
350            fields = l[0].keys()
351        headline = ','.join(fields)
352        out = open(export_file,"wb")
353        out.write(headline +'\n')
354        out.close()
355        out = open(export_file,"a")
356        csv_writer = csv.DictWriter(out,fields,)
357        csv_writer.writerows(l)
358        return export_file
359    ###)
360
361    security.declareProtected(ManagePortal, 'listMembers') ###(
362    def listMembers(self):
363        "list all members"
364        mtool = self.portal_membership
365        member = mtool.getAuthenticatedMember()
366        logger = logging.getLogger('WAeUPTool.listMembers')
367        if str(member) not in ('admin','joachim'):
368            logger.info('%s tried to list members' % (member))
369            return None
370        members = self.portal_directories.members
371        all = members.listEntryIdsAndTitles()
372        l = []
373        for user_id,name in all:
374            d = {}
375            d['user_id'] = user_id
376            d['name'] = name
377            d['pw'] = getattr(getattr(members,user_id),'password')
378            d['email'] = getattr(getattr(members,user_id),'email')
379            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
380            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
381            l += d,
382        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
383        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
384        logger.info('%s dumped memberlist to %s' % (member,export_file))
385        fields = l[0].keys()
386        headline = ','.join(fields)
387        out = open(export_file,"wb")
388        out.write(headline +'\n')
389        out.close()
390        out = open(export_file,"a")
391        csv_writer = csv.DictWriter(out,fields,)
392        csv_writer.writerows(l)
393    ###)
394
395    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
396    def removeDeletedDocIds(self, max=1000):
397        """
398        remove deleted docids from repository commit after max
399        """
400        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
401        repository = getToolByName(self, 'portal_repository')
402        pxtool = getToolByName(self, 'portal_proxies')
403        logger.info('start')
404        pxtool_infos = pxtool.getRevisionsUsed()
405        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
406
407        nb_revs = 0
408        docids_d = {} # all docids
409        unused_docids_d = {} # all docids that are unused
410        ids_unused_revs_docids = [] # ids for revs of unused docids
411        unused_ids = [] # ids for unused revs
412        total = 0
413        idlist = repository.objectIds()
414        to_delete = 0
415        found = False
416        for id in idlist:
417            docid, rev = repository._splitId(id)
418            if docid is None:
419                logger.info("invalid doc_id %s" % docid)
420                continue
421            nb_revs += 1
422            if not pxtool_infos.has_key(docid):
423                found = True
424                to_delete += 1
425                unused_ids.append(id)
426            elif not pxtool_infos[docid].has_key(rev):
427                found = True
428                to_delete += 1
429                unused_ids.append(id)
430            if found and not to_delete % max:
431                found = False
432                #import pdb;pdb.set_trace()
433                repository.manage_delObjects(unused_ids)
434                transaction.commit()
435                logger.info('removed %d total %d unused docids ' % (max,to_delete))
436        else:
437            if unused_ids:
438                repository.manage_delObjects(unused_ids)
439                transaction.commit()
440        logger.info('finished removing %d unused docids ' % (to_delete))
441
442
443###)
444
445    security.declareProtected(View,'getCredential') ###(
446    def getCredential(self,student_id):
447        student_entry = getattr(self.portal_directories.students,student_id,None)
448        if not self.isStaff():
449            mtool = self.portal_membership
450            member = mtool.getAuthenticatedMember()
451            logger = logging.getLogger('WAeUPTool.getCredential')
452            logger.info('%s tried to access password of %s' % (member,student_id))
453            return None
454        if student_entry is None:
455            return None
456        return getattr(student_entry,"password","not set")
457    ###)
458
459    security.declarePublic('checkPassword') ###(
460    def checkPassword(self,student_id,password):
461        student_entry = getattr(self.portal_directories.students,student_id,None)
462        if student_entry is None:
463            return False
464        return getattr(student_entry,"password","not set") == password
465    ###)
466
467    security.declarePublic('checkGenericPassword') ###(
468    def checkGenericPassword(self,member_id):
469        member_entry = getattr(self.portal_directories.members,member_id,None)
470        if member_entry is None:
471            return False
472        ltool = getToolByName(self, 'portal_layouts')
473        unsecure_words = ltool._getOb('members')['w__password'].check_words
474        password = getattr(member_entry,"password","not set")
475        is_unsecure = password in unsecure_words
476        if is_unsecure:
477            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
478            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
479        return is_unsecure
480    ###)
481
482    security.declareProtected(ModifyPortalContent,'editPassword') ###(
483    def editPassword(self,student_id,password):
484        "edit a student password"
485        student_entry = getattr(self.portal_directories.students,student_id,None)
486        if student_entry is None:
487            return
488        setattr(student_entry,'password',password)
489    ###)
490
491    security.declareProtected(ModifyPortalContent,'doCommit') ###(
492    def doCommit(self,logger=None):
493        "commit some transactions"
494        transaction.commit()
495    ###)
496
497    security.declarePublic('loadStudentFoto') ###(
498    def loadStudentFoto(self,student,filename,folder):
499        "return a student passport picture"
500        #import pdb;pdb.set_trace()
501        picture ="%s/import/%s/%s" % (i_home,folder,filename)
502        student_id = student.getId()
503        images_dir = getImagesDir(student_id)
504        if not os.path.exists(images_dir):
505            os.mkdir(images_dir)
506        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
507        for extension in ('.jpg','.JPG'):
508            fullname = "%(picture)s%(extension)s" % vars()
509            if os.path.exists(fullname):
510                copy2(fullname,image_name)
511                return "successfully copied passport picture"
512        return "passport picture not found: %s.jpg or .JPG" % picture
513    ###)
514
515    def old____loadStudentFoto(self,student,filename,folder): ###(
516        "return a student passport picture"
517        app = student.application
518        app_doc = app.getContent()
519        #clear = student.clearance
520        #clear_doc = clear.getContent()
521        #matric_no = clear_doc.matric_no.upper()
522        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
523        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
524        #import pdb;pdb.set_trace()
525        if os.path.exists(picture1):
526            file = open(picture1)
527        elif os.path.exists(picture2):
528            file = open(picture2)
529        else:
530            return "passport picture not found %s" % picture1
531        reopened = False
532        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
533            self.portal_workflow.doActionFor(app,'open')
534            reopened = True
535        outfile = file.read()
536        app_doc.manage_addFile('passport',
537                               file=outfile,
538                               title="%s.jpg" % filename)
539        if reopened:
540            self.portal_workflow.doActionFor(app,'close')
541        return "successfully loaded passport picture"
542    ###)
543
544    security.declareProtected(ModifyPortalContent,'createOne') ###(
545    def createOne(self,students_folder,student_brain,letter,commit=False):
546        sid = self.waeup_tool.generateStudentId(letter)
547        students_folder.invokeFactory('Student', sid)
548        student = getattr(students_folder,sid)
549        self.portal_workflow.doActionFor(student,'return')
550        student.manage_setLocalRoles(sid, ['Owner',])
551        matric_no = student_brain.matric_no
552        jamb_reg_no = student_brain.Entryregno
553        self.students_catalog.addRecord(id = sid,
554                                           matric_no = matric_no,
555                                           jamb_reg_no = jamb_reg_no,
556                                           sex = student_brain.Sex == "F",
557                                           name = "%s %s %s" % (student_brain.Firstname,
558                                                                student_brain.Middlename,
559                                                                student_brain.Lastname)
560                                        )
561        if commit:
562            transaction.commit()
563        return sid,jamb_reg_no
564    ###)
565
566    security.declareProtected(ModifyPortalContent,'addStudent') ###(
567    def addStudent(self,dict):
568        students_folder = self.portal_url.getPortalObject().campus.students
569        sid = self.waeup_tool.generateStudentId('?')
570        students_folder.invokeFactory('Student', sid)
571        student_obj = getattr(students_folder,sid)
572        f2t = self.student_field2types
573        #from pdb import set_trace; set_trace()
574        d = {}
575        #d['jamb_sex']  = 'M'
576        #if dict.get('sex'):
577        #    d['jamb_sex']  = 'F'
578
579        entry_session = dict.get('entry_session')
580        if entry_session == self.getSessionId()[0]:
581            wfaction = 'admit'
582            wft = 'wf_transition_admit'
583            password = None
584        else:
585            wfaction = 'return'
586            wft = 'wf_transition_return'
587            password = self.generatePassword()
588            self.makeStudentMember(sid,password)
589
590        for pt in f2t.keys():
591            student_obj.invokeFactory(pt,f2t[pt]['id'])
592            sub_obj = getattr(student_obj,f2t[pt]['id'])
593            sub_doc = sub_obj.getContent()
594            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
595            d['Title'] = f2t[pt]['title']
596            for field in f2t[pt]['fields']:
597                d[field] = dict.get(field,'')
598            sub_doc.edit(mapping = d)
599            new_state = f2t[pt][wft]
600            if new_state != "remain":
601                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
602        self.portal_workflow.doActionFor(student_obj,wfaction)
603        student_obj.manage_setLocalRoles(sid, ['Owner',])
604        return sid,password
605    ###)
606
607    security.declarePublic('getCertificateBrain') ###(
608    def getCertificateBrain(self,cert_id):
609        "do it"
610        res = ZCatalog.searchResults(self.portal_catalog_real,
611                                {'portal_type':"Certificate",
612                                      'id': cert_id})
613        if res:
614            return res[0]
615        return None
616    ###)
617
618    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
619    def get_csv_filenames(self):
620        "do it"
621        files = [file for file in os.listdir("%s/import/" % (i_home))
622                 if file.endswith('.csv') and file.find('imported') == -1]
623        return files
624    ###)
625
626    security.declarePublic('findStudentByMatricelNo') ###(
627    def findStudentByMatricelNo(self,matric_no):
628        "do it"
629        res = ZCatalog.searchResults(self.portal_catalog_real,
630                                {'portal_type':"StudentClearance",
631                                 'SearchableText': matric_no})
632        if res:
633            return res[0]
634        return None
635    ###)
636
637    security.declarePublic('makeStudentMember') ###(
638    def makeStudentMember(self,sid,password='uNsEt'):
639        """make the student a member"""
640        membership = self.portal_membership
641        membership.addMember(sid,
642                             password ,
643                             roles=('Member',
644                                     'Student',
645                                     ),
646                             domains='',
647                             properties = {'memberareaCreationFlag': False,
648                                           'homeless': True},)
649        member = membership.getMemberById(sid)
650        self.portal_registration.afterAdd(member, sid, password, None)
651        #self.manage_setLocalRoles(sid, ['Owner',])
652    ###)
653
654    security.declareProtected(View,'makeStudentData') ###(
655    def makeStudentData(self,student_id,email=None,phone_nr=None):
656        "create Datastructure for a returning Student"
657        #import pdb;pdb.set_trace()
658        logger = logging.getLogger('WAeUPTool.makeStudentData')
659        students_folder = self.portal_url.getPortalObject().campus.students
660        #res = self.students_catalog(id=student_id)
661        #if res:
662        #    st = res[0]
663        #res = self.returning_import(matric_no = st.matric_no)
664        res = self.returning_import(id = student_id)
665        if res:
666            student = res[0]
667        else:
668            logger.info('Id %s not found in returning_import' % student_id)
669            return
670        logger.info('%s creates data structure' % student_id)
671        s_results = self.results_import(matric_no = student.matric_no)
672        if s_results:
673            lnr = self.getLevelFromResultsCosCode(s_results)
674            level = "%d00" % lnr
675            verdict,eligible = self.getVerdict(s_results[0].Verdict)
676            #if eligible:
677            #    level = "%d00" % (lnr + 1)
678        else:
679            logger.info('matric_no %s not found in results_import' % student.matric_no)
680            level = ''
681            verdict = ''
682        #student should not be allowed to perform this transition
683        #wftool = self.portal_workflow
684        #wftool.doActionFor(student,'return')
685        certcode_org = student.Coursemajorcode
686        certcode = makeCertificateCode(certcode_org)
687        certificate_brain = self.getCertificateBrain(certcode)
688        if not certificate_brain:
689            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
690            logger.info(em)
691        matric_no = student.matric_no
692        sid = student_id
693        student_obj = getattr(students_folder,sid)
694        student_obj.invokeFactory('StudentApplication','application')
695        application = student_obj.application
696        self.portal_workflow.doActionFor(application,'open',dest_container=application)
697        da = {'Title': 'Application Data'}
698        student_obj.invokeFactory('StudentPersonal','personal')
699        da['jamb_reg_no'] = student.Entryregno
700        em = self.getEntryMode(student.Entryregno)
701        da['entry_mode'] = em
702        personal = student_obj.personal
703        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
704        dp = {'Title': 'Personal Data'}
705        student_obj.invokeFactory('StudentClearance','clearance')
706        clearance = student_obj.clearance
707        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
708        dc = {'Title': 'Clearance/Eligibility Record'}
709        dc['matric_no'] = matric_no
710        state = student.State
711        lga = student.LGA
712        if state and lga:
713            lga =  state + ' / ' + lga
714        else:
715            lga = "None"
716        da['jamb_lga'] = dc['lga'] = lga
717        da['app_email'] = dp['email'] = email
718        da['app_mobile'] = dp['phone'] = phone_nr
719        dp['firstname'] = student.Firstname
720        dp['middlename'] = student.Middlename
721        dp['lastname'] = student.Lastname
722        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
723        da['jamb_sex'] = student.Sex
724        dp['sex'] = student.Sex == 'F'
725        dp['perm_address'] = student.Permanent_Address
726        application.getContent().edit(mapping=da)
727        self.portal_workflow.doActionFor(application,'close',dest_container=application)
728        personal.getContent().edit(mapping=dp)
729        clearance.getContent().edit(mapping=dc)
730        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
731        #
732        # Study Course
733        #
734        student_obj.invokeFactory('StudentStudyCourse','study_course')
735        studycourse = student_obj.study_course
736        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
737        dsc = {}
738        dsc['study_course'] = certcode
739        dsc['current_level'] = level
740        dsc['current_verdict'] = verdict
741        dsc['current_mode'] = em
742        dsc['current_session'] = '05'
743        studycourse.getContent().edit(mapping=dsc)
744        #
745        # Level
746        #
747##        l = getattr(studycourse,level,None)
748##        if l is None:
749##            studycourse.invokeFactory('StudentStudyLevel', level)
750##            l = getattr(studycourse, level)
751##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
752##            l.getContent().edit(mapping={'Title': "Level %s" % level})
753###)
754
755    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
756    def admitOneStudent(self,brain,entry_session,pin_password):
757        "create Datastructure for an admitted Student"
758        #import pdb;pdb.set_trace()
759        if not hasattr(self,"_v_certificates"):
760            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
761        students_folder = self.portal_url.getPortalObject().campus.students
762        logger = logging.getLogger('WAeUPTool.admitOneStudent')
763        if brain.status != "admitted":
764            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
765            return
766        pin_parts = brain.pin.split('-')
767        if len(pin_parts) != 3:
768            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
769            return
770        if brain.course_admitted not in self._v_certificates:
771            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
772            return
773        student_id = self.generateStudentId('?')
774        students_folder.invokeFactory('Student', student_id)
775        student_object = getattr(students_folder,student_id)
776        if pin_password:
777            password = pin_parts[2]
778            self.makeStudentMember(student_id,password = password)
779        student_object.manage_setLocalRoles(student_id, ['Owner',])
780        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
781        #
782        # application
783        #
784        student_object.invokeFactory('StudentApplication','application')
785        application = student_object.application
786        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
787        da = {'Title': 'Application Data'}
788        da['jamb_reg_no'] = brain.reg_no
789
790        sex = 'M'
791        if brain.sex:
792            sex = 'F'
793        da['jamb_sex'] = sex
794        da['jamb_age'] = brain.jamb_age
795        #da['app_ac_pin'] = brain.pin
796        da['jamb_lga'] = brain.jamb_lga
797        da['jamb_state'] = brain.jamb_state
798        da['jamb_score'] = brain.aggregate
799        da['app_email'] = brain.email
800        da['app_mobile'] = brain.phone
801        if brain.entry_mode:
802            da['entry_mode'] = brain.entry_mode
803        elif brain.screening_type == 'pume':
804            da['entry_mode'] = 'ume_ft'
805        elif brain.screening_type == 'pde':
806            da['entry_mode'] = 'de_ft'
807        else:
808            da['entry_mode'] = brain.screening_type
809        da['entry_session'] = entry_session
810        da['jamb_lastname'] = brain.lastname
811        da['jamb_middlename'] = brain.middlenames   # different field names!
812        da['jamb_firstname'] = brain.firstname
813        da['screening_application_date'] = brain.application_date
814        da['date_of_birth'] = brain.date_of_birth
815        da['jamb_first_cos'] = brain.course1
816        da['jamb_second_cos'] = brain.course2
817        da['course3'] = brain.course3
818        da['screening_type'] = brain.screening_type
819        da['screening_score'] = brain.screening_score
820        da['screening_date'] = brain.screening_date
821        da['hq_type'] = brain.hq_type
822        da['hq_grade'] = brain.hq_grade
823        da['aos'] = brain.aos
824
825        application.getContent().edit(mapping=da)
826        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
827        #
828        # personal
829        #
830        student_object.invokeFactory('StudentPersonal','personal')
831        personal = student_object.personal
832        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
833        dp = {'Title': 'Personal Data'}
834        dp['sex'] = brain.sex
835        dp['email'] = brain.email
836        dp['phone'] = brain.phone
837        dp['lastname'] = brain.lastname
838        dp['middlename'] = brain.middlenames   # different field names!
839        dp['firstname'] = brain.firstname
840        personal.getContent().edit(mapping=dp)
841        #
842        # clearance
843        #
844        student_object.invokeFactory('StudentClearance','clearance')
845        clearance = student_object.clearance
846        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
847        dc = {'Title': 'Clearance/Eligibility Record'}
848        dc['lga'] = brain.lga
849        dc['birthday'] = brain.date_of_birth
850        clearance.getContent().edit(mapping=dc)
851        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
852        #
853        # study Course
854        #
855        student_object.invokeFactory('StudentStudyCourse','study_course')
856        studycourse = student_object.study_course
857        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
858        dsc = {}
859        dsc['study_course'] = brain.course_admitted
860        dsc['current_verdict'] = ''
861        dsc['current_mode'] = da['entry_mode']
862        if da['entry_mode'].startswith('de'):
863            dsc['current_level'] = '200'
864        else:
865            dsc['current_level'] = '100'
866        dsc['current_session'] = entry_session
867        studycourse.getContent().edit(mapping=dsc)
868        #
869        # payments folder
870        student_object.invokeFactory('PaymentsFolder','payments')
871        payments = getattr(student_object,'payments')
872        dpay = {}
873        dpay['Title'] = 'Payments'
874        payments.getContent().edit(mapping=dpay)
875        self.portal_workflow.doActionFor(payments,'open')
876        #
877        # passport foto
878        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
879                                                              brain.screening_type,
880                                                              brain.reg_no)
881        images_dir = getImagesDir(student_id)
882        #images_dir = os.path.join("%s" % images_base,student_id)
883        letter_dir,student_dir = os.path.split(images_dir)
884        if not os.path.exists(letter_dir):
885            os.mkdir(letter_dir)
886        if not os.path.exists(images_dir):
887            os.mkdir(images_dir)
888        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
889        if os.path.exists(app_picture):
890            copy2(app_picture,image_name)
891        else:
892            logger.info('passport of %s/%s not found: %s' % (student_id,
893                                                             brain.reg_no,
894                                                             app_picture))
895
896        return student_id
897    ###)
898
899    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
900    def makeStudentLevel(self,student_id):
901        "create the StudyLevel for a returning Student"
902        #import pdb;pdb.set_trace()
903        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
904        students_folder = self.portal_url.getPortalObject().campus.students
905        res = self.students_catalog(id=student_id)
906        if res:
907            st = res[0]
908        course = st.course
909        matric_no = st.matric_no
910        level = st.level
911        res = self.results_import(matric_no = matric_no)
912        if res:
913            results = res
914        logger.info('%s creating Level %s' % (student_id,level))
915        #
916        # Level
917        #
918        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
919        studycourse = getattr(student_obj,"study_course",None)
920        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
921        l = getattr(studycourse,level,None)
922        if l is None:
923            studycourse.invokeFactory('StudentStudyLevel', level)
924            l = getattr(studycourse, level)
925            self.portal_workflow.doActionFor(l,'open',dest_container=l)
926            l.getContent().edit(mapping={'Title': "Level %s" % level})
927        ###)
928
929    security.declarePublic('getHallInfo') ###(
930    def getHallInfo(self,bed):
931        """return Hall Info"""
932        info = {}
933        hall,block,room,letter = bed.split('_')
934        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
935        if res and len(res) == 1:
936            hall_brain = res[0]
937            hall_doc = hall_brain.getObject().getContent()
938        else:
939            return info
940        info['hall_title'] = hall_brain.Title
941        info['maintenance_code'] = hall_doc.maintenance_code
942        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
943        batch_doc = None
944        for brain in res:
945            if brain.id.startswith(info['maintenance_code']):
946                batch_doc = brain.getObject().getContent()
947                break
948        if batch_doc is None:
949            info['maintenance_fee'] = None
950        else:
951            info['maintenance_fee'] = batch_doc.cost
952        return info
953    ###)
954
955    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
956    def removePictureFolder(self,student_id):
957        """remove picture_folder by renaming it"""
958        path = 'images'
959        picture_path = os.path.join(i_home,path,student_id)
960        if not os.path.exists(picture_path):
961            return False
962        os.rename(picture_path,picture_path + "_removed")
963        return True
964    ###)
965
966    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
967    def restorePictureFolder(self,student_id):
968        """restore picture_folder by renaming it"""
969        path = 'images'
970        picture_path = os.path.join(i_home,path,student_id)
971        if not os.path.exists(picture_path + "_removed"):
972            return False
973        os.rename(picture_path + "_removed",picture_path)
974        return True
975    ###)
976
977    security.declarePublic('picturesExist') ###(
978    def picturesExist(self, ids,student_id=None):
979        """check if pictures exist in the filesystem"""
980        if student_id is None:
981            student_id = self.getStudentId()
982        if student_id is None:
983            return False
984        picture_path = getImagesDir(student_id)
985        #picture_path = os.path.join(images_base,student_id)
986        if not os.path.exists(picture_path):
987            return False
988        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
989        return set(ids).issubset(set(pictures))
990    ###)
991
992    security.declarePublic('picturesList') ###(
993    def picturesList(self):
994        """check if pictures exist in the filesystem"""
995        path = 'images'
996        student_id = self.getStudentId()
997        #picture_path = os.path.join(i_home,path,student_id)
998        picture_path = getImagesDir(student_id)
999        if not os.path.exists(picture_path):
1000            return []
1001        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1002    ###)
1003
1004    security.declarePublic('showFsPicture') ###(
1005    def showFsPicture(self,path):
1006        """return a picture from the filesystem"""
1007        #picture_path = os.path.join(i_home,path)
1008        picture_path = os.path.join(images_base,path)
1009        response = self.REQUEST.RESPONSE
1010        #import pdb;pdb.set_trace()
1011        registry = getToolByName(self, 'mimetypes_registry')
1012        mimetype = str(registry.lookupExtension(path.lower()) or
1013                    registry.lookupExtension('file.bin'))
1014        if os.path.exists(picture_path):
1015            response.setHeader('Content-type',mimetype)
1016            return open(picture_path).read()
1017        picture_path = os.path.join(i_home,'import',path)
1018        if os.path.exists(picture_path):
1019            return open(picture_path).read()
1020    ###)
1021
1022    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1023    def deleteAllCourses(self,department="All"):
1024        ''' delete the courses'''
1025        pm = self.portal_membership
1026        member = pm.getAuthenticatedMember()
1027
1028        if str(member) not in ("henrik","joachim"):
1029            return "not possible"
1030        if department == "All":
1031            res = self.portal_catalog({'meta_type': 'Department'})
1032        if len(res) < 1:
1033            return "No Departments found"
1034
1035        deleted = []
1036        for dep in res:
1037            cf = dep.getObject().courses
1038            if cf:
1039                cf.manage_delObjects(ids=cf.objectIds())
1040                deleted.append("deleted Courses in %s" % dep.getId)
1041        return "\r".join(deleted)
1042    ###)
1043
1044    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1045    def getLogfileLines(self,filename="event.log",numlines=20):
1046        """Get last NUMLINES lines of logfile FILENAME.
1047
1048        Return last lines' of a file in the instances logfile directory as
1049        a list. The number of returned lines equals `numlines' or less. If
1050        less than `numlines' lines are available, the whole file ist
1051        returned. If the file can not be opened or some other error
1052        occurs, empty list is returend.
1053        """
1054        result = []
1055        lines_hit = 0
1056
1057        # We only handle files in instances' log directory...
1058        logpath = os.path.join(i_home, "log")
1059        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1060        if not filename.startswith( logpath ):
1061            # Attempt to access file outside log-dir...
1062            return []
1063
1064        try:
1065            fd = file( filename, "rb" )
1066        except IOError:
1067            return []
1068        if not fd:
1069            return []
1070
1071        if os.linesep == None:
1072            linesep = '\n'
1073        else:
1074            linesep = os.linesep
1075
1076        # Try to find 'numlines' times a lineseparator, searching from end
1077        # and moving to the beginning of file...
1078        fd.seek( 0, 2) # Move to end of file...
1079        while lines_hit < numlines:
1080            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1081                                          # one step forward...
1082                lines_hit += 1
1083            try:
1084                fd.seek( -2, 1) # Go two bytes back from current pos...
1085            except IOError:
1086                # We cannot go back two bytes. Maybe the file is too small...
1087                break
1088        fd.seek(2,1)
1089
1090        # Read all lines from current position...
1091        result = fd.readlines()
1092        # Remove line endings...
1093        result = [x.strip() for x in result]
1094        fd.close()
1095        return result
1096    ###)
1097
1098    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1099    def getCallbacksFromLog(self,filename):
1100        """fix Online Payment Transactions from Z2.log entries"""
1101        import transaction
1102        import random
1103        from cgi import parse_qs
1104        from urlparse import urlparse
1105        #from pdb import set_trace
1106        wftool = self.portal_workflow
1107        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1108        students_folder = self.portal_url.getPortalObject().campus.students
1109        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1110        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1111        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1112        data = re.compile(s)
1113        start = True
1114        tr_count = 1
1115        total = 0
1116        #name = 'pume_results'
1117        #name = 'epaymentsuccessful_z2log2'
1118        name = filename
1119        no_import = []
1120        imported = []
1121        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1122        try:
1123            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1124        except:
1125            logger.error('Error reading %s' % name)
1126            return
1127        tas = []
1128        for line in transactions:
1129            dict = {}
1130            items = data.search(line)
1131            dict['idict'] = idict = items.groupdict()
1132            #print idict
1133            #from pdb import set_trace;set_trace()
1134            urlparsed = urlparse(idict['get'][4:])
1135            #print urlparsed
1136            path = urlparsed[2].split('/')
1137            dict['student_id'] = student_id = path[8]
1138            dict['payment_id'] = payment_id = path[10]
1139            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1140            tas.append(dict)
1141            tr_count += 1
1142        return tas
1143    ###)
1144
1145    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1146    def importOnlinePaymentTransactions(self):
1147        """load Online Payment Transactions from CSV values"""
1148        import transaction
1149        import random
1150        #from pdb import set_trace
1151        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1152        opt = self.online_payments_import
1153        students_folder = self.portal_url.getPortalObject().campus.students
1154        start = True
1155        tr_count = 1
1156        total = 0
1157        #name = 'pume_results'
1158        name = 'OnlineTransactions'
1159        no_import = []
1160        imported = []
1161        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1162        try:
1163            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1164        except:
1165            logger.error('Error reading %s.csv' % name)
1166            return
1167        for pay_transaction in transactions:
1168            if start:
1169                start = False
1170                logger.info('Start loading from %s.csv' % name)
1171                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1172                no_import.append('%s,"Error"' % s)
1173                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1174                format_error = format + ',"%(Error)s"'
1175            data = {}
1176
1177            # format of the first file sent by Tayo
1178            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1179            #data['student_id'] = student_id = pay_transaction['Payer ID']
1180            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1181            #data['response_code'] = response_code = pay_transaction['Resp Code']
1182            #data['amount'] = amount = pay_transaction['Amount']
1183
1184            # format of the second file sent by Tayo
1185            #data['datetime'] = date = 0
1186            #data['student_id'] = student_id = pay_transaction['Payer ID']
1187            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1188            #data['response_code'] = response_code = '00'
1189            #data['amount'] = amount = pay_transaction['Amount']
1190
1191            # format of the third file sent by Kehinde
1192            data['datetime'] = date = 0
1193            data['student_id'] = student_id = pay_transaction['customer_id']
1194            data['order_id'] = order_id = pay_transaction['merchant_reference']
1195            data['response_code'] = response_code = '00'
1196            data['amount'] = amount = pay_transaction['Amount']
1197
1198            dup = False
1199            if response_code == "12":
1200                continue
1201            try:
1202                opt.addRecord(**data)
1203            except ValueError:
1204                dup = True
1205            #from pdb import set_trace;set_trace()
1206            if dup:
1207                if response_code == "00":
1208                    try:
1209                        opt.modifyRecord(**data)
1210                    except:
1211                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1212                        continue
1213                else:
1214                    pay_transaction['Error'] = "Duplicate order_id"
1215                    no_import.append( format_error % pay_transaction)
1216                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1217                    continue
1218            tr_count += 1
1219            if tr_count > 1000:
1220                if len(no_import) > 0:
1221                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1222                             '\n'.join(no_import) + '\n')
1223                    no_import = []
1224                em = '%d transactions committed\n' % (tr_count)
1225                transaction.commit()
1226                regs = []
1227                logger.info(em)
1228                total += tr_count
1229                tr_count = 0
1230        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1231                                                '\n'.join(no_import))
1232        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1233    ###)
1234
1235    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1236    def mass_create_faculty(self,mapping):
1237        "create a faculty"
1238        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1239        academics_folder = self.portal_url.getPortalObject().campus.academics
1240        fid = mapping['code']
1241        if getattr(academics_folder,fid,None) is not None:
1242            return '', "Faculty with ID: %s exists" % fid
1243        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1244        try:
1245            academics_folder.invokeFactory('Faculty', fid)
1246        except BadRequest,E:
1247            return '', "%s" % E
1248        f = getattr(academics_folder,fid,None)
1249        f.getContent().edit(mapping=mapping)
1250        return fid,''
1251    ###)
1252
1253    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1254    def mass_edit_faculty(self,mapping):
1255        "edit a faculty"
1256        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1257        academics_folder = self.portal_url.getPortalObject().campus.academics
1258        fid = mapping['code']
1259        f = getattr(academics_folder,fid,None)
1260        if f is None:
1261            return '', "Faculty with ID: %s does not exist" % fid
1262        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1263        f.getContent().edit(mapping=mapping)
1264        return fid,''
1265    ###)
1266
1267    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1268    def mass_create_department(self,mapping):
1269        "create a department in the correct faculty"
1270        logger = logging.getLogger('WAeUPTool.mass_create_department')
1271        fid = mapping['faculty_code']
1272        if getattr(self,'_v_faculties',None) is None:
1273            res = self.portal_catalog(portal_type = "Faculty")
1274            self._v_faculties = {}
1275            for f in res:
1276                self._v_faculties[f.getId] = f.getObject()
1277        f = self._v_faculties.get(fid,None)
1278        if f is None:
1279            return '', "No Faculty with ID: %s" % fid
1280        else:
1281            did = mapping.get('code')
1282            d = getattr(f,did,None)
1283            if d is None or d.portal_type == "Faculty":
1284                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1285                try:
1286                    f.invokeFactory('Department', did)
1287                except BadRequest,E:
1288                    return '', "%s" % E
1289                d = getattr(f,did)
1290                d.invokeFactory('CoursesFolder','courses')
1291                courses = getattr(d,'courses')
1292                dict = {'Title': 'Courses'}
1293                courses.getContent().edit(mapping=dict)
1294                d.invokeFactory('CertificatesFolder','certificates')
1295                certificates = getattr(d,'certificates')
1296                dict = {'Title': 'Certificates'}
1297                certificates.getContent().edit(mapping=dict)
1298            d.getContent().edit(mapping=mapping)
1299        return did,''
1300    ###)
1301
1302    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1303    def mass_edit_department(self,mapping):
1304        "create a department in the correct faculty"
1305        logger = logging.getLogger('WAeUPTool.mass_create_department')
1306        academics_folder = self.portal_url.getPortalObject().campus.academics
1307        fid = mapping['faculty_code']
1308        did = mapping.get('code')
1309        try:
1310            d = getattr(getattr(academics_folder,fid),did,None)
1311        except KeyError:
1312            return '', "Department %s or Faculty %s wrong" % (did,fid)
1313        else:
1314            if d is None or d.portal_type == "Faculty":
1315                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1316            d.getContent().edit(mapping=mapping)
1317        return did,''
1318    ###)
1319
1320    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1321    def mass_create_course(self,mapping):
1322        #import pdb;pdb.set_trace()
1323        if getattr(self,'_v_course_list',None) is None:
1324            self._v_course_list = []
1325        if getattr(self,'_v_departments',None) is None:
1326            res = self.portal_catalog(portal_type = "Department")
1327            self._v_department_courses = {}
1328            for d in res:
1329                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1330        did = mapping['department_code']
1331        d = self._v_department_courses.get(did,None)
1332        if d is None:
1333            return '', "No Department with ID: %s" % did
1334        course_id = mapping.get('code')
1335        if course_id in self._v_course_list:
1336            return '', "Duplicate Course ID: %s" % did
1337        c = getattr(d,course_id,None)
1338        if c is not None:
1339            return '', "Duplicate Course ID: %s" % did
1340        try:
1341            d.invokeFactory('Course', course_id)
1342        except BadRequest,E:
1343            return '', "%s" % E
1344        self._v_course_list.append(course_id)
1345        c = getattr(d,course_id)
1346        c.getContent().edit(mapping=mapping)
1347        return course_id,''
1348    ###)
1349
1350    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1351    def mass_edit_course(self,mapping):
1352        #import pdb;pdb.set_trace()
1353        course_id = mapping.get('code')
1354        res = self.portal_catalog(id=course_id)
1355        if not res:
1356            return '', "No Course with ID: %s" % course_id
1357        c = res[0].getObject()
1358        c.getContent().edit(mapping=mapping)
1359        return course_id,''
1360    ###)
1361
1362    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1363    def mass_create_certificate(self,mapping):
1364        if getattr(self,'_v_certificate_list',None) is None:
1365            self._v_certificate_list = []
1366        if getattr(self,'_v_department_certificates',None) is None:
1367            res = self.portal_catalog(portal_type = "Department")
1368            self._v_department_certificates = {}
1369            for d in res:
1370                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1371        did = mapping['department_code']
1372        d = self._v_department_certificates.get(did,None)
1373        if d is None:
1374            return '', "No Department with ID: %s" % did
1375        certificate_id = mapping.get('code')
1376        if certificate_id in self._v_certificate_list:
1377            return '', "Duplicate Certificate ID: %s" % did
1378        c = getattr(d,certificate_id,None)
1379        if c is not None:
1380            return '', "Duplicate Certificate ID: %s" % did
1381        try:
1382            d.invokeFactory('Certificate', certificate_id)
1383        except BadRequest,E:
1384            return '', "%s" % E
1385        self._v_certificate_list.append(certificate_id)
1386        c = getattr(d,certificate_id)
1387        c.getContent().edit(mapping=mapping)
1388        return certificate_id,''
1389    ###)
1390
1391    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1392    def mass_edit_certificate(self,mapping):
1393        #import pdb;pdb.set_trace()
1394        certificate_id = mapping.get('code')
1395        res = self.portal_catalog(id=certificate_id)
1396        if not res:
1397            return '', "No Certificate with ID: %s" % did
1398        c = res[0].getObject()
1399        c.getContent().edit(mapping=mapping)
1400        return certificate_id,''
1401    ###)
1402
1403    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1404    def mass_create_application(self,mapping):
1405        #import pdb;pdb.set_trace()
1406        reg_no = mapping.get('reg_no')
1407        try:
1408            self.applicants_catalog.addRecord(**mapping)
1409        except ValueError:
1410            return '', "applicant record with reg_no %s already exists" % reg_no
1411        return reg_no,''
1412    ###)
1413
1414    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1415    def mass_edit_application(self,mapping):
1416        #import pdb;pdb.set_trace()
1417        reg_no = mapping.get('reg_no')
1418        status = mapping.get('status')
1419        res = self.applicants_catalog(reg_no = reg_no)
1420        if len(res):
1421            if res[0].status == 'created' and status != 'created':
1422                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1423            elif status == 'created' and res[0].status != 'created':
1424                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1425            else:
1426                self.applicants_catalog.modifyRecord(**mapping)
1427                return reg_no,''
1428        else:
1429            return '', "applicant record with reg_no %s does not exist" % reg_no
1430
1431    ###)
1432
1433    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1434    def mass_create_course_result(self,mapping):
1435        #import pdb;pdb.set_trace()
1436        students_folder = self.portal_url.getPortalObject().campus.students
1437        if getattr(self,'_v_courses',None) is None:
1438            res = self.courses_catalog()
1439            self._v_courses = {}
1440            for brain in res:
1441                self._v_courses[brain.code] = brain
1442        if getattr(self,'_v_level_created',None) is None:
1443            self._v_level_created = []
1444        course_id = mapping.get('code')
1445        if course_id not in self._v_courses.keys():
1446            return '', "No course with ID: %s" % did
1447        id_key = ''
1448        for id_key in ('student_id','matric_no'):
1449            id_field = mapping.get(id_key,None)
1450            if id_field is not None:
1451                student_id = id_field
1452                break
1453        query = Eq(id_key,id_field)
1454        res = self.students_catalog.evalAdvancedQuery(query)
1455        if not res:
1456            return '', "no student with %(id_key)s %(id_field)s" % vars()
1457        if id_field != "student_id":
1458            mapping['student_id'] = res[0].id
1459        if mapping['student_id'] not in self._v_level_created:
1460            try:
1461                context = getattr(getattr(students_folder,
1462                                          "%(student_id)s" % mapping),
1463                                  'study_course')
1464            except:
1465                return '', "could not create level %(level_id)s for %(student_id)s" % mapping
1466            if mapping['level_id'] not in context.objectIds():
1467                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % mapping)
1468                level = getattr(context,"%(level_id)s" % mapping)
1469                self.portal_workflow.doActionFor(level,'open')
1470                level.getContent().edit(mapping={'session': "%(session_id)s" % mapping})
1471                self.portal_workflow.doActionFor(level,'close_for_edit')
1472            self._v_level_created += mapping['student_id'],
1473        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping
1474        for k in ('semester','credits',):
1475            mapping[k] = getattr(self._v_courses[course_id],k)
1476        try:
1477            self.course_results.addRecord(**mapping)
1478        except ValueError:
1479            return '', "course result already exists: %s" % key
1480        return key,''
1481    ###)
1482
1483    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1484    def mass_edit_course_result(self,mapping):
1485        #import pdb;pdb.set_trace()
1486        # if getattr(self,'_v_courses',None) is None:
1487        #     res = self.courses_catalog()
1488        #     self._v_courses = {}
1489        #     for brain in res:
1490        #         self._v_courses[brain.code] = brain
1491        # course_id = mapping.get('code')
1492        # if course_id not in self._v_courses.keys():
1493        #     return '', "No course with ID: %s" % did
1494        id_key = ''
1495        for id_key in ('student_id','matric_no'):
1496            id_field = mapping.get(id_key,None)
1497            if id_field is not None:
1498                student_id = id_field
1499                break
1500        query = Eq(id_key,id_field)
1501        res = self.students_catalog.evalAdvancedQuery(query)
1502        if not res:
1503            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1504        if id_field != "student_id":
1505            mapping['student_id'] = res[0].id
1506        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % mapping
1507        # for k in ('semester',):
1508        #     mapping[k] = getattr(self._v_courses[course_id],k)
1509        try:
1510            self.course_results.modifyRecord(**mapping)
1511        except KeyError:
1512            return '', "No course result to edit: %s" % key
1513        return key,''
1514    ###)
1515
1516    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1517    def mass_create_certificate_course(self,mapping):
1518        if getattr(self,'_v_courses',None) is None:
1519            res = self.courses_catalog()
1520            self._v_courses= [course.code for course in res]
1521        if getattr(self,'_v_ceritficates',None) is None:
1522            res = self.portal_catalog(portal_type = "Certificate")
1523            self._v_certificates = {}
1524            for cert in res:
1525                self._v_certificates[cert.getId] = cert.getObject()
1526        certificate_course_id = mapping.get('code')
1527        if certificate_course_id not in self._v_courses:
1528            return '', "No Course with ID: %s" % certificate_course_id
1529        cert_id = mapping['certificate_code']
1530        cert = self._v_certificates.get(cert_id,None)
1531        if cert is None:
1532            return '', "No Certificate with ID: %s" % cert_id
1533        level_id = mapping.get('level')
1534        level = getattr(cert,level_id,None)
1535        if level is None:
1536            cert.invokeFactory('StudyLevel', level_id)
1537            level = getattr(cert,level_id,None)
1538        elif hasattr(level,certificate_course_id):
1539            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1540            (certificate_course_id,cert_id,level_id)
1541        level.invokeFactory('CertificateCourse', certificate_course_id)
1542        c = getattr(level,certificate_course_id)
1543        c.getContent().edit(mapping=mapping)
1544        return certificate_course_id,''
1545    ###)
1546
1547    field2types_student = {   ###(
1548                      'StudentApplication':
1549                          {'id': 'application',
1550                           'title': 'Application Data',
1551                           'wf_transition_return': 'close',
1552                           'wf_transition_admit': 'remain',
1553                           'fields':
1554                             ('jamb_reg_no',
1555                              'entry_mode',
1556                              'entry_session',
1557                              'jamb_score',
1558                              'app_email',
1559                              'jamb_age',
1560                              'jamb_state',
1561                              'jamb_lga',
1562                              )
1563                              },
1564                      #'StudentPume':
1565                      #    {'id': 'pume',
1566                      #     'title': 'Pume Data',
1567                      #     'wf_transition_return': 'close',
1568                      #     'wf_transition_admit': 'close',
1569                      #     'fields':
1570                      #       ('pume_score',
1571                      #        )
1572                      #        },
1573                      'StudentClearance':
1574                          {'id': 'clearance',
1575                           'title': 'Clearance/Eligibility Record',
1576                           'wf_transition_return': 'close',
1577                           'wf_transition_admit': 'remain',
1578                           'fields':
1579                             ('matric_no',
1580                              'nationality',
1581                              'lga',
1582                              'birthday',
1583                              )
1584                              },
1585                         'StudentPersonal':
1586                          {'id': 'personal',
1587                           'title': 'Personal Data',
1588                           'wf_transition_return': 'open',
1589                           'wf_transition_admit': 'remain',
1590                           'fields':
1591                             ('firstname',
1592                              'middlename',
1593                              'lastname',
1594                              'sex',
1595                              'email',
1596                              'phone',
1597                              'perm_address',
1598                              )
1599                              },
1600                         'StudentStudyCourse':
1601                          {'id': 'study_course',
1602                           'title': 'Study Course',
1603                           'wf_transition_return': 'open',
1604                           'wf_transition_admit': 'remain',
1605                           'fields':
1606                             ('study_course',
1607                              'current_level',
1608                              'current_session',
1609                              'current_mode',
1610                              'current_verdict',
1611                              'previous_verdict',
1612                              )
1613                              },
1614
1615                         'PaymentsFolder':
1616                          {'id': 'payments',
1617                           'title': 'Payments',
1618                           'wf_transition_return': 'open',
1619                           'wf_transition_admit': 'open',
1620                           'fields':
1621                             ()
1622                              },
1623                         }
1624    ###)
1625
1626    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1627    def mass_create_student(self,mapping):
1628        "create a students record due import"
1629        logger = logging.getLogger('WAeUPTool.mass_create_student')
1630        students_folder = self.portal_url.getPortalObject().campus.students
1631        jamb_reg_no = mapping.get('jamb_reg_no',None)
1632        if jamb_reg_no:
1633            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1634            if res:
1635                return '',"jamb_reg_no exists"
1636        matric_no = mapping.get('matric_no',None)
1637        if matric_no:
1638            res = self.students_catalog(matric_no = matric_no)
1639            if res:
1640                return '',"matric_no exists"
1641        sid = self.waeup_tool.generateStudentId('?')
1642        students_folder.invokeFactory('Student', sid)
1643        student_obj = getattr(students_folder,sid)
1644        f2t = self.field2types_student
1645        d = {}
1646        transition = mapping.get('reg_transition','admit')
1647        if transition not in ('admit','return'):
1648            return '',"no valid transition provided"
1649        for pt in f2t.keys():
1650            student_obj.invokeFactory(pt,f2t[pt]['id'])
1651            sub_obj = getattr(student_obj,f2t[pt]['id'])
1652            sub_doc = sub_obj.getContent()
1653            d['Title'] = f2t[pt]['title']
1654            for field in f2t[pt]['fields']:
1655                d[field] = mapping.get(field,'')
1656
1657            if pt == "StudentApplication":
1658                d['jamb_sex']  = 'M'
1659                if mapping.get('sex'):
1660                    d['jamb_sex']  = 'F'
1661                d['jamb_firstname'] = mapping.get('firstname',None)
1662                d['jamb_middlename'] = mapping.get('middlename',None)
1663                d['jamb_lastname'] = mapping.get('lastname',None)
1664
1665            # if pt == "StudyCourse":
1666            #     for von,zu in (('entry_mode','current_mode'),
1667            #                    ('entry_session','current_session')):
1668            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1669            #             d[zu] = mapping[von]
1670            sub_doc.edit(mapping = d)
1671
1672            #import pdb;pdb.set_trace()
1673            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1674            if new_state != "remain":
1675                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1676        self.portal_workflow.doActionFor(student_obj,transition)
1677        student_obj.manage_setLocalRoles(sid, ['Owner',])
1678        return sid,''
1679    ###)
1680
1681    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1682    def mass_edit_student(self,mapping):
1683        "edit a students record due import"
1684        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1685        students_folder = self.portal_url.getPortalObject().campus.students
1686        sid = mapping.get('id',None)
1687        jamb_reg_no = mapping.get('jamb_reg_no',None)
1688        matric_no = mapping.get('matric_no',None)
1689        editable_keys = mapping.keys()
1690        if sid:
1691            res = self.students_catalog(id = sid)
1692            if not res:
1693                return '',"no student with id %s" % sid
1694            if matric_no and res[0].matric_no and\
1695              matric_no != res[0].matric_no:
1696                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1697            if jamb_reg_no and res[0].jamb_reg_no and\
1698              jamb_reg_no != res[0].jamb_reg_no:
1699                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1700        elif jamb_reg_no:
1701            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1702            if not res:
1703                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1704            editable_keys.remove('jamb_reg_no')
1705        elif matric_no:
1706            res = self.students_catalog(matric_no = matric_no)
1707            if not res:
1708                return '',"no student with matric_no %s" % matric_no
1709            editable_keys.remove('matric_no')
1710
1711        ## included only to change wf state from admitted to returning
1712        #if res[0].review_state not in ('admitted','objection_raised'):
1713        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1714        ## end inclusion
1715
1716        sid = res[0].id
1717        student_obj = getattr(students_folder,sid)
1718        f2t = self.field2types_student
1719        d = {}
1720        #import pdb;pdb.set_trace()
1721        any_change = False
1722        for pt in f2t.keys():
1723            if pt == "student_application":
1724                d['jamb_sex']  = 'M'
1725                if mapping.get('sex'):
1726                    d['jamb_sex']  = 'F'
1727            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1728            if intersect:
1729                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1730                if sub_obj is None:
1731                    try:
1732                        student_obj.invokeFactory(pt,f2t[pt]['id'])
1733                    except:
1734                        continue
1735                    sub_obj = getattr(student_obj,f2t[pt]['id'])
1736                    d['Title'] = f2t[pt]['title']
1737                sub_doc = sub_obj.getContent()
1738                for field in intersect:
1739                    changed = False
1740                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1741                        any_change = True
1742                        changed = True
1743                        d[field] = mapping.get(field,'')
1744                    if changed:
1745                        sub_doc.edit(mapping = d)
1746
1747
1748        ## included only to change wf state from admitted to returning
1749        #    if res[0].review_state in ('admitted','objection_raised'):
1750        #        new_state = f2t[pt]['wf_transition_return']
1751        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1752        #        if sub_obj and new_state != "remain":
1753        #            try:
1754        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1755        #            except:
1756        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1757        #                pass
1758        #if res[0].review_state in ('admitted','objection_raised'):
1759        #    wfaction = 'return'
1760        #    try:
1761        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1762        #        logger.info('%s, wf state changed' % sid)
1763        #        any_change = True
1764        #    except:
1765        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1766        #        pass
1767        ## end inclusion
1768
1769
1770        if any_change:
1771            return sid,''
1772        else:
1773            return sid,'not modified'
1774    ###)
1775
1776    security.declareProtected(ModifyPortalContent,"importData")###(
1777    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1778        """load data from CSV values"""
1779        import transaction
1780        import random
1781
1782        pm = self.portal_membership
1783        member = pm.getAuthenticatedMember()
1784
1785        logger = logging.getLogger('WAeUPTool.importData')
1786        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1787        students_folder = self.portal_url.getPortalObject().campus.students
1788        start = True
1789        tr_count = 0
1790        total_imported = 0
1791        total_not_imported = 0
1792        total = 0
1793        iname = "import_%s" % name
1794        if iname == 'import_application':
1795            commit_after = 2000
1796        else:
1797            commit_after = 100
1798        stool = getToolByName(self, 'portal_schemas')
1799        ltool = getToolByName(self, 'portal_layouts')
1800        schema = stool._getOb(iname)
1801        if schema is None:
1802            em = 'No such schema %s' % iname
1803            logger.error('No such schema %s' % iname)
1804            return em
1805        layout = ltool._getOb(iname)
1806        if layout is None:
1807            em = 'No such layout %s' % iname
1808            logger.error(em)
1809            return em
1810        validators = {}
1811        for widget in layout.keys():
1812            validators[widget] = layout[widget].validate
1813        mode = "create"
1814        if edit:
1815            mode = "edit"
1816        importer_name = "mass_%(mode)s_%(name)s" % vars()
1817        importer = getattr(self, '%s' % importer_name,None)
1818        if importer is None:
1819            em = 'No importer function %s' % importer_name
1820            logger.error(em)
1821            return em
1822        not_imported = []
1823        imported = []
1824        try:
1825            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1826        except:
1827            em = 'Error reading %s.csv' % filename
1828            logger.error(em)
1829            return em
1830        for item in items:
1831            #import pdb;pdb.set_trace()
1832            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1833            if start:
1834                start = False
1835                adapters = [MappingStorageAdapter(schema, item)]
1836                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1837                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1838                import_keys = [k.strip() for k in attrs if not k.startswith('ignore')]
1839                diff2schema = set(import_keys).difference(set(schema.keys()))
1840                diff2layout = set(import_keys).difference(set(layout.keys()))
1841                if diff2schema:
1842                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1843                    return em
1844                if mode == "create":
1845                    required_keys = [layout.getIdUnprefixed(id)
1846                                     for id,widget in layout.objectItems()
1847                                     if widget.is_required]
1848                    if not set(required_keys).issubset(set(import_keys)):
1849                        diff2import = set(required_keys).difference(set(import_keys))
1850                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1851                        return em
1852                s = ','.join(['"%s"' % fn for fn in import_keys])
1853                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1854                s = '"id",' + s
1855                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1856                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1857                format_error = format + ',"%(Error)s"'
1858                format = '"%(id)s",'+ format
1859
1860            dm = DataModel(item, adapters,context=self)
1861            ds = DataStructure(data=item,datamodel=dm)
1862            error_string = ""
1863            total += 1
1864            error_count = 0
1865            for k in import_keys:
1866                if not validators[k](ds,mode=mode):
1867                    if error_count:
1868                        error_string += ' ++ '
1869                    error_string += "%s: %s" % (k,
1870                                                  self.translation_service(ds.getError(k),
1871                                                                           ds.getErrorMapping(k)))
1872                    error_count += 1
1873            if not error_string:
1874                temp_item = item.copy()
1875                temp_item.update(dm)
1876                temp_item['id'],error = importer(temp_item)
1877                if error:
1878                    error_string += error
1879                else:
1880                    item = temp_item
1881            if error_string:
1882                item['Error'] = error_string
1883                not_imported.append(format_error % item)
1884                total_not_imported += 1
1885            else:
1886                em = format % item
1887                imported.append(em)
1888                tr_count += 1
1889                total_imported += 1
1890                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1891
1892            if total and not total % commit_after:
1893                transaction.commit()
1894                if len(not_imported) > 0:
1895                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1896                             '\n'.join(not_imported) + '\n')
1897                    not_imported = []
1898                if len(imported) > 0:
1899                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1900                             '\n'.join(imported) + '\n')
1901                    imported = []
1902                em = '%d transactions committed\n' % (tr_count)
1903                regs = []
1904                logger.info(em)
1905                tr_count = 0
1906                #if total > 100:
1907                #    return
1908        if len(imported) > 0:
1909            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1910                                                '\n'.join(imported))
1911        if len(not_imported) > 0:
1912            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1913                                                '\n'.join(not_imported))
1914        em = "Finished: %d imported, %d not imported (of total %d)" % (total_imported,total_not_imported,total)
1915        logger.info(em)
1916        return em
1917    ###)
1918
1919    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1920    def moveImagesToFS(self,student_id="O738726"):
1921        "move the images to the filesystem"
1922        images_dir = getImagesDir(student_id)
1923        #images_dir = os.path.join("%s" % images_base,student_id)
1924        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1925        stool = getToolByName(self, 'portal_schemas')
1926        schemas = ['student_application',
1927                   'student_clearance',
1928                   ]
1929        created = False
1930        for schema_id in schemas:
1931            schema = stool._getOb(schema_id)
1932            object = getattr(student_folder,schema_id[len('student_'):],None)
1933            if object is None:
1934                continue
1935            doc = object.getContent()
1936            for key in schema.keys():
1937                if schema[key].meta_type != "CPS Image Field":
1938                    continue
1939                #import pdb;pdb.set_trace()
1940                image = getattr(doc,key,None)
1941                if not image or not hasattr(image,"data"):
1942                    continue
1943                if not created:
1944                    if not os.path.exists(images_dir):
1945                        os.mkdir(images_dir)
1946                    created = True
1947                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1948                open(filename,"wb").write(str(image.data))
1949    ###)
1950
1951    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1952    def movePassportToFS(self,student_id="O738726"):
1953        "move the passports to the filesystem"
1954        images_dir = os.path.join("%s" % i_home,'passports')
1955        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1956        stool = getToolByName(self, 'portal_schemas')
1957        schemas = ['student_application',
1958                   #'student_clearance',
1959                   ]
1960        created = False
1961        for schema_id in schemas:
1962            schema = stool._getOb(schema_id)
1963            object = getattr(student_folder,schema_id[len('student_'):],None)
1964            if object is None:
1965                continue
1966            doc = object.getContent()
1967            for key in schema.keys():
1968                if schema[key].meta_type != "CPS Image Field":
1969                    continue
1970                #import pdb;pdb.set_trace()
1971                image = getattr(doc,key)
1972                if not hasattr(image,"data"):
1973                    continue
1974                if not created:
1975                    if not os.path.exists(images_dir):
1976                        os.mkdir(images_dir)
1977                    created = True
1978                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1979                open(filename,"wb").write(str(image.data))
1980    ###)
1981
1982InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.