source: WAeUP_SRP/base/WAeUPTool.py @ 3000

Last change on this file since 3000 was 2999, checked in by joachim, 17 years ago

for last checkin
  • Property svn:keywords set to Id
File size: 87.3 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2999 2008-01-08 19:10:04Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64
65class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66    """WAeUP tool"""
67
68    id = 'waeup_tool'
69    meta_type = 'WAeUP Tool'
70    _actions = ()
71    security = ClassSecurityInfo()
72    security.declareObjectProtected(View)
73    manage_options = ( ActionProviderBase.manage_options
74                     + SimpleItem.manage_options
75                     )
76
77    security.declareProtected(View,'re_split') ###(
78    def re_split(self,split_string,string):
79        return re.split(split_string,string)
80
81
82    security.declareProtected(View,'difference') ###(
83    def difference(self,l1,l2):
84        return set(l1).difference(set(l2))
85
86    def rwrite(self,s): ###(
87        response = self.REQUEST.RESPONSE
88        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
89        response.write("%s<br />\r\n" % s)
90    ###)
91
92    def addtodict(self,d,key,item): ###(
93        d[key].append(item)
94        return d[key]
95    ###)
96
97    def sleep(self,secs): ###(
98        "sleep"
99        import time
100        time.sleep(secs)
101        return
102
103###)
104
105    security.declareProtected(View,'updateRoleMappingsFor') ###(
106    def updateRoleMappingsFor(self,wf_definition,ob):
107        "do so for public"
108        wf_def = getattr(self.portal_workflow,wf_definition)
109        wf_def.updateRoleMappingsFor(ob)
110    ###)
111
112    security.declareProtected(View,'getStatesLgas') ###(
113    def getStatesLgas(self):
114        """return lga info"""
115        voc = getattr(self.portal_vocabularies,'local_gov_areas')
116        states = []
117        lgas  = []
118        d = {}
119        wd = {}
120        for k,v in voc.items():
121            parts = v.split(' / ')
122            if len(parts) == 1:
123                state = parts[0].lower()
124                lga = ""
125            elif len(parts) == 2:
126                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
127                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
128            else:
129                continue
130            if state not in states:
131                states.append(state)
132            if lga not in lgas:
133                lgas.append(lga)
134            words = re.split('[^a-zA-Z0-9/]',k)
135            words.sort()
136            wd[k] = words
137            d[k] = v
138        mapping = {}
139        mapping['word_dict'] = wd
140        mapping['lga_dict'] = d
141        mapping['states'] = states
142        mapping['lgas'] = lgas
143        return mapping
144    ###)
145
146    security.declareProtected(View,'getAccessInfo') ###(
147    def getAccessInfo(self,context):
148        "return a dict with access_info"
149        logger = logging.getLogger('WAeUPTool.getAccessInfo')
150        mtool = self.portal_membership
151        member = mtool.getAuthenticatedMember()
152        member_id = str(member)
153        info = {}
154        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
155        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
156        is_staff = info['is_staff'] = not is_anonymous and not is_student
157        roles = member.getRolesInContext(context)
158        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
159                                                                            "SectionManager" in roles or
160                                                                            "Manager" in roles)
161        is_allowed = info['is_allowed'] = not is_anonymous
162        requested_id = context.getStudentId()
163        student_id  = None
164        if not is_student and requested_id:
165            student_id  = requested_id
166        elif is_anonymous or member_id != requested_id:
167            logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
168        else:
169            student_id = member_id
170        info['student_id'] = student_id
171        return info
172    ###)
173
174    security.declareProtected(ModifyPortalContent,'openLog') ###(
175    def openLog(self,name):
176        """open a log file"""
177        version = 1
178        path = "%s/log/%s_%d.log" % (i_home,name,version)
179        while os.path.exists(path):
180            version += 1
181            path = "%s/log/%s_%d.log" % (i_home,name,version)
182        log = open(path,"w")
183        return log
184    ###)
185
186    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
187    def bypassQueueCatalog(self,enable=True):
188        """bypass the QueueCatalog by setting all indexes to process imediate,
189        if enable is True (default) the old settings are restored
190        """
191
192    ###)
193
194    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
195    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
196        """measure Object access Time"""
197        import random
198        if hasattr(self,'portal_catalog_real'):
199            aq_portal = self.portal_catalog_real.evalAdvancedQuery
200        else:
201            aq_portal = self.portal_catalog.evalAdvancedQuery
202        nr_pts = int(nr_pts)
203        probe = int(probe)
204        intervall = probe/10
205        objects = ("application","clearance","personal")
206        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
207        #i = random.randrange(num_objects)
208        count = 0
209        found = 0
210        not_found = 0
211        t_found = 0
212        t_not_found = 0
213        time_found = time_not_found = 0.0
214        t_time_found = t_time_not_found = 0.0
215        accessed = []
216        t_min = 1000
217        t_max = 0
218        #import pdb;pdb.set_trace()
219        students = self.portal_catalog(portal_type="Student")
220        num_students = len(students)
221        if method == "d":
222            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
223            res = aq_portal(query)
224            brains = {}
225            for r in res:
226                sid = r.relative_path.split('/')[-2]
227                if brains.has_key(sid):
228                    brains[sid][r.portal_type] = r
229                else:
230                    brains[sid] = {r.portal_type : r}
231            brains_list = brains.keys()
232            num_objects = len(brains_list)
233        else:
234            num_objects = num_students
235        print "="*40
236        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
237                                                                        probe,
238                                                                        nr_pts,
239                                                                        num_objects)
240        print "nr found/not time found/not min/max"
241        elapse = time.time()
242        i_elapse = time.time()
243        c_elapse = time.clock()
244        for c in range(1,probe + 1):
245            i = random.randrange(num_objects)
246            if method in ('a','b','c'):
247                student_brain = students[i]
248            elif method == "d":
249                #import pdb;pdb.set_trace()
250                student_brain = brains[brains_list[i]]
251            if method == "c":
252                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
253                res = aq_portal(query)
254                this_portal_types = [r.portal_type for r in res]
255            for i in range(nr_pts):
256                oid = objects[i]
257                if method == "a":
258                    try:
259                        student_path = student_brain.getPath()
260                        path = "%s/%s" % (student_path,oid)
261                        doc = self.unrestrictedTraverse(path).getContent()
262                        found += 1
263                        i_time = time.time() - i_elapse
264                        time_found += i_time
265                    except:
266                        not_found += 1
267                        i_time = time.time() - i_elapse
268                        time_not_found += i_time
269                        pass
270                elif method == "b":
271                    try:
272                        student_object = student_brain.getObject()
273                        doc = getattr(student_object,oid).getContent()
274                        found += 1
275                        i_time = time.time() - i_elapse
276                        time_found += i_time
277                    except:
278                        i_time = time.time() - i_elapse
279                        time_not_found += i_time
280                        not_found += 1
281                        pass
282                elif method == "c":
283                    if portal_types[i] in this_portal_types:
284                        found += 1
285                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
286                        i_time = time.time() - i_elapse
287                        time_found += i_time
288                    else:
289                        not_found += 1
290                        i_time = time.time() - i_elapse
291                        time_not_found += i_time
292                elif method == "d":
293                    if student_brain.has_key(portal_types[i]):
294                        found += 1
295                        doc = student_brain[portal_types[i]].getObject().getContent()
296                        i_time = time.time() - i_elapse
297                        time_found += i_time
298                    else:
299                        not_found += 1
300                        i_time = time.time() - i_elapse
301                        time_not_found += i_time
302                i_elapse = time.time()
303            if c and (c % intervall == 0):
304                #i_time = time.time() - i_elapse
305                t_per = 0.0
306                if found:
307                    t_per = time_found/found
308                if t_per > t_max:
309                    t_max = t_per
310                if t_per > 0.0 and t_per < t_min:
311                    t_min = t_per
312                itf = 0.0
313                if found:
314                    itf = time_found/found
315                itnf = 0.0
316                if not_found :
317                    itnf = time_not_found / not_found
318                interval_time = time_found + time_not_found
319                s = "%(c)d: %(found)d/%(not_found)d " % vars()
320                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
321                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
322                print s
323                t_found += found
324                t_not_found += not_found
325                t_time_found += time_found
326                t_time_not_found += time_not_found
327                time_found = time_not_found = 0.0
328                found = not_found = 0
329        # t_found += found
330        # t_not_found += not_found
331        elapse = time.time() - elapse
332        itf = 0.0
333        if t_found:
334            itf = t_time_found/t_found
335        itnf = 0.0
336        if t_not_found:
337            itnf = t_time_not_found / t_not_found
338        #c_elapse = time.clock() - c_elapse
339        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
340        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
341        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
342        print "-"*40
343        print s
344        rel_found = float(t_found)/probe
345        rel_not_found = float(t_not_found)/probe
346        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
347        print estimated_total_time
348    ###)
349
350    security.declareProtected(ModifyPortalContent,'writeLog') ###(
351    def writeLog(self,logfile,s):
352        """write to the log file"""
353        logfile.write(s)
354
355###)
356
357    def generateStudentId(self,letter): ###(
358        import random
359        r = random
360        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
361        if letter == '?':
362            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
363        sid = "%c%d" % (letter,r.randint(99999,1000000))
364        students = self.portal_url.getPortalObject().campus.students
365##        while hasattr(students, sid):
366##            sid = "%c%d" % (letter,r.randint(99999,1000000))
367        while self.students_catalog(id = sid):
368            sid = "%c%d" % (letter,r.randint(99999,1000000))
369        return sid
370    ###)
371
372    def generatePassword(self,s=None): ###(
373        import random
374        r = random
375        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
376        if s is None:
377            s = 'abcdefghklmnpqrstuvwxy23456789'
378        pw = ''
379        while len(pw) < 6:
380            pw += r.choice(s)
381        return pw
382    ###)
383
384    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
385    def dumpSchoolfeePayments(self):
386        "dump paid schoolfees"
387        mtool = self.portal_membership
388        member = mtool.getAuthenticatedMember()
389        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
390        aq_student = self.students_catalog.evalAdvancedQuery
391        query = In('review_state',('schoolfee_paid',
392                                   'courses_registered',
393                                   'courses_validated',
394                                   ))
395        res = aq_student(query)
396        #import pdb;pdb.set_trace()
397        l = []
398        logger.info("start for %d" % len(res))
399        count = 1
400        log_after = 100
401        for student in res:
402            if not count % log_after:
403                logger.info("processed %d total %d" % (log_after,count))
404            count += 1
405            fee_dict =self.getSchoolFee(student)
406            fulltime = student.mode.endswith('_ft')
407            d = {}
408            d['student_id'] = student.id
409            d['name'] = student.name
410            d['amount'] = fee_dict.get(new_returning)
411            l += d,
412        csv_name = self.dumpListToCSV(l,'payments')
413        logger.info('%s dumped payments to %s' % (member,export_file))
414    ###)
415
416    security.declarePublic('dumpListToCSV') ###(
417    def dumpListToCSV(self,l,filename,fields=None):
418        """dump a list of dicts to a CSV file"""
419        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
420        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
421        if fields is None:
422            fields = l[0].keys()
423        headline = ','.join(fields)
424        out = open(export_file,"wb")
425        out.write(headline +'\n')
426        out.close()
427        out = open(export_file,"a")
428        csv_writer = csv.DictWriter(out,fields,)
429        csv_writer.writerows(l)
430        return export_file
431    ###)
432
433    security.declareProtected(ManagePortal, 'listMembers') ###(
434    def listMembers(self):
435        "list all members"
436        mtool = self.portal_membership
437        member = mtool.getAuthenticatedMember()
438        logger = logging.getLogger('WAeUPTool.listMembers')
439        if str(member) not in ('admin','joachim'):
440            logger.info('%s tried to list members' % (member))
441            return None
442        members = self.portal_directories.members
443        all = members.listEntryIdsAndTitles()
444        l = []
445        for user_id,name in all:
446            d = {}
447            d['user_id'] = user_id
448            d['name'] = name
449            d['pw'] = getattr(getattr(members,user_id),'password')
450            d['email'] = getattr(getattr(members,user_id),'email')
451            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
452            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
453            l += d,
454        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
455        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
456        logger.info('%s dumped memberlist to %s' % (member,export_file))
457        fields = l[0].keys()
458        headline = ','.join(fields)
459        out = open(export_file,"wb")
460        out.write(headline +'\n')
461        out.close()
462        out = open(export_file,"a")
463        csv_writer = csv.DictWriter(out,fields,)
464        csv_writer.writerows(l)
465    ###)
466
467    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
468    def removeDeletedDocIds(self, max=1000):
469        """
470        remove deleted docids from repository commit after max
471        """
472        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
473        repository = getToolByName(self, 'portal_repository')
474        pxtool = getToolByName(self, 'portal_proxies')
475        logger.info('start')
476        pxtool_infos = pxtool.getRevisionsUsed()
477        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
478
479        nb_revs = 0
480        docids_d = {} # all docids
481        unused_docids_d = {} # all docids that are unused
482        ids_unused_revs_docids = [] # ids for revs of unused docids
483        unused_ids = [] # ids for unused revs
484        total = 0
485        idlist = repository.objectIds()
486        to_delete = 0
487        found = False
488        for id in idlist:
489            docid, rev = repository._splitId(id)
490            if docid is None:
491                logger.info("invalid doc_id %s" % docid)
492                continue
493            nb_revs += 1
494            if not pxtool_infos.has_key(docid):
495                found = True
496                to_delete += 1
497                unused_ids.append(id)
498            elif not pxtool_infos[docid].has_key(rev):
499                found = True
500                to_delete += 1
501                unused_ids.append(id)
502            if found and not to_delete % max:
503                found = False
504                #import pdb;pdb.set_trace()
505                repository.manage_delObjects(unused_ids)
506                transaction.commit()
507                logger.info('removed %d total %d unused docids ' % (max,to_delete))
508        else:
509            if unused_ids:
510                repository.manage_delObjects(unused_ids)
511                transaction.commit()
512        logger.info('finished removing %d unused docids ' % (to_delete))
513
514
515###)
516
517    security.declareProtected(View,'getCredential') ###(
518    def getCredential(self,student_id):
519        student_entry = getattr(self.portal_directories.students,student_id,None)
520        if not self.isStaff():
521            mtool = self.portal_membership
522            member = mtool.getAuthenticatedMember()
523            logger = logging.getLogger('WAeUPTool.getCredential')
524            logger.info('%s tried to access password of %s' % (member,student_id))
525            return None
526        if student_entry is None:
527            return None
528        return getattr(student_entry,"password","not set")
529    ###)
530
531    security.declarePublic('checkPassword') ###(
532    def checkPassword(self,student_id,password):
533        student_entry = getattr(self.portal_directories.students,student_id,None)
534        if student_entry is None:
535            return False
536        return getattr(student_entry,"password","not set") == password
537    ###)
538
539    security.declarePublic('checkGenericPassword') ###(
540    def checkGenericPassword(self,member_id):
541        member_entry = getattr(self.portal_directories.members,member_id,None)
542        if member_entry is None:
543            return False
544        ltool = getToolByName(self, 'portal_layouts')
545        unsecure_words = ltool._getOb('members')['w__password'].check_words
546        password = getattr(member_entry,"password","not set")
547        is_unsecure = password in unsecure_words
548        if is_unsecure:
549            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
550            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
551        return is_unsecure
552    ###)
553
554    security.declareProtected(ModifyPortalContent,'editPassword') ###(
555    def editPassword(self,student_id,password):
556        "edit a student password"
557        student_entry = getattr(self.portal_directories.students,student_id,None)
558        if student_entry is None:
559            return
560        setattr(student_entry,'password',password)
561    ###)
562
563    security.declareProtected(ModifyPortalContent,'doCommit') ###(
564    def doCommit(self,logger=None):
565        "commit some transactions"
566        transaction.commit()
567    ###)
568
569    security.declarePublic('loadStudentFoto') ###(
570    def loadStudentFoto(self,student,filename,folder):
571        "return a student passport picture"
572        #import pdb;pdb.set_trace()
573        picture ="%s/import/%s/%s" % (i_home,folder,filename)
574        student_id = student.getId()
575        images_dir = getImagesDir(student_id)
576        if not os.path.exists(images_dir):
577            os.mkdir(images_dir)
578        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
579        for extension in ('.jpg','.JPG'):
580            fullname = "%(picture)s%(extension)s" % vars()
581            if os.path.exists(fullname):
582                copy2(fullname,image_name)
583                return "successfully copied passport picture"
584        return "passport picture not found: %s.jpg or .JPG" % picture
585    ###)
586
587    def old____loadStudentFoto(self,student,filename,folder): ###(
588        "return a student passport picture"
589        app = student.application
590        app_doc = app.getContent()
591        #clear = student.clearance
592        #clear_doc = clear.getContent()
593        #matric_no = clear_doc.matric_no.upper()
594        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
595        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
596        #import pdb;pdb.set_trace()
597        if os.path.exists(picture1):
598            file = open(picture1)
599        elif os.path.exists(picture2):
600            file = open(picture2)
601        else:
602            return "passport picture not found %s" % picture1
603        reopened = False
604        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
605            self.portal_workflow.doActionFor(app,'open')
606            reopened = True
607        outfile = file.read()
608        app_doc.manage_addFile('passport',
609                               file=outfile,
610                               title="%s.jpg" % filename)
611        if reopened:
612            self.portal_workflow.doActionFor(app,'close')
613        return "successfully loaded passport picture"
614    ###)
615
616    security.declareProtected(ModifyPortalContent,'createOne') ###(
617    def createOne(self,students_folder,student_brain,letter,commit=False):
618        sid = self.waeup_tool.generateStudentId(letter)
619        students_folder.invokeFactory('Student', sid)
620        student = getattr(students_folder,sid)
621        self.portal_workflow.doActionFor(student,'return')
622        student.manage_setLocalRoles(sid, ['Owner',])
623        matric_no = student_brain.matric_no
624        jamb_reg_no = student_brain.Entryregno
625        self.students_catalog.addRecord(id = sid,
626                                           matric_no = matric_no,
627                                           jamb_reg_no = jamb_reg_no,
628                                           sex = student_brain.Sex == "F",
629                                           name = "%s %s %s" % (student_brain.Firstname,
630                                                                student_brain.Middlename,
631                                                                student_brain.Lastname)
632                                        )
633        if commit:
634            transaction.commit()
635        return sid,jamb_reg_no
636    ###)
637
638    security.declareProtected(ModifyPortalContent,'addStudent') ###(
639    def addStudent(self,dict):
640        students_folder = self.portal_url.getPortalObject().campus.students
641        sid = self.waeup_tool.generateStudentId('?')
642        students_folder.invokeFactory('Student', sid)
643        student_obj = getattr(students_folder,sid)
644        f2t = self.student_field2types
645        #from pdb import set_trace; set_trace()
646        d = {}
647        #d['jamb_sex']  = 'M'
648        #if dict.get('sex'):
649        #    d['jamb_sex']  = 'F'
650
651        entry_session = dict.get('entry_session')
652        if entry_session == self.getSessionId()[0]:
653            wfaction = 'admit'
654            wft = 'wf_transition_admit'
655            password = None
656        else:
657            wfaction = 'return'
658            wft = 'wf_transition_return'
659            password = self.generatePassword()
660            self.makeStudentMember(sid,password)
661
662        for pt in f2t.keys():
663            student_obj.invokeFactory(pt,f2t[pt]['id'])
664            sub_obj = getattr(student_obj,f2t[pt]['id'])
665            sub_doc = sub_obj.getContent()
666            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
667            d['Title'] = f2t[pt]['title']
668            for field in f2t[pt]['fields']:
669                d[field] = dict.get(field,'')
670            sub_doc.edit(mapping = d)
671            new_state = f2t[pt][wft]
672            if new_state != "remain":
673                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
674        self.portal_workflow.doActionFor(student_obj,wfaction)
675        student_obj.manage_setLocalRoles(sid, ['Owner',])
676        return sid,password
677    ###)
678
679    security.declarePublic('getCertificateBrain') ###(
680    def getCertificateBrain(self,cert_id):
681        "do it"
682        res = ZCatalog.searchResults(self.portal_catalog_real,
683                                {'portal_type':"Certificate",
684                                      'id': cert_id})
685        if res:
686            return res[0]
687        return None
688    ###)
689
690    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
691    def get_csv_filenames(self):
692        "do it"
693        files = [file for file in os.listdir("%s/import/" % (i_home))
694                 if file.endswith('.csv') and file.find('imported') == -1]
695        return files
696    ###)
697
698    security.declarePublic('findStudentByMatricelNo') ###(
699    def findStudentByMatricelNo(self,matric_no):
700        "do it"
701        res = ZCatalog.searchResults(self.portal_catalog_real,
702                                {'portal_type':"StudentClearance",
703                                 'SearchableText': matric_no})
704        if res:
705            return res[0]
706        return None
707    ###)
708
709    security.declarePublic('makeStudentMember') ###(
710    def makeStudentMember(self,sid,password='uNsEt'):
711        """make the student a member"""
712        membership = self.portal_membership
713        membership.addMember(sid,
714                             password ,
715                             roles=('Member',
716                                     'Student',
717                                     ),
718                             domains='',
719                             properties = {'memberareaCreationFlag': False,
720                                           'homeless': True},)
721        member = membership.getMemberById(sid)
722        self.portal_registration.afterAdd(member, sid, password, None)
723        #self.manage_setLocalRoles(sid, ['Owner',])
724    ###)
725
726    security.declareProtected(View,'makeStudentData') ###(
727    def makeStudentData(self,student_id,email=None,phone_nr=None):
728        "create Datastructure for a returning Student"
729        #import pdb;pdb.set_trace()
730        logger = logging.getLogger('WAeUPTool.makeStudentData')
731        students_folder = self.portal_url.getPortalObject().campus.students
732        #res = self.students_catalog(id=student_id)
733        #if res:
734        #    st = res[0]
735        #res = self.returning_import(matric_no = st.matric_no)
736        res = self.returning_import(id = student_id)
737        if res:
738            student = res[0]
739        else:
740            logger.info('Id %s not found in returning_import' % student_id)
741            return
742        logger.info('%s creates data structure' % student_id)
743        s_results = self.results_import(matric_no = student.matric_no)
744        if s_results:
745            lnr = self.getLevelFromResultsCosCode(s_results)
746            level = "%d00" % lnr
747            verdict,eligible = self.getVerdict(s_results[0].Verdict)
748            #if eligible:
749            #    level = "%d00" % (lnr + 1)
750        else:
751            logger.info('matric_no %s not found in results_import' % student.matric_no)
752            level = ''
753            verdict = ''
754        #student should not be allowed to perform this transition
755        #wftool = self.portal_workflow
756        #wftool.doActionFor(student,'return')
757        certcode_org = student.Coursemajorcode
758        certcode = makeCertificateCode(certcode_org)
759        certificate_brain = self.getCertificateBrain(certcode)
760        if not certificate_brain:
761            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
762            logger.info(em)
763        matric_no = student.matric_no
764        sid = student_id
765        student_obj = getattr(students_folder,sid)
766        student_obj.invokeFactory('StudentApplication','application')
767        application = student_obj.application
768        self.portal_workflow.doActionFor(application,'open',dest_container=application)
769        da = {'Title': 'Application Data'}
770        student_obj.invokeFactory('StudentPersonal','personal')
771        da['jamb_reg_no'] = student.Entryregno
772        em = self.getEntryMode(student.Entryregno)
773        da['entry_mode'] = em
774        personal = student_obj.personal
775        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
776        dp = {'Title': 'Personal Data'}
777        student_obj.invokeFactory('StudentClearance','clearance')
778        clearance = student_obj.clearance
779        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
780        dc = {'Title': 'Clearance/Eligibility Record'}
781        dc['matric_no'] = matric_no
782        state = student.State
783        lga = student.LGA
784        if state and lga:
785            lga =  state + ' / ' + lga
786        else:
787            lga = "None"
788        da['jamb_lga'] = dc['lga'] = lga
789        da['app_email'] = dp['email'] = email
790        da['app_mobile'] = dp['phone'] = phone_nr
791        dp['firstname'] = student.Firstname
792        dp['middlename'] = student.Middlename
793        dp['lastname'] = student.Lastname
794        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
795        da['jamb_sex'] = student.Sex
796        dp['sex'] = student.Sex == 'F'
797        dp['perm_address'] = student.Permanent_Address
798        application.getContent().edit(mapping=da)
799        self.portal_workflow.doActionFor(application,'close',dest_container=application)
800        personal.getContent().edit(mapping=dp)
801        clearance.getContent().edit(mapping=dc)
802        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
803        #
804        # Study Course
805        #
806        student_obj.invokeFactory('StudentStudyCourse','study_course')
807        studycourse = student_obj.study_course
808        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
809        dsc = {}
810        dsc['study_course'] = certcode
811        dsc['current_level'] = level
812        dsc['current_verdict'] = verdict
813        dsc['current_mode'] = em
814        dsc['current_session'] = '05'
815        studycourse.getContent().edit(mapping=dsc)
816        #
817        # Level
818        #
819##        l = getattr(studycourse,level,None)
820##        if l is None:
821##            studycourse.invokeFactory('StudentStudyLevel', level)
822##            l = getattr(studycourse, level)
823##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
824##            l.getContent().edit(mapping={'Title': "Level %s" % level})
825###)
826
827    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
828    def admitOneStudent(self,brain,entry_session,pin_password):
829        "create Datastructure for an admitted Student"
830        #import pdb;pdb.set_trace()
831        if not hasattr(self,"_v_certificates"):
832            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
833        students_folder = self.portal_url.getPortalObject().campus.students
834        logger = logging.getLogger('WAeUPTool.admitOneStudent')
835        if brain.status != "admitted":
836            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
837            return
838        pin_parts = brain.pin.split('-')
839        if pin_parts and len(pin_parts) != 3:
840            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
841            return
842        if brain.course_admitted not in self._v_certificates:
843            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
844            return
845        if brain.sex not in (True,False):
846            logger.info('sex of %s not available' % (brain.reg_no))
847            return           
848        student_id = self.generateStudentId('?')
849        students_folder.invokeFactory('Student', student_id)
850        student_object = getattr(students_folder,student_id)
851        if pin_password:
852            password = pin_parts[2]
853            self.makeStudentMember(student_id,password = password)
854        student_object.manage_setLocalRoles(student_id, ['Owner',])
855        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
856        #
857        # application
858        #
859        student_object.invokeFactory('StudentApplication','application')
860        application = student_object.application
861        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
862        da = {'Title': 'Application Data'}
863        da['jamb_reg_no'] = brain.reg_no
864
865        sex = 'M'
866        if brain.sex:
867            sex = 'F'
868        da['jamb_sex'] = sex
869        da['jamb_age'] = brain.jamb_age
870        #da['app_ac_pin'] = brain.pin
871        da['jamb_lga'] = brain.jamb_lga
872        da['jamb_state'] = brain.jamb_state
873        da['jamb_score'] = brain.aggregate
874        da['app_email'] = brain.email
875        da['app_mobile'] = brain.phone
876        if brain.entry_mode:
877            da['entry_mode'] = brain.entry_mode
878        elif brain.screening_type == 'pume':
879            da['entry_mode'] = 'ume_ft'
880        elif brain.screening_type == 'pde':
881            da['entry_mode'] = 'de_ft'
882        elif brain.screening_type == 'pce':
883            da['entry_mode'] = 'pce'
884        elif brain.screening_type == 'prence':
885            da['entry_mode'] = 'prence'
886        else:
887            da['entry_mode'] = ''
888        da['entry_session'] = entry_session
889        da['jamb_lastname'] = brain.lastname
890        da['jamb_middlename'] = brain.middlenames   # different field names!
891        da['jamb_firstname'] = brain.firstname
892        da['screening_application_date'] = brain.application_date
893        da['date_of_birth'] = brain.date_of_birth
894        da['jamb_first_cos'] = brain.course1
895        da['jamb_second_cos'] = brain.course2
896        da['course3'] = brain.course3
897        da['screening_type'] = brain.screening_type
898        da['screening_score'] = brain.screening_score
899        da['screening_date'] = brain.screening_date
900        da['hq_type'] = brain.hq_type
901        da['hq_grade'] = brain.hq_grade
902        da['aos'] = brain.aos
903
904        application.getContent().edit(mapping=da)
905        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
906        #
907        # personal
908        #
909        student_object.invokeFactory('StudentPersonal','personal')
910        personal = student_object.personal
911        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
912        dp = {'Title': 'Personal Data'}
913        dp['sex'] = brain.sex
914        dp['email'] = brain.email
915        dp['phone'] = brain.phone
916        dp['lastname'] = brain.lastname
917        dp['middlename'] = brain.middlenames   # different field names!
918        dp['firstname'] = brain.firstname
919        personal.getContent().edit(mapping=dp)
920        #
921        # clearance
922        #
923        student_object.invokeFactory('StudentClearance','clearance')
924        clearance = student_object.clearance
925        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
926        dc = {'Title': 'Clearance/Eligibility Record'}
927        dc['lga'] = brain.lga
928        dc['birthday'] = brain.date_of_birth
929        clearance.getContent().edit(mapping=dc)
930        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
931        #
932        # study Course
933        #
934        student_object.invokeFactory('StudentStudyCourse','study_course')
935        studycourse = student_object.study_course
936        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
937        dsc = {}
938        dsc['study_course'] = brain.course_admitted
939        dsc['current_verdict'] = ''
940        dsc['current_mode'] = da['entry_mode']
941        if da['entry_mode'].startswith('de'):
942            dsc['current_level'] = '200'
943        else:
944            dsc['current_level'] = '100'
945        dsc['current_session'] = entry_session
946        studycourse.getContent().edit(mapping=dsc)
947        #
948        # payments folder
949        student_object.invokeFactory('PaymentsFolder','payments')
950        payments = getattr(student_object,'payments')
951        dpay = {}
952        dpay['Title'] = 'Payments'
953        payments.getContent().edit(mapping=dpay)
954        self.portal_workflow.doActionFor(payments,'open')
955        #
956        # passport foto
957        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
958                                                              brain.screening_type,
959                                                              brain.reg_no)
960        images_dir = getImagesDir(student_id)
961        #images_dir = os.path.join("%s" % images_base,student_id)
962        letter_dir,student_dir = os.path.split(images_dir)
963        if not os.path.exists(letter_dir):
964            os.mkdir(letter_dir)
965        if not os.path.exists(images_dir):
966            os.mkdir(images_dir)
967        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
968        if os.path.exists(app_picture):
969            copy2(app_picture,image_name)
970        else:
971            logger.info('passport of %s/%s not found: %s' % (student_id,
972                                                             brain.reg_no,
973                                                             app_picture))
974
975        return student_id
976    ###)
977
978    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
979    def makeStudentLevel(self,student_id):
980        "create the StudyLevel for a returning Student"
981        #import pdb;pdb.set_trace()
982        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
983        students_folder = self.portal_url.getPortalObject().campus.students
984        res = self.students_catalog(id=student_id)
985        if res:
986            st = res[0]
987        course = st.course
988        matric_no = st.matric_no
989        level = st.level
990        res = self.results_import(matric_no = matric_no)
991        if res:
992            results = res
993        logger.info('%s creating Level %s' % (student_id,level))
994        #
995        # Level
996        #
997        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
998        studycourse = getattr(student_obj,"study_course",None)
999        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1000        l = getattr(studycourse,level,None)
1001        if l is None:
1002            studycourse.invokeFactory('StudentStudyLevel', level)
1003            l = getattr(studycourse, level)
1004            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1005            l.getContent().edit(mapping={'Title': "Level %s" % level})
1006        ###)
1007
1008    security.declarePublic('getHallInfo') ###(
1009    def getHallInfo(self,bed):
1010        """return Hall Info"""
1011        info = {}
1012        hall,block,room,letter = bed.split('_')
1013        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1014        if res and len(res) == 1:
1015            hall_brain = res[0]
1016            hall_doc = hall_brain.getObject().getContent()
1017        else:
1018            return info
1019        info['hall_title'] = hall_brain.Title
1020        info['maintenance_code'] = hall_doc.maintenance_code
1021        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1022        batch_doc = None
1023        for brain in res:
1024            if brain.id.startswith(info['maintenance_code']):
1025                batch_doc = brain.getObject().getContent()
1026                break
1027        if batch_doc is None:
1028            info['maintenance_fee'] = ''
1029        else:
1030            info['maintenance_fee'] = batch_doc.cost
1031        return info
1032    ###)
1033
1034    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1035    def removePictureFolder(self,student_id):
1036        """remove picture_folder by renaming it"""
1037        path = 'images'
1038        picture_path = os.path.join(i_home,path,student_id)
1039        if not os.path.exists(picture_path):
1040            return False
1041        os.rename(picture_path,picture_path + "_removed")
1042        return True
1043    ###)
1044
1045    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1046    def restorePictureFolder(self,student_id):
1047        """restore picture_folder by renaming it"""
1048        path = 'images'
1049        picture_path = os.path.join(i_home,path,student_id)
1050        if not os.path.exists(picture_path + "_removed"):
1051            return False
1052        os.rename(picture_path + "_removed",picture_path)
1053        return True
1054    ###)
1055
1056    security.declarePublic('picturesExist') ###(
1057    def picturesExist(self, ids,student_id=None):
1058        """check if pictures exist in the filesystem"""
1059        if student_id is None:
1060            student_id = self.getStudentId()
1061        if student_id is None:
1062            return False
1063        picture_path = getImagesDir(student_id)
1064        #picture_path = os.path.join(images_base,student_id)
1065        if not os.path.exists(picture_path):
1066            return False
1067        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1068        return set(ids).issubset(set(pictures))
1069    ###)
1070
1071    security.declarePublic('picturesList') ###(
1072    def picturesList(self):
1073        """check if pictures exist in the filesystem"""
1074        path = 'images'
1075        student_id = self.getStudentId()
1076        #picture_path = os.path.join(i_home,path,student_id)
1077        picture_path = getImagesDir(student_id)
1078        if not os.path.exists(picture_path):
1079            return []
1080        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1081    ###)
1082
1083    security.declarePublic('showFsPicture') ###(
1084    def showFsPicture(self,path):
1085        """return a picture from the filesystem"""
1086        #picture_path = os.path.join(i_home,path)
1087        picture_path = os.path.join(images_base,path)
1088        response = self.REQUEST.RESPONSE
1089        #import pdb;pdb.set_trace()
1090        registry = getToolByName(self, 'mimetypes_registry')
1091        mimetype = str(registry.lookupExtension(path.lower()) or
1092                    registry.lookupExtension('file.bin'))
1093        if os.path.exists(picture_path):
1094            response.setHeader('Content-type',mimetype)
1095            return open(picture_path).read()
1096        picture_path = os.path.join(i_home,'import',path)
1097        if os.path.exists(picture_path):
1098            return open(picture_path).read()
1099    ###)
1100
1101    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1102    def deleteAllCourses(self,department="All"):
1103        ''' delete the courses'''
1104        pm = self.portal_membership
1105        member = pm.getAuthenticatedMember()
1106
1107        if str(member) not in ("henrik","joachim"):
1108            return "not possible"
1109        if department == "All":
1110            res = self.portal_catalog({'meta_type': 'Department'})
1111        if len(res) < 1:
1112            return "No Departments found"
1113
1114        deleted = []
1115        for dep in res:
1116            cf = dep.getObject().courses
1117            if cf:
1118                cf.manage_delObjects(ids=cf.objectIds())
1119                deleted.append("deleted Courses in %s" % dep.getId)
1120        return "\r".join(deleted)
1121    ###)
1122
1123    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1124    def getLogfileLines(self,filename="event.log",numlines=20):
1125        """Get last NUMLINES lines of logfile FILENAME.
1126
1127        Return last lines' of a file in the instances logfile directory as
1128        a list. The number of returned lines equals `numlines' or less. If
1129        less than `numlines' lines are available, the whole file ist
1130        returned. If the file can not be opened or some other error
1131        occurs, empty list is returend.
1132        """
1133        result = []
1134        lines_hit = 0
1135
1136        # We only handle files in instances' log directory...
1137        logpath = os.path.join(i_home, "log")
1138        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1139        if not filename.startswith( logpath ):
1140            # Attempt to access file outside log-dir...
1141            return []
1142
1143        try:
1144            fd = file( filename, "rb" )
1145        except IOError:
1146            return []
1147        if not fd:
1148            return []
1149
1150        if os.linesep == None:
1151            linesep = '\n'
1152        else:
1153            linesep = os.linesep
1154
1155        # Try to find 'numlines' times a lineseparator, searching from end
1156        # and moving to the beginning of file...
1157        fd.seek( 0, 2) # Move to end of file...
1158        while lines_hit < numlines:
1159            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1160                                          # one step forward...
1161                lines_hit += 1
1162            try:
1163                fd.seek( -2, 1) # Go two bytes back from current pos...
1164            except IOError:
1165                # We cannot go back two bytes. Maybe the file is too small...
1166                break
1167        fd.seek(2,1)
1168
1169        # Read all lines from current position...
1170        result = fd.readlines()
1171        # Remove line endings...
1172        result = [x.strip() for x in result]
1173        fd.close()
1174        return result
1175    ###)
1176
1177    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1178    def getCallbacksFromLog(self,filename):
1179        """fix Online Payment Transactions from Z2.log entries"""
1180        import transaction
1181        import random
1182        from cgi import parse_qs
1183        from urlparse import urlparse
1184        #from pdb import set_trace
1185        wftool = self.portal_workflow
1186        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1187        students_folder = self.portal_url.getPortalObject().campus.students
1188        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1189        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1190        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1191        data = re.compile(s)
1192        start = True
1193        tr_count = 1
1194        total = 0
1195        #name = 'pume_results'
1196        #name = 'epaymentsuccessful_z2log2'
1197        name = filename
1198        no_import = []
1199        imported = []
1200        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1201        try:
1202            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1203        except:
1204            logger.error('Error reading %s' % name)
1205            return
1206        tas = []
1207        for line in transactions:
1208            dict = {}
1209            items = data.search(line)
1210            dict['idict'] = idict = items.groupdict()
1211            #print idict
1212            #from pdb import set_trace;set_trace()
1213            urlparsed = urlparse(idict['get'][4:])
1214            #print urlparsed
1215            path = urlparsed[2].split('/')
1216            dict['student_id'] = student_id = path[8]
1217            dict['payment_id'] = payment_id = path[10]
1218            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1219            tas.append(dict)
1220            tr_count += 1
1221        return tas
1222    ###)
1223
1224    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1225    def importOnlinePaymentTransactions(self):
1226        """load Online Payment Transactions from CSV values"""
1227        import transaction
1228        import random
1229        #from pdb import set_trace
1230        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1231        opt = self.online_payments_import
1232        students_folder = self.portal_url.getPortalObject().campus.students
1233        start = True
1234        tr_count = 1
1235        total = 0
1236        #name = 'pume_results'
1237        name = 'OnlineTransactions'
1238        no_import = []
1239        imported = []
1240        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1241        try:
1242            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1243        except:
1244            logger.error('Error reading %s.csv' % name)
1245            return
1246        for pay_transaction in transactions:
1247            if start:
1248                start = False
1249                logger.info('Start loading from %s.csv' % name)
1250                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1251                no_import.append('%s,"Error"' % s)
1252                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1253                format_error = format + ',"%(Error)s"'
1254            data = {}
1255
1256            # format of the first file sent by Tayo
1257            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1258            #data['student_id'] = student_id = pay_transaction['Payer ID']
1259            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1260            #data['response_code'] = response_code = pay_transaction['Resp Code']
1261            #data['amount'] = amount = pay_transaction['Amount']
1262
1263            # format of the second file sent by Tayo
1264            #data['datetime'] = date = 0
1265            #data['student_id'] = student_id = pay_transaction['Payer ID']
1266            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1267            #data['response_code'] = response_code = '00'
1268            #data['amount'] = amount = pay_transaction['Amount']
1269
1270            # format of the third file sent by Kehinde
1271            data['datetime'] = date = 0
1272            data['student_id'] = student_id = pay_transaction['customer_id']
1273            data['order_id'] = order_id = pay_transaction['merchant_reference']
1274            data['response_code'] = response_code = '00'
1275            data['amount'] = amount = pay_transaction['Amount']
1276
1277            dup = False
1278            if response_code == "12":
1279                continue
1280            try:
1281                opt.addRecord(**data)
1282            except ValueError:
1283                dup = True
1284            #from pdb import set_trace;set_trace()
1285            if dup:
1286                if response_code == "00":
1287                    try:
1288                        opt.modifyRecord(**data)
1289                    except:
1290                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1291                        continue
1292                else:
1293                    pay_transaction['Error'] = "Duplicate order_id"
1294                    no_import.append( format_error % pay_transaction)
1295                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1296                    continue
1297            tr_count += 1
1298            if tr_count > 1000:
1299                if len(no_import) > 0:
1300                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1301                             '\n'.join(no_import) + '\n')
1302                    no_import = []
1303                em = '%d transactions committed\n' % (tr_count)
1304                transaction.commit()
1305                regs = []
1306                logger.info(em)
1307                total += tr_count
1308                tr_count = 0
1309        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1310                                                '\n'.join(no_import))
1311        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1312    ###)
1313
1314    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1315    def mass_create_faculty(self,mapping):
1316        "create a faculty"
1317        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1318        academics_folder = self.portal_url.getPortalObject().campus.academics
1319        fid = mapping['code']
1320        if getattr(academics_folder,fid,None) is not None:
1321            return '', "Faculty with ID: %s exists" % fid
1322        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1323        try:
1324            academics_folder.invokeFactory('Faculty', fid)
1325        except BadRequest,E:
1326            return '', "%s" % E
1327        f = getattr(academics_folder,fid,None)
1328        f.getContent().edit(mapping=mapping)
1329        return fid,''
1330    ###)
1331
1332    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1333    def mass_edit_faculty(self,mapping):
1334        "edit a faculty"
1335        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1336        academics_folder = self.portal_url.getPortalObject().campus.academics
1337        fid = mapping['code']
1338        f = getattr(academics_folder,fid,None)
1339        if f is None:
1340            return '', "Faculty with ID: %s does not exist" % fid
1341        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1342        f.getContent().edit(mapping=mapping)
1343        return fid,''
1344    ###)
1345
1346    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1347    def mass_create_department(self,mapping):
1348        "create a department in the correct faculty"
1349        logger = logging.getLogger('WAeUPTool.mass_create_department')
1350        fid = mapping['faculty_code']
1351        if getattr(self,'_v_faculties',None) is None:
1352            res = self.portal_catalog(portal_type = "Faculty")
1353            self._v_faculties = {}
1354            for f in res:
1355                self._v_faculties[f.getId] = f.getObject()
1356        f = self._v_faculties.get(fid,None)
1357        if f is None:
1358            return '', "No Faculty with ID: %s" % fid
1359        else:
1360            did = mapping.get('code')
1361            d = getattr(f,did,None)
1362            if d is None or d.portal_type == "Faculty":
1363                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1364                try:
1365                    f.invokeFactory('Department', did)
1366                except BadRequest,E:
1367                    return '', "%s" % E
1368                d = getattr(f,did)
1369                d.invokeFactory('CoursesFolder','courses')
1370                courses = getattr(d,'courses')
1371                dict = {'Title': 'Courses'}
1372                courses.getContent().edit(mapping=dict)
1373                d.invokeFactory('CertificatesFolder','certificates')
1374                certificates = getattr(d,'certificates')
1375                dict = {'Title': 'Certificates'}
1376                certificates.getContent().edit(mapping=dict)
1377            d.getContent().edit(mapping=mapping)
1378        return did,''
1379    ###)
1380
1381    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1382    def mass_edit_department(self,mapping):
1383        "create a department in the correct faculty"
1384        logger = logging.getLogger('WAeUPTool.mass_create_department')
1385        academics_folder = self.portal_url.getPortalObject().campus.academics
1386        fid = mapping['faculty_code']
1387        did = mapping.get('code')
1388        try:
1389            d = getattr(getattr(academics_folder,fid),did,None)
1390        except KeyError:
1391            return '', "Department %s or Faculty %s wrong" % (did,fid)
1392        else:
1393            if d is None or d.portal_type == "Faculty":
1394                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1395            d.getContent().edit(mapping=mapping)
1396        return did,''
1397    ###)
1398
1399    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1400    def mass_create_course(self,mapping):
1401        #import pdb;pdb.set_trace()
1402        if getattr(self,'_v_course_list',None) is None:
1403            self._v_course_list = []
1404        if getattr(self,'_v_departments',None) is None:
1405            res = self.portal_catalog(portal_type = "Department")
1406            self._v_department_courses = {}
1407            for d in res:
1408                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1409        did = mapping['department_code']
1410        d = self._v_department_courses.get(did,None)
1411        if d is None:
1412            return '', "No Department with ID: %s" % did
1413        course_id = mapping.get('code')
1414        if course_id in self._v_course_list:
1415            return '', "Duplicate Course ID: %s" % did
1416        c = getattr(d,course_id,None)
1417        if c is not None:
1418            return '', "Duplicate Course ID: %s" % did
1419        try:
1420            d.invokeFactory('Course', course_id)
1421        except BadRequest,E:
1422            return '', "%s" % E
1423        self._v_course_list.append(course_id)
1424        c = getattr(d,course_id)
1425        c.getContent().edit(mapping=mapping)
1426        return course_id,''
1427    ###)
1428
1429    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1430    def mass_edit_course(self,mapping):
1431        #import pdb;pdb.set_trace()
1432        course_id = mapping.get('code')
1433        res = self.portal_catalog(id=course_id)
1434        if not res:
1435            return '', "No Course with ID: %s" % course_id
1436        c = res[0].getObject()
1437        c.getContent().edit(mapping=mapping)
1438        return course_id,''
1439    ###)
1440
1441    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1442    def mass_create_certificate(self,mapping):
1443        if getattr(self,'_v_certificate_list',None) is None:
1444            self._v_certificate_list = []
1445        if getattr(self,'_v_department_certificates',None) is None:
1446            res = self.portal_catalog(portal_type = "Department")
1447            self._v_department_certificates = {}
1448            for d in res:
1449                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1450        did = mapping['department_code']
1451        d = self._v_department_certificates.get(did,None)
1452        if d is None:
1453            return '', "No Department with ID: %s" % did
1454        certificate_id = mapping.get('code')
1455        if certificate_id in self._v_certificate_list:
1456            return '', "Duplicate Certificate ID: %s" % did
1457        c = getattr(d,certificate_id,None)
1458        if c is not None:
1459            return '', "Duplicate Certificate ID: %s" % did
1460        try:
1461            d.invokeFactory('Certificate', certificate_id)
1462        except BadRequest,E:
1463            return '', "%s" % E
1464        self._v_certificate_list.append(certificate_id)
1465        c = getattr(d,certificate_id)
1466        c.getContent().edit(mapping=mapping)
1467        return certificate_id,''
1468    ###)
1469
1470    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1471    def mass_edit_certificate(self,mapping):
1472        #import pdb;pdb.set_trace()
1473        certificate_id = mapping.get('code')
1474        res = self.portal_catalog(id=certificate_id)
1475        if not res:
1476            return '', "No Certificate with ID: %s" % did
1477        c = res[0].getObject()
1478        c.getContent().edit(mapping=mapping)
1479        return certificate_id,''
1480    ###)
1481
1482    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1483    def mass_create_application(self,mapping):
1484        #import pdb;pdb.set_trace()
1485        reg_no = mapping.get('reg_no')
1486        try:
1487            self.applicants_catalog.addRecord(**mapping)
1488        except ValueError:
1489            return '', "applicant record with reg_no %s already exists" % reg_no
1490        return reg_no,''
1491    ###)
1492
1493    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1494    def mass_edit_application(self,mapping):
1495        #import pdb;pdb.set_trace()
1496        reg_no = mapping.get('reg_no')
1497        status = mapping.get('status')
1498        res = self.applicants_catalog(reg_no = reg_no)
1499        if len(res):
1500            if res[0].status == 'created' and status != 'created':
1501                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1502            elif status == 'created' and res[0].status != 'created':
1503                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1504            else:
1505                self.applicants_catalog.modifyRecord(**mapping)
1506                return reg_no,''
1507        else:
1508            return '', "applicant record with reg_no %s does not exist" % reg_no
1509
1510    ###)
1511
1512    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1513    def mass_create_course_result(self,mapping):
1514        #import pdb;pdb.set_trace()
1515        students_folder = self.portal_url.getPortalObject().campus.students
1516        if getattr(self,'_v_courses',None) is None:
1517            res = self.courses_catalog()
1518            self._v_courses = {}
1519            for brain in res:
1520                self._v_courses[brain.code] = brain
1521        if getattr(self,'_v_level_created',None) is None:
1522            self._v_level_created = []
1523        course_id = mapping.get('code')
1524        if course_id not in self._v_courses.keys():
1525            return '', "No course with ID: %s" % course_id
1526        id_key = ''
1527        for id_key in ('student_id','matric_no'):
1528            id_field = mapping.get(id_key,None)
1529            if id_field is not None:
1530                student_id = id_field
1531                break
1532        query = Eq(id_key,id_field)
1533        res = self.students_catalog.evalAdvancedQuery(query)
1534        if not res:
1535            return '', "no student with %(id_key)s %(id_field)s" % vars()
1536        student_rec = res[0]
1537        if id_field != "student_id":
1538            mapping['student_id'] = student_rec.id
1539        student_id = student_rec.id
1540        level_id = mapping['level_id']
1541        code = mapping['code']
1542        if student_id not in self._v_level_created:
1543            try:
1544                context = getattr(getattr(students_folder,
1545                                          "%(student_id)s" % vars()),
1546                                  'study_course')
1547            except:
1548                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1549            if level_id not in context.objectIds():
1550                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1551                level = getattr(context,"%(level_id)s" % vars())
1552                self.portal_workflow.doActionFor(level,'open')
1553                # the session string must not be copied into the level object
1554                current_verdict = getattr(student_rec,'current_verdict','')
1555                current_session = getattr(student_rec,'current_session','')
1556                if current_verdict and student_rec.current_level == level_id:
1557                    level.getContent().edit(mapping={'verdict': "%s" %
1558                                                     current_verdict,
1559                                                     'session': "%s" %
1560                                                     current_session,
1561                                                    })
1562                    self.portal_workflow.doActionFor(level,'close')
1563            self._v_level_created += student_id,
1564        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1565        for k in ('semester','credits',):
1566            mapping[k] = getattr(self._v_courses[course_id],k)
1567        try:
1568            self.course_results.addRecord(**mapping)
1569        except ValueError:
1570            return '', "course result already exists: %s" % key
1571        return key,''
1572    ###)
1573
1574    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1575    def mass_edit_course_result(self,mapping):
1576        #import pdb;pdb.set_trace()
1577        # if getattr(self,'_v_courses',None) is None:
1578        #     res = self.courses_catalog()
1579        #     self._v_courses = {}
1580        #     for brain in res:
1581        #         self._v_courses[brain.code] = brain
1582        # course_id = mapping.get('code')
1583        # if course_id not in self._v_courses.keys():
1584        #     return '', "No course with ID: %s" % did
1585        id_key = ''
1586        for id_key in ('student_id','matric_no'):
1587            id_field = mapping.get(id_key,None)
1588            if id_field is not None:
1589                student_id = id_field
1590                break
1591        query = Eq(id_key,id_field)
1592        res = self.students_catalog.evalAdvancedQuery(query)
1593        if not res:
1594            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1595        if id_field != "student_id":
1596            mapping['student_id'] = res[0].id
1597        if not res:
1598            return '', "no student with %(id_key)s %(id_field)s" % vars()
1599        student_rec = res[0]
1600        student_id = student_rec.id
1601        level_id = mapping['level_id']
1602        code = mapping['code']
1603        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1604        # for k in ('semester',):
1605        #     mapping[k] = getattr(self._v_courses[course_id],k)
1606        try:
1607            self.course_results.modifyRecord(**mapping)
1608        except KeyError:
1609            return '', "No course result to edit: %s" % key
1610        return key,''
1611    ###)
1612
1613    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1614    def mass_create_certificate_course(self,mapping):
1615        if getattr(self,'_v_courses',None) is None:
1616            res = self.courses_catalog()
1617            self._v_courses= [course.code for course in res]
1618        if getattr(self,'_v_ceritficates',None) is None:
1619            res = self.portal_catalog(portal_type = "Certificate")
1620            self._v_certificates = {}
1621            for cert in res:
1622                self._v_certificates[cert.getId] = cert.getObject()
1623        certificate_course_id = mapping.get('code')
1624        if certificate_course_id not in self._v_courses:
1625            return '', "No Course with ID: %s" % certificate_course_id
1626        cert_id = mapping['certificate_code']
1627        cert = self._v_certificates.get(cert_id,None)
1628        if cert is None:
1629            return '', "No Certificate with ID: %s" % cert_id
1630        level_id = mapping.get('level')
1631        level = getattr(cert,level_id,None)
1632        if level is None:
1633            cert.invokeFactory('StudyLevel', level_id)
1634            level = getattr(cert,level_id,None)
1635        elif hasattr(level,certificate_course_id):
1636            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1637            (certificate_course_id,cert_id,level_id)
1638        level.invokeFactory('CertificateCourse', certificate_course_id)
1639        c = getattr(level,certificate_course_id)
1640        c.getContent().edit(mapping=mapping)
1641        return certificate_course_id,''
1642    ###)
1643
1644    field2types_student = {   ###(
1645                      'StudentApplication':
1646                          {'id': 'application',
1647                           'title': 'Application Data',
1648                           'wf_transition_return': 'close',
1649                           'wf_transition_admit': 'remain',
1650                           'fields':
1651                             ('jamb_reg_no',
1652                              'entry_mode',
1653                              'entry_session',
1654                              'jamb_score',
1655                              'app_email',
1656                              'jamb_age',
1657                              'jamb_state',
1658                              'jamb_lga',
1659                              )
1660                              },
1661                      #'StudentPume':
1662                      #    {'id': 'pume',
1663                      #     'title': 'Pume Data',
1664                      #     'wf_transition_return': 'close',
1665                      #     'wf_transition_admit': 'close',
1666                      #     'fields':
1667                      #       ('pume_score',
1668                      #        )
1669                      #        },
1670                      'StudentClearance':
1671                          {'id': 'clearance',
1672                           'title': 'Clearance/Eligibility Record',
1673                           'wf_transition_return': 'close',
1674                           'wf_transition_admit': 'remain',
1675                           'fields':
1676                             ('matric_no',
1677                              'nationality',
1678                              'lga',
1679                              'birthday',
1680                              )
1681                              },
1682                         'StudentPersonal':
1683                          {'id': 'personal',
1684                           'title': 'Personal Data',
1685                           'wf_transition_return': 'open',
1686                           'wf_transition_admit': 'remain',
1687                           'fields':
1688                             ('firstname',
1689                              'middlename',
1690                              'lastname',
1691                              'sex',
1692                              'email',
1693                              'phone',
1694                              'perm_address',
1695                              )
1696                              },
1697                         'StudentStudyCourse':
1698                          {'id': 'study_course',
1699                           'title': 'Study Course',
1700                           'wf_transition_return': 'open',
1701                           'wf_transition_admit': 'remain',
1702                           'fields':
1703                             ('study_course',
1704                              'current_level',
1705                              'current_session',
1706                              'current_mode',
1707                              'current_verdict',
1708                              'previous_verdict',
1709                              )
1710                              },
1711                         # 'StudentStudyLevel':
1712                         #  {'id': 'current_level',
1713                         #   'title': '',
1714                         #   'wf_transition_return': 'open',
1715                         #   'wf_transition_admit': 'remain',
1716                         #   'fields':
1717                         #     ('verdict',
1718                         #      'session',
1719                         #      )
1720                         #      },
1721                         'PaymentsFolder':
1722                          {'id': 'payments',
1723                           'title': 'Payments',
1724                           'wf_transition_return': 'open',
1725                           'wf_transition_admit': 'open',
1726                           'fields':
1727                             ()
1728                              },
1729                         }
1730    ###)
1731
1732    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1733    def mass_create_student(self,mapping):
1734        "create a students record due import"
1735        logger = logging.getLogger('WAeUPTool.mass_create_student')
1736        students_folder = self.portal_url.getPortalObject().campus.students
1737        jamb_reg_no = mapping.get('jamb_reg_no',None)
1738        if jamb_reg_no:
1739            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1740            if res:
1741                return '',"jamb_reg_no exists"
1742        matric_no = mapping.get('matric_no',None)
1743        if matric_no:
1744            res = self.students_catalog(matric_no = matric_no)
1745            if res:
1746                return '',"matric_no exists"
1747        sid = self.waeup_tool.generateStudentId('?')
1748        students_folder.invokeFactory('Student', sid)
1749        student_obj = getattr(students_folder,sid)
1750        f2t = self.field2types_student
1751        d = {}
1752        transition = mapping.get('reg_transition','admit')
1753        if transition not in ('admit','return'):
1754            return '',"no valid transition provided"
1755        for pt in f2t.keys():
1756            student_obj.invokeFactory(pt,f2t[pt]['id'])
1757            sub_obj = getattr(student_obj,f2t[pt]['id'])
1758            sub_doc = sub_obj.getContent()
1759            d['Title'] = f2t[pt]['title']
1760            for field in f2t[pt]['fields']:
1761                d[field] = mapping.get(field,'')
1762
1763            if pt == "StudentApplication":
1764                d['jamb_sex']  = 'M'
1765                if mapping.get('sex'):
1766                    d['jamb_sex']  = 'F'
1767                d['jamb_firstname'] = mapping.get('firstname',None)
1768                d['jamb_middlename'] = mapping.get('middlename',None)
1769                d['jamb_lastname'] = mapping.get('lastname',None)
1770
1771            # if pt == "StudyCourse":
1772            #     for von,zu in (('entry_mode','current_mode'),
1773            #                    ('entry_session','current_session')):
1774            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1775            #             d[zu] = mapping[von]
1776            sub_doc.edit(mapping = d)
1777
1778            #import pdb;pdb.set_trace()
1779            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1780            if new_state != "remain":
1781                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1782        self.portal_workflow.doActionFor(student_obj,transition)
1783        student_obj.manage_setLocalRoles(sid, ['Owner',])
1784        return sid,''
1785    ###)
1786
1787    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1788    def mass_edit_student(self,mapping):
1789        wftool = self.portal_workflow
1790        "edit a students record due import"
1791        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1792        students_folder = self.portal_url.getPortalObject().campus.students
1793        sid = mapping.get('id',None)
1794        jamb_reg_no = mapping.get('jamb_reg_no',None)
1795        matric_no = mapping.get('matric_no',None)
1796        editable_keys = mapping.keys()
1797        if sid:
1798            res = self.students_catalog(id = sid)
1799            if not res:
1800                return '',"no student with id %s" % sid
1801            if matric_no and res[0].matric_no and\
1802              matric_no != res[0].matric_no:
1803                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1804            if jamb_reg_no and res[0].jamb_reg_no and\
1805              jamb_reg_no != res[0].jamb_reg_no:
1806                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1807        elif jamb_reg_no:
1808            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1809            if not res:
1810                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1811            editable_keys.remove('jamb_reg_no')
1812        elif matric_no:
1813            res = self.students_catalog(matric_no = matric_no)
1814            if not res:
1815                return '',"no student with matric_no %s" % matric_no
1816            editable_keys.remove('matric_no')
1817
1818        ## included only to change wf state from admitted to returning
1819        #if res[0].review_state not in ('admitted','objection_raised'):
1820        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1821        ## end inclusion
1822
1823        sid = res[0].id
1824        student_obj = getattr(students_folder,sid)
1825        f2t = self.field2types_student
1826        d = {}
1827        #import pdb;pdb.set_trace()
1828        any_change = False
1829        #special treatment for StudentStudyLevel
1830        d['verdict']  = mapping.get('current_verdict','')
1831        d['session']  = mapping.get('current_session','')
1832        current_level = mapping.get('current_level','')
1833        while d['session'] and d['verdict'] and current_level:
1834            sub_obj = getattr(student_obj,'study_course',None)
1835            if sub_obj is None:
1836                break
1837            level_obj = getattr(sub_obj,current_level,None)
1838            if  level_obj is None:
1839                break
1840            any_change = True
1841            level_obj.getContent().edit(mapping = d)
1842            try:
1843                wftool.doActionFor(level_obj,'close')
1844            except:
1845                pass
1846            break
1847        for pt in f2t.keys():
1848            if pt == "StudentApplication":
1849                d['jamb_sex']  = 'M'
1850                if mapping.get('sex'):
1851                    d['jamb_sex']  = 'F'
1852            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1853            if intersect and pt not in ('StudentStudyLevel',):
1854                object_id = f2t[pt]['id']
1855                sub_obj = getattr(student_obj,object_id,None)
1856                if sub_obj is None:
1857                    try:
1858                        student_obj.invokeFactory(pt,object_id)
1859                    except:
1860                        continue
1861                    sub_obj = getattr(student_obj,object_id)
1862                    if f2t[pt]['title'] != '':
1863                        d['Title'] = f2t[pt]['title']
1864                sub_doc = sub_obj.getContent()
1865                for field in intersect:
1866                    changed = False
1867                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1868                        any_change = True
1869                        changed = True
1870                        d[field] = mapping.get(field,'')
1871                    if changed:
1872                        sub_doc.edit(mapping = d)
1873
1874
1875        ## included only to change wf state from admitted to returning
1876        #    if res[0].review_state in ('admitted','objection_raised'):
1877        #        new_state = f2t[pt]['wf_transition_return']
1878        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1879        #        if sub_obj and new_state != "remain":
1880        #            try:
1881        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1882        #            except:
1883        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1884        #                pass
1885        #if res[0].review_state in ('admitted','objection_raised'):
1886        #    wfaction = 'return'
1887        #    try:
1888        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1889        #        logger.info('%s, wf state changed' % sid)
1890        #        any_change = True
1891        #    except:
1892        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1893        #        pass
1894        ## end inclusion
1895
1896
1897        if any_change:
1898            return sid,''
1899        else:
1900            return sid,'not modified'
1901    ###)
1902
1903    security.declareProtected(ModifyPortalContent,"importData")###(
1904    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1905        """load data from CSV values"""
1906        import transaction
1907        import random
1908
1909        pm = self.portal_membership
1910        member = pm.getAuthenticatedMember()
1911
1912        logger = logging.getLogger('WAeUPTool.importData')
1913        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1914        students_folder = self.portal_url.getPortalObject().campus.students
1915        start = True
1916        tr_count = 0
1917        total_imported = 0
1918        total_not_imported = 0
1919        total = 0
1920        iname = "import_%s" % name
1921        if iname in ('import_application','import_course_result',):
1922            commit_after = 2000
1923        else:
1924            commit_after = 100
1925        stool = getToolByName(self, 'portal_schemas')
1926        ltool = getToolByName(self, 'portal_layouts')
1927        schema = stool._getOb(iname)
1928        if schema is None:
1929            em = 'No such schema %s' % iname
1930            logger.error('No such schema %s' % iname)
1931            return em
1932        layout = ltool._getOb(iname)
1933        if layout is None:
1934            em = 'No such layout %s' % iname
1935            logger.error(em)
1936            return em
1937        validators = {}
1938        for widget in layout.keys():
1939            validators[widget] = layout[widget].validate
1940        mode = "create"
1941        if edit:
1942            mode = "edit"
1943        importer_name = "mass_%(mode)s_%(name)s" % vars()
1944        importer = getattr(self, '%s' % importer_name,None)
1945        if importer is None:
1946            em = 'No importer function %s' % importer_name
1947            logger.error(em)
1948            return em
1949        not_imported = []
1950        imported = []
1951        try:
1952            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1953        except:
1954            em = 'Error reading %s.csv' % filename
1955            logger.error(em)
1956            return em
1957        for item in items:
1958            #import pdb;pdb.set_trace()
1959            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1960            if start:
1961                start = False
1962                adapters = [MappingStorageAdapter(schema, item)]
1963                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1964                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1965                import_keys = [k.strip() for k in attrs if not k.startswith('ignore')]
1966                diff2schema = set(import_keys).difference(set(schema.keys()))
1967                diff2layout = set(import_keys).difference(set(layout.keys()))
1968                if diff2schema:
1969                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1970                    return em
1971                if mode == "create":
1972                    required_keys = [layout.getIdUnprefixed(id)
1973                                     for id,widget in layout.objectItems()
1974                                     if widget.is_required]
1975                    if not set(required_keys).issubset(set(import_keys)):
1976                        diff2import = set(required_keys).difference(set(import_keys))
1977                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1978                        return em
1979                s = ','.join(['"%s"' % fn for fn in import_keys])
1980                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1981                s = '"id",' + s
1982                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1983                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1984                format_error = format + ',"%(Error)s"'
1985                format = '"%(id)s",'+ format
1986
1987            dm = DataModel(item, adapters,context=self)
1988            ds = DataStructure(data=item,datamodel=dm)
1989            error_string = ""
1990            total += 1
1991            error_count = 0
1992            for k in import_keys:
1993                if not validators[k](ds,mode=mode):
1994                    if error_count:
1995                        error_string += ' ++ '
1996                    error_string += "%s: %s" % (k,
1997                                                  self.translation_service(ds.getError(k),
1998                                                                           ds.getErrorMapping(k)))
1999                    error_count += 1
2000            if not error_string:
2001                temp_item = item.copy()
2002                temp_item.update(dm)
2003                temp_item['id'],error = importer(temp_item)
2004                if error:
2005                    error_string += error
2006                else:
2007                    item = temp_item
2008            if error_string:
2009                item['Error'] = error_string
2010                not_imported.append(format_error % item)
2011                total_not_imported += 1
2012            else:
2013                em = format % item
2014                imported.append(em)
2015                tr_count += 1
2016                total_imported += 1
2017                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
2018
2019            if total and not total % commit_after:
2020                transaction.commit()
2021                if len(not_imported) > 0:
2022                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2023                             '\n'.join(not_imported) + '\n')
2024                    not_imported = []
2025                if len(imported) > 0:
2026                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2027                             '\n'.join(imported) + '\n')
2028                    imported = []
2029                em = '%d transactions committed\n' % (tr_count)
2030                regs = []
2031                logger.info(em)
2032                tr_count = 0
2033                #if total > 100:
2034                #    return
2035        if len(imported) > 0:
2036            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2037                                                '\n'.join(imported))
2038        if len(not_imported) > 0:
2039            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2040                                                '\n'.join(not_imported))
2041        em = "Finished import from %s.csv: %d imported, %d not imported (of total %d)" % (filename,total_imported,total_not_imported,total)
2042        logger.info(em)
2043        return em
2044    ###)
2045
2046    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2047    def moveImagesToFS(self,student_id="O738726"):
2048        "move the images to the filesystem"
2049        images_dir = getImagesDir(student_id)
2050        #images_dir = os.path.join("%s" % images_base,student_id)
2051        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2052        stool = getToolByName(self, 'portal_schemas')
2053        schemas = ['student_application',
2054                   'student_clearance',
2055                   ]
2056        created = False
2057        for schema_id in schemas:
2058            schema = stool._getOb(schema_id)
2059            object = getattr(student_folder,schema_id[len('student_'):],None)
2060            if object is None:
2061                continue
2062            doc = object.getContent()
2063            for key in schema.keys():
2064                if schema[key].meta_type != "CPS Image Field":
2065                    continue
2066                #import pdb;pdb.set_trace()
2067                image = getattr(doc,key,None)
2068                if not image or not hasattr(image,"data"):
2069                    continue
2070                if not created:
2071                    if not os.path.exists(images_dir):
2072                        os.mkdir(images_dir)
2073                    created = True
2074                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2075                open(filename,"wb").write(str(image.data))
2076    ###)
2077
2078    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2079    def movePassportToFS(self,student_id="O738726"):
2080        "move the passports to the filesystem"
2081        images_dir = os.path.join("%s" % i_home,'passports')
2082        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2083        stool = getToolByName(self, 'portal_schemas')
2084        schemas = ['student_application',
2085                   #'student_clearance',
2086                   ]
2087        created = False
2088        for schema_id in schemas:
2089            schema = stool._getOb(schema_id)
2090            object = getattr(student_folder,schema_id[len('student_'):],None)
2091            if object is None:
2092                continue
2093            doc = object.getContent()
2094            for key in schema.keys():
2095                if schema[key].meta_type != "CPS Image Field":
2096                    continue
2097                #import pdb;pdb.set_trace()
2098                image = getattr(doc,key)
2099                if not hasattr(image,"data"):
2100                    continue
2101                if not created:
2102                    if not os.path.exists(images_dir):
2103                        os.mkdir(images_dir)
2104                    created = True
2105                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2106                open(filename,"wb").write(str(image.data))
2107    ###)
2108
2109InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.