source: WAeUP_SRP/base/WAeUPTool.py @ 3003

Last change on this file since 3003 was 3003, checked in by Henrik Bettermann, 17 years ago

jamb_sex added
  • Property svn:keywords set to Id
File size: 87.3 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3003 2008-01-09 21:03:58Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64
65class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66    """WAeUP tool"""
67
68    id = 'waeup_tool'
69    meta_type = 'WAeUP Tool'
70    _actions = ()
71    security = ClassSecurityInfo()
72    security.declareObjectProtected(View)
73    manage_options = ( ActionProviderBase.manage_options
74                     + SimpleItem.manage_options
75                     )
76
77    security.declareProtected(View,'re_split') ###(
78    def re_split(self,split_string,string):
79        return re.split(split_string,string)
80
81
82    security.declareProtected(View,'difference') ###(
83    def difference(self,l1,l2):
84        return set(l1).difference(set(l2))
85
86    def rwrite(self,s): ###(
87        response = self.REQUEST.RESPONSE
88        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
89        response.write("%s<br />\r\n" % s)
90    ###)
91
92    def addtodict(self,d,key,item): ###(
93        d[key].append(item)
94        return d[key]
95    ###)
96
97    def sleep(self,secs): ###(
98        "sleep"
99        import time
100        time.sleep(secs)
101        return
102
103###)
104
105    security.declareProtected(View,'updateRoleMappingsFor') ###(
106    def updateRoleMappingsFor(self,wf_definition,ob):
107        "do so for public"
108        wf_def = getattr(self.portal_workflow,wf_definition)
109        wf_def.updateRoleMappingsFor(ob)
110    ###)
111
112    security.declareProtected(View,'getStatesLgas') ###(
113    def getStatesLgas(self):
114        """return lga info"""
115        voc = getattr(self.portal_vocabularies,'local_gov_areas')
116        states = []
117        lgas  = []
118        d = {}
119        wd = {}
120        for k,v in voc.items():
121            parts = v.split(' / ')
122            if len(parts) == 1:
123                state = parts[0].lower()
124                lga = ""
125            elif len(parts) == 2:
126                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
127                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
128            else:
129                continue
130            if state not in states:
131                states.append(state)
132            if lga not in lgas:
133                lgas.append(lga)
134            words = re.split('[^a-zA-Z0-9/]',k)
135            words.sort()
136            wd[k] = words
137            d[k] = v
138        mapping = {}
139        mapping['word_dict'] = wd
140        mapping['lga_dict'] = d
141        mapping['states'] = states
142        mapping['lgas'] = lgas
143        return mapping
144    ###)
145
146    security.declareProtected(View,'getAccessInfo') ###(
147    def getAccessInfo(self,context):
148        "return a dict with access_info"
149        logger = logging.getLogger('WAeUPTool.getAccessInfo')
150        mtool = self.portal_membership
151        member = mtool.getAuthenticatedMember()
152        member_id = str(member)
153        info = {}
154        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
155        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
156        is_staff = info['is_staff'] = not is_anonymous and not is_student
157        roles = member.getRolesInContext(context)
158        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
159                                                                            "SectionManager" in roles or
160                                                                            "Manager" in roles)
161        is_allowed = info['is_allowed'] = not is_anonymous
162        requested_id = context.getStudentId()
163        student_id  = None
164        if not is_student and requested_id:
165            student_id  = requested_id
166        elif is_anonymous or member_id != requested_id:
167            logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
168        else:
169            student_id = member_id
170        info['student_id'] = student_id
171        return info
172    ###)
173
174    security.declareProtected(ModifyPortalContent,'openLog') ###(
175    def openLog(self,name):
176        """open a log file"""
177        version = 1
178        path = "%s/log/%s_%d.log" % (i_home,name,version)
179        while os.path.exists(path):
180            version += 1
181            path = "%s/log/%s_%d.log" % (i_home,name,version)
182        log = open(path,"w")
183        return log
184    ###)
185
186    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
187    def bypassQueueCatalog(self,enable=True):
188        """bypass the QueueCatalog by setting all indexes to process imediate,
189        if enable is True (default) the old settings are restored
190        """
191
192    ###)
193
194    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
195    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
196        """measure Object access Time"""
197        import random
198        if hasattr(self,'portal_catalog_real'):
199            aq_portal = self.portal_catalog_real.evalAdvancedQuery
200        else:
201            aq_portal = self.portal_catalog.evalAdvancedQuery
202        nr_pts = int(nr_pts)
203        probe = int(probe)
204        intervall = probe/10
205        objects = ("application","clearance","personal")
206        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
207        #i = random.randrange(num_objects)
208        count = 0
209        found = 0
210        not_found = 0
211        t_found = 0
212        t_not_found = 0
213        time_found = time_not_found = 0.0
214        t_time_found = t_time_not_found = 0.0
215        accessed = []
216        t_min = 1000
217        t_max = 0
218        #import pdb;pdb.set_trace()
219        students = self.portal_catalog(portal_type="Student")
220        num_students = len(students)
221        if method == "d":
222            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
223            res = aq_portal(query)
224            brains = {}
225            for r in res:
226                sid = r.relative_path.split('/')[-2]
227                if brains.has_key(sid):
228                    brains[sid][r.portal_type] = r
229                else:
230                    brains[sid] = {r.portal_type : r}
231            brains_list = brains.keys()
232            num_objects = len(brains_list)
233        else:
234            num_objects = num_students
235        print "="*40
236        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
237                                                                        probe,
238                                                                        nr_pts,
239                                                                        num_objects)
240        print "nr found/not time found/not min/max"
241        elapse = time.time()
242        i_elapse = time.time()
243        c_elapse = time.clock()
244        for c in range(1,probe + 1):
245            i = random.randrange(num_objects)
246            if method in ('a','b','c'):
247                student_brain = students[i]
248            elif method == "d":
249                #import pdb;pdb.set_trace()
250                student_brain = brains[brains_list[i]]
251            if method == "c":
252                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
253                res = aq_portal(query)
254                this_portal_types = [r.portal_type for r in res]
255            for i in range(nr_pts):
256                oid = objects[i]
257                if method == "a":
258                    try:
259                        student_path = student_brain.getPath()
260                        path = "%s/%s" % (student_path,oid)
261                        doc = self.unrestrictedTraverse(path).getContent()
262                        found += 1
263                        i_time = time.time() - i_elapse
264                        time_found += i_time
265                    except:
266                        not_found += 1
267                        i_time = time.time() - i_elapse
268                        time_not_found += i_time
269                        pass
270                elif method == "b":
271                    try:
272                        student_object = student_brain.getObject()
273                        doc = getattr(student_object,oid).getContent()
274                        found += 1
275                        i_time = time.time() - i_elapse
276                        time_found += i_time
277                    except:
278                        i_time = time.time() - i_elapse
279                        time_not_found += i_time
280                        not_found += 1
281                        pass
282                elif method == "c":
283                    if portal_types[i] in this_portal_types:
284                        found += 1
285                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
286                        i_time = time.time() - i_elapse
287                        time_found += i_time
288                    else:
289                        not_found += 1
290                        i_time = time.time() - i_elapse
291                        time_not_found += i_time
292                elif method == "d":
293                    if student_brain.has_key(portal_types[i]):
294                        found += 1
295                        doc = student_brain[portal_types[i]].getObject().getContent()
296                        i_time = time.time() - i_elapse
297                        time_found += i_time
298                    else:
299                        not_found += 1
300                        i_time = time.time() - i_elapse
301                        time_not_found += i_time
302                i_elapse = time.time()
303            if c and (c % intervall == 0):
304                #i_time = time.time() - i_elapse
305                t_per = 0.0
306                if found:
307                    t_per = time_found/found
308                if t_per > t_max:
309                    t_max = t_per
310                if t_per > 0.0 and t_per < t_min:
311                    t_min = t_per
312                itf = 0.0
313                if found:
314                    itf = time_found/found
315                itnf = 0.0
316                if not_found :
317                    itnf = time_not_found / not_found
318                interval_time = time_found + time_not_found
319                s = "%(c)d: %(found)d/%(not_found)d " % vars()
320                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
321                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
322                print s
323                t_found += found
324                t_not_found += not_found
325                t_time_found += time_found
326                t_time_not_found += time_not_found
327                time_found = time_not_found = 0.0
328                found = not_found = 0
329        # t_found += found
330        # t_not_found += not_found
331        elapse = time.time() - elapse
332        itf = 0.0
333        if t_found:
334            itf = t_time_found/t_found
335        itnf = 0.0
336        if t_not_found:
337            itnf = t_time_not_found / t_not_found
338        #c_elapse = time.clock() - c_elapse
339        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
340        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
341        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
342        print "-"*40
343        print s
344        rel_found = float(t_found)/probe
345        rel_not_found = float(t_not_found)/probe
346        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
347        print estimated_total_time
348    ###)
349
350    security.declareProtected(ModifyPortalContent,'writeLog') ###(
351    def writeLog(self,logfile,s):
352        """write to the log file"""
353        logfile.write(s)
354
355###)
356
357    def generateStudentId(self,letter): ###(
358        import random
359        r = random
360        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
361        if letter == '?':
362            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
363        sid = "%c%d" % (letter,r.randint(99999,1000000))
364        students = self.portal_url.getPortalObject().campus.students
365##        while hasattr(students, sid):
366##            sid = "%c%d" % (letter,r.randint(99999,1000000))
367        while self.students_catalog(id = sid):
368            sid = "%c%d" % (letter,r.randint(99999,1000000))
369        return sid
370    ###)
371
372    def generatePassword(self,s=None): ###(
373        import random
374        r = random
375        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
376        if s is None:
377            s = 'abcdefghklmnpqrstuvwxy23456789'
378        pw = ''
379        while len(pw) < 6:
380            pw += r.choice(s)
381        return pw
382    ###)
383
384    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
385    def dumpSchoolfeePayments(self):
386        "dump paid schoolfees"
387        mtool = self.portal_membership
388        member = mtool.getAuthenticatedMember()
389        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
390        aq_student = self.students_catalog.evalAdvancedQuery
391        query = In('review_state',('schoolfee_paid',
392                                   'courses_registered',
393                                   'courses_validated',
394                                   ))
395        res = aq_student(query)
396        #import pdb;pdb.set_trace()
397        l = []
398        logger.info("start for %d" % len(res))
399        count = 1
400        log_after = 100
401        for student in res:
402            if not count % log_after:
403                logger.info("processed %d total %d" % (log_after,count))
404            count += 1
405            fee_dict =self.getSchoolFee(student)
406            fulltime = student.mode.endswith('_ft')
407            d = {}
408            d['student_id'] = student.id
409            d['name'] = student.name
410            d['amount'] = fee_dict.get(new_returning)
411            l += d,
412        csv_name = self.dumpListToCSV(l,'payments')
413        logger.info('%s dumped payments to %s' % (member,export_file))
414    ###)
415
416    security.declarePublic('dumpListToCSV') ###(
417    def dumpListToCSV(self,l,filename,fields=None):
418        """dump a list of dicts to a CSV file"""
419        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
420        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
421        if fields is None:
422            fields = l[0].keys()
423        headline = ','.join(fields)
424        out = open(export_file,"wb")
425        out.write(headline +'\n')
426        out.close()
427        out = open(export_file,"a")
428        csv_writer = csv.DictWriter(out,fields,)
429        csv_writer.writerows(l)
430        return export_file
431    ###)
432
433    security.declareProtected(ManagePortal, 'listMembers') ###(
434    def listMembers(self):
435        "list all members"
436        mtool = self.portal_membership
437        member = mtool.getAuthenticatedMember()
438        logger = logging.getLogger('WAeUPTool.listMembers')
439        if str(member) not in ('admin','joachim'):
440            logger.info('%s tried to list members' % (member))
441            return None
442        members = self.portal_directories.members
443        all = members.listEntryIdsAndTitles()
444        l = []
445        for user_id,name in all:
446            d = {}
447            d['user_id'] = user_id
448            d['name'] = name
449            d['pw'] = getattr(getattr(members,user_id),'password')
450            d['email'] = getattr(getattr(members,user_id),'email')
451            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
452            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
453            l += d,
454        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
455        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
456        logger.info('%s dumped memberlist to %s' % (member,export_file))
457        fields = l[0].keys()
458        headline = ','.join(fields)
459        out = open(export_file,"wb")
460        out.write(headline +'\n')
461        out.close()
462        out = open(export_file,"a")
463        csv_writer = csv.DictWriter(out,fields,)
464        csv_writer.writerows(l)
465    ###)
466
467    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
468    def removeDeletedDocIds(self, max=1000):
469        """
470        remove deleted docids from repository commit after max
471        """
472        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
473        repository = getToolByName(self, 'portal_repository')
474        pxtool = getToolByName(self, 'portal_proxies')
475        logger.info('start')
476        pxtool_infos = pxtool.getRevisionsUsed()
477        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
478
479        nb_revs = 0
480        docids_d = {} # all docids
481        unused_docids_d = {} # all docids that are unused
482        ids_unused_revs_docids = [] # ids for revs of unused docids
483        unused_ids = [] # ids for unused revs
484        total = 0
485        idlist = repository.objectIds()
486        to_delete = 0
487        found = False
488        for id in idlist:
489            docid, rev = repository._splitId(id)
490            if docid is None:
491                logger.info("invalid doc_id %s" % docid)
492                continue
493            nb_revs += 1
494            if not pxtool_infos.has_key(docid):
495                found = True
496                to_delete += 1
497                unused_ids.append(id)
498            elif not pxtool_infos[docid].has_key(rev):
499                found = True
500                to_delete += 1
501                unused_ids.append(id)
502            if found and not to_delete % max:
503                found = False
504                #import pdb;pdb.set_trace()
505                repository.manage_delObjects(unused_ids)
506                transaction.commit()
507                logger.info('removed %d total %d unused docids ' % (max,to_delete))
508        else:
509            if unused_ids:
510                repository.manage_delObjects(unused_ids)
511                transaction.commit()
512        logger.info('finished removing %d unused docids ' % (to_delete))
513
514
515###)
516
517    security.declareProtected(View,'getCredential') ###(
518    def getCredential(self,student_id):
519        student_entry = getattr(self.portal_directories.students,student_id,None)
520        if not self.isStaff():
521            mtool = self.portal_membership
522            member = mtool.getAuthenticatedMember()
523            logger = logging.getLogger('WAeUPTool.getCredential')
524            logger.info('%s tried to access password of %s' % (member,student_id))
525            return None
526        if student_entry is None:
527            return None
528        return getattr(student_entry,"password","not set")
529    ###)
530
531    security.declarePublic('checkPassword') ###(
532    def checkPassword(self,student_id,password):
533        student_entry = getattr(self.portal_directories.students,student_id,None)
534        if student_entry is None:
535            return False
536        return getattr(student_entry,"password","not set") == password
537    ###)
538
539    security.declarePublic('checkGenericPassword') ###(
540    def checkGenericPassword(self,member_id):
541        member_entry = getattr(self.portal_directories.members,member_id,None)
542        if member_entry is None:
543            return False
544        ltool = getToolByName(self, 'portal_layouts')
545        unsecure_words = ltool._getOb('members')['w__password'].check_words
546        password = getattr(member_entry,"password","not set")
547        is_unsecure = password in unsecure_words
548        if is_unsecure:
549            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
550            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
551        return is_unsecure
552    ###)
553
554    security.declareProtected(ModifyPortalContent,'editPassword') ###(
555    def editPassword(self,student_id,password):
556        "edit a student password"
557        student_entry = getattr(self.portal_directories.students,student_id,None)
558        if student_entry is None:
559            return
560        setattr(student_entry,'password',password)
561    ###)
562
563    security.declareProtected(ModifyPortalContent,'doCommit') ###(
564    def doCommit(self,logger=None):
565        "commit some transactions"
566        transaction.commit()
567    ###)
568
569    security.declarePublic('loadStudentFoto') ###(
570    def loadStudentFoto(self,student,filename,folder):
571        "return a student passport picture"
572        #import pdb;pdb.set_trace()
573        picture ="%s/import/%s/%s" % (i_home,folder,filename)
574        student_id = student.getId()
575        images_dir = getImagesDir(student_id)
576        if not os.path.exists(images_dir):
577            os.mkdir(images_dir)
578        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
579        for extension in ('.jpg','.JPG'):
580            fullname = "%(picture)s%(extension)s" % vars()
581            if os.path.exists(fullname):
582                copy2(fullname,image_name)
583                return "successfully copied passport picture"
584        return "passport picture not found: %s.jpg or .JPG" % picture
585    ###)
586
587    def old____loadStudentFoto(self,student,filename,folder): ###(
588        "return a student passport picture"
589        app = student.application
590        app_doc = app.getContent()
591        #clear = student.clearance
592        #clear_doc = clear.getContent()
593        #matric_no = clear_doc.matric_no.upper()
594        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
595        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
596        #import pdb;pdb.set_trace()
597        if os.path.exists(picture1):
598            file = open(picture1)
599        elif os.path.exists(picture2):
600            file = open(picture2)
601        else:
602            return "passport picture not found %s" % picture1
603        reopened = False
604        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
605            self.portal_workflow.doActionFor(app,'open')
606            reopened = True
607        outfile = file.read()
608        app_doc.manage_addFile('passport',
609                               file=outfile,
610                               title="%s.jpg" % filename)
611        if reopened:
612            self.portal_workflow.doActionFor(app,'close')
613        return "successfully loaded passport picture"
614    ###)
615
616    security.declareProtected(ModifyPortalContent,'createOne') ###(
617    def createOne(self,students_folder,student_brain,letter,commit=False):
618        sid = self.waeup_tool.generateStudentId(letter)
619        students_folder.invokeFactory('Student', sid)
620        student = getattr(students_folder,sid)
621        self.portal_workflow.doActionFor(student,'return')
622        student.manage_setLocalRoles(sid, ['Owner',])
623        matric_no = student_brain.matric_no
624        jamb_reg_no = student_brain.Entryregno
625        self.students_catalog.addRecord(id = sid,
626                                           matric_no = matric_no,
627                                           jamb_reg_no = jamb_reg_no,
628                                           sex = student_brain.Sex == "F",
629                                           name = "%s %s %s" % (student_brain.Firstname,
630                                                                student_brain.Middlename,
631                                                                student_brain.Lastname)
632                                        )
633        if commit:
634            transaction.commit()
635        return sid,jamb_reg_no
636    ###)
637
638    security.declareProtected(ModifyPortalContent,'addStudent') ###(
639    def addStudent(self,dict):
640        students_folder = self.portal_url.getPortalObject().campus.students
641        sid = self.waeup_tool.generateStudentId('?')
642        students_folder.invokeFactory('Student', sid)
643        student_obj = getattr(students_folder,sid)
644        f2t = self.student_field2types
645        #from pdb import set_trace; set_trace()
646        d = {}
647        #d['jamb_sex']  = 'M'
648        #if dict.get('sex'):
649        #    d['jamb_sex']  = 'F'
650
651        entry_session = dict.get('entry_session')
652        if entry_session == self.getSessionId()[0]:
653            wfaction = 'admit'
654            wft = 'wf_transition_admit'
655            password = None
656        else:
657            wfaction = 'return'
658            wft = 'wf_transition_return'
659            password = self.generatePassword()
660            self.makeStudentMember(sid,password)
661
662        for pt in f2t.keys():
663            student_obj.invokeFactory(pt,f2t[pt]['id'])
664            sub_obj = getattr(student_obj,f2t[pt]['id'])
665            sub_doc = sub_obj.getContent()
666            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
667            d['Title'] = f2t[pt]['title']
668            for field in f2t[pt]['fields']:
669                d[field] = dict.get(field,'')
670            sub_doc.edit(mapping = d)
671            new_state = f2t[pt][wft]
672            if new_state != "remain":
673                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
674        self.portal_workflow.doActionFor(student_obj,wfaction)
675        student_obj.manage_setLocalRoles(sid, ['Owner',])
676        return sid,password
677    ###)
678
679    security.declarePublic('getCertificateBrain') ###(
680    def getCertificateBrain(self,cert_id):
681        "do it"
682        res = ZCatalog.searchResults(self.portal_catalog_real,
683                                {'portal_type':"Certificate",
684                                      'id': cert_id})
685        if res:
686            return res[0]
687        return None
688    ###)
689
690    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
691    def get_csv_filenames(self):
692        "do it"
693        files = [file for file in os.listdir("%s/import/" % (i_home))
694                 if file.endswith('.csv') and file.find('imported') == -1]
695        return files
696    ###)
697
698    security.declarePublic('findStudentByMatricelNo') ###(
699    def findStudentByMatricelNo(self,matric_no):
700        "do it"
701        res = ZCatalog.searchResults(self.portal_catalog_real,
702                                {'portal_type':"StudentClearance",
703                                 'SearchableText': matric_no})
704        if res:
705            return res[0]
706        return None
707    ###)
708
709    security.declarePublic('makeStudentMember') ###(
710    def makeStudentMember(self,sid,password='uNsEt'):
711        """make the student a member"""
712        membership = self.portal_membership
713        membership.addMember(sid,
714                             password ,
715                             roles=('Member',
716                                     'Student',
717                                     ),
718                             domains='',
719                             properties = {'memberareaCreationFlag': False,
720                                           'homeless': True},)
721        member = membership.getMemberById(sid)
722        self.portal_registration.afterAdd(member, sid, password, None)
723        #self.manage_setLocalRoles(sid, ['Owner',])
724    ###)
725
726    security.declareProtected(View,'makeStudentData') ###(
727    def makeStudentData(self,student_id,email=None,phone_nr=None):
728        "create Datastructure for a returning Student"
729        #import pdb;pdb.set_trace()
730        logger = logging.getLogger('WAeUPTool.makeStudentData')
731        students_folder = self.portal_url.getPortalObject().campus.students
732        #res = self.students_catalog(id=student_id)
733        #if res:
734        #    st = res[0]
735        #res = self.returning_import(matric_no = st.matric_no)
736        res = self.returning_import(id = student_id)
737        if res:
738            student = res[0]
739        else:
740            logger.info('Id %s not found in returning_import' % student_id)
741            return
742        logger.info('%s creates data structure' % student_id)
743        s_results = self.results_import(matric_no = student.matric_no)
744        if s_results:
745            lnr = self.getLevelFromResultsCosCode(s_results)
746            level = "%d00" % lnr
747            verdict,eligible = self.getVerdict(s_results[0].Verdict)
748            #if eligible:
749            #    level = "%d00" % (lnr + 1)
750        else:
751            logger.info('matric_no %s not found in results_import' % student.matric_no)
752            level = ''
753            verdict = ''
754        #student should not be allowed to perform this transition
755        #wftool = self.portal_workflow
756        #wftool.doActionFor(student,'return')
757        certcode_org = student.Coursemajorcode
758        certcode = makeCertificateCode(certcode_org)
759        certificate_brain = self.getCertificateBrain(certcode)
760        if not certificate_brain:
761            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
762            logger.info(em)
763        matric_no = student.matric_no
764        sid = student_id
765        student_obj = getattr(students_folder,sid)
766        student_obj.invokeFactory('StudentApplication','application')
767        application = student_obj.application
768        self.portal_workflow.doActionFor(application,'open',dest_container=application)
769        da = {'Title': 'Application Data'}
770        student_obj.invokeFactory('StudentPersonal','personal')
771        da['jamb_reg_no'] = student.Entryregno
772        em = self.getEntryMode(student.Entryregno)
773        da['entry_mode'] = em
774        personal = student_obj.personal
775        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
776        dp = {'Title': 'Personal Data'}
777        student_obj.invokeFactory('StudentClearance','clearance')
778        clearance = student_obj.clearance
779        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
780        dc = {'Title': 'Clearance/Eligibility Record'}
781        dc['matric_no'] = matric_no
782        state = student.State
783        lga = student.LGA
784        if state and lga:
785            lga =  state + ' / ' + lga
786        else:
787            lga = "None"
788        da['jamb_lga'] = dc['lga'] = lga
789        da['app_email'] = dp['email'] = email
790        da['app_mobile'] = dp['phone'] = phone_nr
791        dp['firstname'] = student.Firstname
792        dp['middlename'] = student.Middlename
793        dp['lastname'] = student.Lastname
794        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
795        da['jamb_sex'] = student.Sex
796        dp['sex'] = student.Sex == 'F'
797        dp['perm_address'] = student.Permanent_Address
798        application.getContent().edit(mapping=da)
799        self.portal_workflow.doActionFor(application,'close',dest_container=application)
800        personal.getContent().edit(mapping=dp)
801        clearance.getContent().edit(mapping=dc)
802        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
803        #
804        # Study Course
805        #
806        student_obj.invokeFactory('StudentStudyCourse','study_course')
807        studycourse = student_obj.study_course
808        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
809        dsc = {}
810        dsc['study_course'] = certcode
811        dsc['current_level'] = level
812        dsc['current_verdict'] = verdict
813        dsc['current_mode'] = em
814        dsc['current_session'] = '05'
815        studycourse.getContent().edit(mapping=dsc)
816        #
817        # Level
818        #
819##        l = getattr(studycourse,level,None)
820##        if l is None:
821##            studycourse.invokeFactory('StudentStudyLevel', level)
822##            l = getattr(studycourse, level)
823##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
824##            l.getContent().edit(mapping={'Title': "Level %s" % level})
825###)
826
827    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
828    def admitOneStudent(self,brain,entry_session,pin_password):
829        "create Datastructure for an admitted Student"
830        #import pdb;pdb.set_trace()
831        if not hasattr(self,"_v_certificates"):
832            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
833        students_folder = self.portal_url.getPortalObject().campus.students
834        logger = logging.getLogger('WAeUPTool.admitOneStudent')
835        if brain.status != "admitted":
836            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
837            return
838        pin_parts = brain.pin.split('-')
839        if pin_parts and len(pin_parts) != 3:
840            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
841            return
842        if brain.course_admitted not in self._v_certificates:
843            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
844            return
845        if brain.sex not in (True,False):
846            logger.info('sex of %s not available' % (brain.reg_no))
847            return
848        student_id = self.generateStudentId('?')
849        students_folder.invokeFactory('Student', student_id)
850        student_object = getattr(students_folder,student_id)
851        if pin_password:
852            password = pin_parts[2]
853            self.makeStudentMember(student_id,password = password)
854        student_object.manage_setLocalRoles(student_id, ['Owner',])
855        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
856        #
857        # application
858        #
859        student_object.invokeFactory('StudentApplication','application')
860        application = student_object.application
861        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
862        da = {'Title': 'Application Data'}
863        da['jamb_reg_no'] = brain.reg_no
864
865        sex = 'M'
866        if brain.sex:
867            sex = 'F'
868        da['jamb_sex'] = sex
869        da['jamb_age'] = brain.jamb_age
870        #da['app_ac_pin'] = brain.pin
871        da['jamb_lga'] = brain.jamb_lga
872        da['jamb_state'] = brain.jamb_state
873        da['jamb_score'] = brain.aggregate
874        da['app_email'] = brain.email
875        da['app_mobile'] = brain.phone
876        if brain.entry_mode:
877            da['entry_mode'] = brain.entry_mode
878        elif brain.screening_type == 'pume':
879            da['entry_mode'] = 'ume_ft'
880        elif brain.screening_type == 'pde':
881            da['entry_mode'] = 'de_ft'
882        elif brain.screening_type == 'pce':
883            da['entry_mode'] = 'pce'
884        elif brain.screening_type == 'prence':
885            da['entry_mode'] = 'prence'
886        else:
887            da['entry_mode'] = ''
888        da['entry_session'] = entry_session
889        da['jamb_lastname'] = brain.lastname
890        da['jamb_middlename'] = brain.middlenames   # different field names!
891        da['jamb_firstname'] = brain.firstname
892        da['screening_application_date'] = brain.application_date
893        da['date_of_birth'] = brain.date_of_birth
894        da['jamb_first_cos'] = brain.course1
895        da['jamb_second_cos'] = brain.course2
896        da['course3'] = brain.course3
897        da['screening_type'] = brain.screening_type
898        da['screening_score'] = brain.screening_score
899        da['screening_date'] = brain.screening_date
900        da['hq_type'] = brain.hq_type
901        da['hq_grade'] = brain.hq_grade
902        da['aos'] = brain.aos
903
904        application.getContent().edit(mapping=da)
905        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
906        #
907        # personal
908        #
909        student_object.invokeFactory('StudentPersonal','personal')
910        personal = student_object.personal
911        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
912        dp = {'Title': 'Personal Data'}
913        dp['sex'] = brain.sex
914        dp['email'] = brain.email
915        dp['phone'] = brain.phone
916        dp['lastname'] = brain.lastname
917        dp['middlename'] = brain.middlenames   # different field names!
918        dp['firstname'] = brain.firstname
919        personal.getContent().edit(mapping=dp)
920        #
921        # clearance
922        #
923        student_object.invokeFactory('StudentClearance','clearance')
924        clearance = student_object.clearance
925        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
926        dc = {'Title': 'Clearance/Eligibility Record'}
927        dc['lga'] = brain.lga
928        dc['birthday'] = brain.date_of_birth
929        clearance.getContent().edit(mapping=dc)
930        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
931        #
932        # study Course
933        #
934        student_object.invokeFactory('StudentStudyCourse','study_course')
935        studycourse = student_object.study_course
936        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
937        dsc = {}
938        dsc['study_course'] = brain.course_admitted
939        dsc['current_verdict'] = ''
940        dsc['current_mode'] = da['entry_mode']
941        if da['entry_mode'].startswith('de'):
942            dsc['current_level'] = '200'
943        else:
944            dsc['current_level'] = '100'
945        dsc['current_session'] = entry_session
946        studycourse.getContent().edit(mapping=dsc)
947        #
948        # payments folder
949        student_object.invokeFactory('PaymentsFolder','payments')
950        payments = getattr(student_object,'payments')
951        dpay = {}
952        dpay['Title'] = 'Payments'
953        payments.getContent().edit(mapping=dpay)
954        self.portal_workflow.doActionFor(payments,'open')
955        #
956        # passport foto
957        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
958                                                              brain.screening_type,
959                                                              brain.reg_no)
960        images_dir = getImagesDir(student_id)
961        #images_dir = os.path.join("%s" % images_base,student_id)
962        letter_dir,student_dir = os.path.split(images_dir)
963        if not os.path.exists(letter_dir):
964            os.mkdir(letter_dir)
965        if not os.path.exists(images_dir):
966            os.mkdir(images_dir)
967        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
968        if os.path.exists(app_picture):
969            copy2(app_picture,image_name)
970        else:
971            logger.info('passport of %s/%s not found: %s' % (student_id,
972                                                             brain.reg_no,
973                                                             app_picture))
974
975        return student_id
976    ###)
977
978    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
979    def makeStudentLevel(self,student_id):
980        "create the StudyLevel for a returning Student"
981        #import pdb;pdb.set_trace()
982        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
983        students_folder = self.portal_url.getPortalObject().campus.students
984        res = self.students_catalog(id=student_id)
985        if res:
986            st = res[0]
987        course = st.course
988        matric_no = st.matric_no
989        level = st.level
990        res = self.results_import(matric_no = matric_no)
991        if res:
992            results = res
993        logger.info('%s creating Level %s' % (student_id,level))
994        #
995        # Level
996        #
997        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
998        studycourse = getattr(student_obj,"study_course",None)
999        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1000        l = getattr(studycourse,level,None)
1001        if l is None:
1002            studycourse.invokeFactory('StudentStudyLevel', level)
1003            l = getattr(studycourse, level)
1004            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1005            l.getContent().edit(mapping={'Title': "Level %s" % level})
1006        ###)
1007
1008    security.declarePublic('getHallInfo') ###(
1009    def getHallInfo(self,bed):
1010        """return Hall Info"""
1011        info = {}
1012        hall,block,room,letter = bed.split('_')
1013        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1014        if res and len(res) == 1:
1015            hall_brain = res[0]
1016            hall_doc = hall_brain.getObject().getContent()
1017        else:
1018            return info
1019        info['hall_title'] = hall_brain.Title
1020        info['maintenance_code'] = hall_doc.maintenance_code
1021        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1022        batch_doc = None
1023        for brain in res:
1024            if brain.id.startswith(info['maintenance_code']):
1025                batch_doc = brain.getObject().getContent()
1026                break
1027        if batch_doc is None:
1028            info['maintenance_fee'] = ''
1029        else:
1030            info['maintenance_fee'] = batch_doc.cost
1031        return info
1032    ###)
1033
1034    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1035    def removePictureFolder(self,student_id):
1036        """remove picture_folder by renaming it"""
1037        path = 'images'
1038        picture_path = os.path.join(i_home,path,student_id)
1039        if not os.path.exists(picture_path):
1040            return False
1041        os.rename(picture_path,picture_path + "_removed")
1042        return True
1043    ###)
1044
1045    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1046    def restorePictureFolder(self,student_id):
1047        """restore picture_folder by renaming it"""
1048        path = 'images'
1049        picture_path = os.path.join(i_home,path,student_id)
1050        if not os.path.exists(picture_path + "_removed"):
1051            return False
1052        os.rename(picture_path + "_removed",picture_path)
1053        return True
1054    ###)
1055
1056    security.declarePublic('picturesExist') ###(
1057    def picturesExist(self, ids,student_id=None):
1058        """check if pictures exist in the filesystem"""
1059        if student_id is None:
1060            student_id = self.getStudentId()
1061        if student_id is None:
1062            return False
1063        picture_path = getImagesDir(student_id)
1064        #picture_path = os.path.join(images_base,student_id)
1065        if not os.path.exists(picture_path):
1066            return False
1067        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1068        return set(ids).issubset(set(pictures))
1069    ###)
1070
1071    security.declarePublic('picturesList') ###(
1072    def picturesList(self):
1073        """check if pictures exist in the filesystem"""
1074        path = 'images'
1075        student_id = self.getStudentId()
1076        #picture_path = os.path.join(i_home,path,student_id)
1077        picture_path = getImagesDir(student_id)
1078        if not os.path.exists(picture_path):
1079            return []
1080        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1081    ###)
1082
1083    security.declarePublic('showFsPicture') ###(
1084    def showFsPicture(self,path):
1085        """return a picture from the filesystem"""
1086        #picture_path = os.path.join(i_home,path)
1087        picture_path = os.path.join(images_base,path)
1088        response = self.REQUEST.RESPONSE
1089        #import pdb;pdb.set_trace()
1090        registry = getToolByName(self, 'mimetypes_registry')
1091        mimetype = str(registry.lookupExtension(path.lower()) or
1092                    registry.lookupExtension('file.bin'))
1093        if os.path.exists(picture_path):
1094            response.setHeader('Content-type',mimetype)
1095            return open(picture_path).read()
1096        picture_path = os.path.join(i_home,'import',path)
1097        if os.path.exists(picture_path):
1098            return open(picture_path).read()
1099    ###)
1100
1101    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1102    def deleteAllCourses(self,department="All"):
1103        ''' delete the courses'''
1104        pm = self.portal_membership
1105        member = pm.getAuthenticatedMember()
1106
1107        if str(member) not in ("henrik","joachim"):
1108            return "not possible"
1109        if department == "All":
1110            res = self.portal_catalog({'meta_type': 'Department'})
1111        if len(res) < 1:
1112            return "No Departments found"
1113
1114        deleted = []
1115        for dep in res:
1116            cf = dep.getObject().courses
1117            if cf:
1118                cf.manage_delObjects(ids=cf.objectIds())
1119                deleted.append("deleted Courses in %s" % dep.getId)
1120        return "\r".join(deleted)
1121    ###)
1122
1123    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1124    def getLogfileLines(self,filename="event.log",numlines=20):
1125        """Get last NUMLINES lines of logfile FILENAME.
1126
1127        Return last lines' of a file in the instances logfile directory as
1128        a list. The number of returned lines equals `numlines' or less. If
1129        less than `numlines' lines are available, the whole file ist
1130        returned. If the file can not be opened or some other error
1131        occurs, empty list is returend.
1132        """
1133        result = []
1134        lines_hit = 0
1135
1136        # We only handle files in instances' log directory...
1137        logpath = os.path.join(i_home, "log")
1138        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1139        if not filename.startswith( logpath ):
1140            # Attempt to access file outside log-dir...
1141            return []
1142
1143        try:
1144            fd = file( filename, "rb" )
1145        except IOError:
1146            return []
1147        if not fd:
1148            return []
1149
1150        if os.linesep == None:
1151            linesep = '\n'
1152        else:
1153            linesep = os.linesep
1154
1155        # Try to find 'numlines' times a lineseparator, searching from end
1156        # and moving to the beginning of file...
1157        fd.seek( 0, 2) # Move to end of file...
1158        while lines_hit < numlines:
1159            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1160                                          # one step forward...
1161                lines_hit += 1
1162            try:
1163                fd.seek( -2, 1) # Go two bytes back from current pos...
1164            except IOError:
1165                # We cannot go back two bytes. Maybe the file is too small...
1166                break
1167        fd.seek(2,1)
1168
1169        # Read all lines from current position...
1170        result = fd.readlines()
1171        # Remove line endings...
1172        result = [x.strip() for x in result]
1173        fd.close()
1174        return result
1175    ###)
1176
1177    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1178    def getCallbacksFromLog(self,filename):
1179        """fix Online Payment Transactions from Z2.log entries"""
1180        import transaction
1181        import random
1182        from cgi import parse_qs
1183        from urlparse import urlparse
1184        #from pdb import set_trace
1185        wftool = self.portal_workflow
1186        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1187        students_folder = self.portal_url.getPortalObject().campus.students
1188        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1189        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1190        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1191        data = re.compile(s)
1192        start = True
1193        tr_count = 1
1194        total = 0
1195        #name = 'pume_results'
1196        #name = 'epaymentsuccessful_z2log2'
1197        name = filename
1198        no_import = []
1199        imported = []
1200        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1201        try:
1202            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1203        except:
1204            logger.error('Error reading %s' % name)
1205            return
1206        tas = []
1207        for line in transactions:
1208            dict = {}
1209            items = data.search(line)
1210            dict['idict'] = idict = items.groupdict()
1211            #print idict
1212            #from pdb import set_trace;set_trace()
1213            urlparsed = urlparse(idict['get'][4:])
1214            #print urlparsed
1215            path = urlparsed[2].split('/')
1216            dict['student_id'] = student_id = path[8]
1217            dict['payment_id'] = payment_id = path[10]
1218            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1219            tas.append(dict)
1220            tr_count += 1
1221        return tas
1222    ###)
1223
1224    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1225    def importOnlinePaymentTransactions(self):
1226        """load Online Payment Transactions from CSV values"""
1227        import transaction
1228        import random
1229        #from pdb import set_trace
1230        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1231        opt = self.online_payments_import
1232        students_folder = self.portal_url.getPortalObject().campus.students
1233        start = True
1234        tr_count = 1
1235        total = 0
1236        #name = 'pume_results'
1237        name = 'OnlineTransactions'
1238        no_import = []
1239        imported = []
1240        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1241        try:
1242            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1243        except:
1244            logger.error('Error reading %s.csv' % name)
1245            return
1246        for pay_transaction in transactions:
1247            if start:
1248                start = False
1249                logger.info('Start loading from %s.csv' % name)
1250                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1251                no_import.append('%s,"Error"' % s)
1252                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1253                format_error = format + ',"%(Error)s"'
1254            data = {}
1255
1256            # format of the first file sent by Tayo
1257            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1258            #data['student_id'] = student_id = pay_transaction['Payer ID']
1259            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1260            #data['response_code'] = response_code = pay_transaction['Resp Code']
1261            #data['amount'] = amount = pay_transaction['Amount']
1262
1263            # format of the second file sent by Tayo
1264            #data['datetime'] = date = 0
1265            #data['student_id'] = student_id = pay_transaction['Payer ID']
1266            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1267            #data['response_code'] = response_code = '00'
1268            #data['amount'] = amount = pay_transaction['Amount']
1269
1270            # format of the third file sent by Kehinde
1271            data['datetime'] = date = 0
1272            data['student_id'] = student_id = pay_transaction['customer_id']
1273            data['order_id'] = order_id = pay_transaction['merchant_reference']
1274            data['response_code'] = response_code = '00'
1275            data['amount'] = amount = pay_transaction['Amount']
1276
1277            dup = False
1278            if response_code == "12":
1279                continue
1280            try:
1281                opt.addRecord(**data)
1282            except ValueError:
1283                dup = True
1284            #from pdb import set_trace;set_trace()
1285            if dup:
1286                if response_code == "00":
1287                    try:
1288                        opt.modifyRecord(**data)
1289                    except:
1290                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1291                        continue
1292                else:
1293                    pay_transaction['Error'] = "Duplicate order_id"
1294                    no_import.append( format_error % pay_transaction)
1295                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1296                    continue
1297            tr_count += 1
1298            if tr_count > 1000:
1299                if len(no_import) > 0:
1300                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1301                             '\n'.join(no_import) + '\n')
1302                    no_import = []
1303                em = '%d transactions committed\n' % (tr_count)
1304                transaction.commit()
1305                regs = []
1306                logger.info(em)
1307                total += tr_count
1308                tr_count = 0
1309        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1310                                                '\n'.join(no_import))
1311        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1312    ###)
1313
1314    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1315    def mass_create_faculty(self,mapping):
1316        "create a faculty"
1317        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1318        academics_folder = self.portal_url.getPortalObject().campus.academics
1319        fid = mapping['code']
1320        if getattr(academics_folder,fid,None) is not None:
1321            return '', "Faculty with ID: %s exists" % fid
1322        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1323        try:
1324            academics_folder.invokeFactory('Faculty', fid)
1325        except BadRequest,E:
1326            return '', "%s" % E
1327        f = getattr(academics_folder,fid,None)
1328        f.getContent().edit(mapping=mapping)
1329        return fid,''
1330    ###)
1331
1332    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1333    def mass_edit_faculty(self,mapping):
1334        "edit a faculty"
1335        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1336        academics_folder = self.portal_url.getPortalObject().campus.academics
1337        fid = mapping['code']
1338        f = getattr(academics_folder,fid,None)
1339        if f is None:
1340            return '', "Faculty with ID: %s does not exist" % fid
1341        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1342        f.getContent().edit(mapping=mapping)
1343        return fid,''
1344    ###)
1345
1346    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1347    def mass_create_department(self,mapping):
1348        "create a department in the correct faculty"
1349        logger = logging.getLogger('WAeUPTool.mass_create_department')
1350        fid = mapping['faculty_code']
1351        if getattr(self,'_v_faculties',None) is None:
1352            res = self.portal_catalog(portal_type = "Faculty")
1353            self._v_faculties = {}
1354            for f in res:
1355                self._v_faculties[f.getId] = f.getObject()
1356        f = self._v_faculties.get(fid,None)
1357        if f is None:
1358            return '', "No Faculty with ID: %s" % fid
1359        else:
1360            did = mapping.get('code')
1361            d = getattr(f,did,None)
1362            if d is None or d.portal_type == "Faculty":
1363                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1364                try:
1365                    f.invokeFactory('Department', did)
1366                except BadRequest,E:
1367                    return '', "%s" % E
1368                d = getattr(f,did)
1369                d.invokeFactory('CoursesFolder','courses')
1370                courses = getattr(d,'courses')
1371                dict = {'Title': 'Courses'}
1372                courses.getContent().edit(mapping=dict)
1373                d.invokeFactory('CertificatesFolder','certificates')
1374                certificates = getattr(d,'certificates')
1375                dict = {'Title': 'Certificates'}
1376                certificates.getContent().edit(mapping=dict)
1377            d.getContent().edit(mapping=mapping)
1378        return did,''
1379    ###)
1380
1381    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1382    def mass_edit_department(self,mapping):
1383        "create a department in the correct faculty"
1384        logger = logging.getLogger('WAeUPTool.mass_create_department')
1385        academics_folder = self.portal_url.getPortalObject().campus.academics
1386        fid = mapping['faculty_code']
1387        did = mapping.get('code')
1388        try:
1389            d = getattr(getattr(academics_folder,fid),did,None)
1390        except KeyError:
1391            return '', "Department %s or Faculty %s wrong" % (did,fid)
1392        else:
1393            if d is None or d.portal_type == "Faculty":
1394                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1395            d.getContent().edit(mapping=mapping)
1396        return did,''
1397    ###)
1398
1399    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1400    def mass_create_course(self,mapping):
1401        #import pdb;pdb.set_trace()
1402        if getattr(self,'_v_course_list',None) is None:
1403            self._v_course_list = []
1404        if getattr(self,'_v_departments',None) is None:
1405            res = self.portal_catalog(portal_type = "Department")
1406            self._v_department_courses = {}
1407            for d in res:
1408                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1409        did = mapping['department_code']
1410        d = self._v_department_courses.get(did,None)
1411        if d is None:
1412            return '', "No Department with ID: %s" % did
1413        course_id = mapping.get('code')
1414        if course_id in self._v_course_list:
1415            return '', "Duplicate Course ID: %s" % did
1416        c = getattr(d,course_id,None)
1417        if c is not None:
1418            return '', "Duplicate Course ID: %s" % did
1419        try:
1420            d.invokeFactory('Course', course_id)
1421        except BadRequest,E:
1422            return '', "%s" % E
1423        self._v_course_list.append(course_id)
1424        c = getattr(d,course_id)
1425        c.getContent().edit(mapping=mapping)
1426        return course_id,''
1427    ###)
1428
1429    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1430    def mass_edit_course(self,mapping):
1431        #import pdb;pdb.set_trace()
1432        course_id = mapping.get('code')
1433        res = self.portal_catalog(id=course_id)
1434        if not res:
1435            return '', "No Course with ID: %s" % course_id
1436        c = res[0].getObject()
1437        c.getContent().edit(mapping=mapping)
1438        return course_id,''
1439    ###)
1440
1441    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1442    def mass_create_certificate(self,mapping):
1443        if getattr(self,'_v_certificate_list',None) is None:
1444            self._v_certificate_list = []
1445        if getattr(self,'_v_department_certificates',None) is None:
1446            res = self.portal_catalog(portal_type = "Department")
1447            self._v_department_certificates = {}
1448            for d in res:
1449                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1450        did = mapping['department_code']
1451        d = self._v_department_certificates.get(did,None)
1452        if d is None:
1453            return '', "No Department with ID: %s" % did
1454        certificate_id = mapping.get('code')
1455        if certificate_id in self._v_certificate_list:
1456            return '', "Duplicate Certificate ID: %s" % did
1457        c = getattr(d,certificate_id,None)
1458        if c is not None:
1459            return '', "Duplicate Certificate ID: %s" % did
1460        try:
1461            d.invokeFactory('Certificate', certificate_id)
1462        except BadRequest,E:
1463            return '', "%s" % E
1464        self._v_certificate_list.append(certificate_id)
1465        c = getattr(d,certificate_id)
1466        c.getContent().edit(mapping=mapping)
1467        return certificate_id,''
1468    ###)
1469
1470    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1471    def mass_edit_certificate(self,mapping):
1472        #import pdb;pdb.set_trace()
1473        certificate_id = mapping.get('code')
1474        res = self.portal_catalog(id=certificate_id)
1475        if not res:
1476            return '', "No Certificate with ID: %s" % did
1477        c = res[0].getObject()
1478        c.getContent().edit(mapping=mapping)
1479        return certificate_id,''
1480    ###)
1481
1482    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1483    def mass_create_application(self,mapping):
1484        #import pdb;pdb.set_trace()
1485        reg_no = mapping.get('reg_no')
1486        try:
1487            self.applicants_catalog.addRecord(**mapping)
1488        except ValueError:
1489            return '', "applicant record with reg_no %s already exists" % reg_no
1490        return reg_no,''
1491    ###)
1492
1493    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1494    def mass_edit_application(self,mapping):
1495        #import pdb;pdb.set_trace()
1496        reg_no = mapping.get('reg_no')
1497        status = mapping.get('status')
1498        res = self.applicants_catalog(reg_no = reg_no)
1499        if len(res):
1500            if res[0].status == 'created' and status != 'created':
1501                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1502            elif status == 'created' and res[0].status != 'created':
1503                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1504            else:
1505                self.applicants_catalog.modifyRecord(**mapping)
1506                return reg_no,''
1507        else:
1508            return '', "applicant record with reg_no %s does not exist" % reg_no
1509
1510    ###)
1511
1512    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1513    def mass_create_course_result(self,mapping):
1514        #import pdb;pdb.set_trace()
1515        students_folder = self.portal_url.getPortalObject().campus.students
1516        if getattr(self,'_v_courses',None) is None:
1517            res = self.courses_catalog()
1518            self._v_courses = {}
1519            for brain in res:
1520                self._v_courses[brain.code] = brain
1521        if getattr(self,'_v_level_created',None) is None:
1522            self._v_level_created = []
1523        course_id = mapping.get('code')
1524        if course_id not in self._v_courses.keys():
1525            return '', "No course with ID: %s" % course_id
1526        id_key = ''
1527        for id_key in ('student_id','matric_no'):
1528            id_field = mapping.get(id_key,None)
1529            if id_field is not None:
1530                student_id = id_field
1531                break
1532        query = Eq(id_key,id_field)
1533        res = self.students_catalog.evalAdvancedQuery(query)
1534        if not res:
1535            return '', "no student with %(id_key)s %(id_field)s" % vars()
1536        student_rec = res[0]
1537        if id_field != "student_id":
1538            mapping['student_id'] = student_rec.id
1539        student_id = student_rec.id
1540        level_id = mapping['level_id']
1541        code = mapping['code']
1542        if student_id not in self._v_level_created:
1543            try:
1544                context = getattr(getattr(students_folder,
1545                                          "%(student_id)s" % vars()),
1546                                  'study_course')
1547            except:
1548                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1549            if level_id not in context.objectIds():
1550                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1551                level = getattr(context,"%(level_id)s" % vars())
1552                self.portal_workflow.doActionFor(level,'open')
1553                # the session string must not be copied into the level object
1554                current_verdict = getattr(student_rec,'current_verdict','')
1555                current_session = getattr(student_rec,'current_session','')
1556                if current_verdict and student_rec.current_level == level_id:
1557                    level.getContent().edit(mapping={'verdict': "%s" %
1558                                                     current_verdict,
1559                                                     'session': "%s" %
1560                                                     current_session,
1561                                                    })
1562                    self.portal_workflow.doActionFor(level,'close')
1563            self._v_level_created += student_id,
1564        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1565        for k in ('semester','credits',):
1566            mapping[k] = getattr(self._v_courses[course_id],k)
1567        try:
1568            self.course_results.addRecord(**mapping)
1569        except ValueError:
1570            return '', "course result already exists: %s" % key
1571        return key,''
1572    ###)
1573
1574    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1575    def mass_edit_course_result(self,mapping):
1576        #import pdb;pdb.set_trace()
1577        # if getattr(self,'_v_courses',None) is None:
1578        #     res = self.courses_catalog()
1579        #     self._v_courses = {}
1580        #     for brain in res:
1581        #         self._v_courses[brain.code] = brain
1582        # course_id = mapping.get('code')
1583        # if course_id not in self._v_courses.keys():
1584        #     return '', "No course with ID: %s" % did
1585        id_key = ''
1586        for id_key in ('student_id','matric_no'):
1587            id_field = mapping.get(id_key,None)
1588            if id_field is not None:
1589                student_id = id_field
1590                break
1591        query = Eq(id_key,id_field)
1592        res = self.students_catalog.evalAdvancedQuery(query)
1593        if not res:
1594            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1595        if id_field != "student_id":
1596            mapping['student_id'] = res[0].id
1597        if not res:
1598            return '', "no student with %(id_key)s %(id_field)s" % vars()
1599        student_rec = res[0]
1600        student_id = student_rec.id
1601        level_id = mapping['level_id']
1602        code = mapping['code']
1603        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1604        # for k in ('semester',):
1605        #     mapping[k] = getattr(self._v_courses[course_id],k)
1606        try:
1607            self.course_results.modifyRecord(**mapping)
1608        except KeyError:
1609            return '', "No course result to edit: %s" % key
1610        return key,''
1611    ###)
1612
1613    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1614    def mass_create_certificate_course(self,mapping):
1615        if getattr(self,'_v_courses',None) is None:
1616            res = self.courses_catalog()
1617            self._v_courses= [course.code for course in res]
1618        if getattr(self,'_v_ceritficates',None) is None:
1619            res = self.portal_catalog(portal_type = "Certificate")
1620            self._v_certificates = {}
1621            for cert in res:
1622                self._v_certificates[cert.getId] = cert.getObject()
1623        certificate_course_id = mapping.get('code')
1624        if certificate_course_id not in self._v_courses:
1625            return '', "No Course with ID: %s" % certificate_course_id
1626        cert_id = mapping['certificate_code']
1627        cert = self._v_certificates.get(cert_id,None)
1628        if cert is None:
1629            return '', "No Certificate with ID: %s" % cert_id
1630        level_id = mapping.get('level')
1631        level = getattr(cert,level_id,None)
1632        if level is None:
1633            cert.invokeFactory('StudyLevel', level_id)
1634            level = getattr(cert,level_id,None)
1635        elif hasattr(level,certificate_course_id):
1636            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1637            (certificate_course_id,cert_id,level_id)
1638        level.invokeFactory('CertificateCourse', certificate_course_id)
1639        c = getattr(level,certificate_course_id)
1640        c.getContent().edit(mapping=mapping)
1641        return certificate_course_id,''
1642    ###)
1643
1644    field2types_student = {   ###(
1645                      'StudentApplication':
1646                          {'id': 'application',
1647                           'title': 'Application Data',
1648                           'wf_transition_return': 'close',
1649                           'wf_transition_admit': 'remain',
1650                           'fields':
1651                             ('jamb_reg_no',
1652                              'entry_mode',
1653                              'entry_session',
1654                              'jamb_score',
1655                              'app_email',
1656                              'jamb_age',
1657                              'jamb_state',
1658                              'jamb_lga',
1659                              'jamb_sex',
1660                              )
1661                              },
1662                      #'StudentPume':
1663                      #    {'id': 'pume',
1664                      #     'title': 'Pume Data',
1665                      #     'wf_transition_return': 'close',
1666                      #     'wf_transition_admit': 'close',
1667                      #     'fields':
1668                      #       ('pume_score',
1669                      #        )
1670                      #        },
1671                      'StudentClearance':
1672                          {'id': 'clearance',
1673                           'title': 'Clearance/Eligibility Record',
1674                           'wf_transition_return': 'close',
1675                           'wf_transition_admit': 'remain',
1676                           'fields':
1677                             ('matric_no',
1678                              'nationality',
1679                              'lga',
1680                              'birthday',
1681                              )
1682                              },
1683                         'StudentPersonal':
1684                          {'id': 'personal',
1685                           'title': 'Personal Data',
1686                           'wf_transition_return': 'open',
1687                           'wf_transition_admit': 'remain',
1688                           'fields':
1689                             ('firstname',
1690                              'middlename',
1691                              'lastname',
1692                              'sex',
1693                              'email',
1694                              'phone',
1695                              'perm_address',
1696                              )
1697                              },
1698                         'StudentStudyCourse':
1699                          {'id': 'study_course',
1700                           'title': 'Study Course',
1701                           'wf_transition_return': 'open',
1702                           'wf_transition_admit': 'remain',
1703                           'fields':
1704                             ('study_course',
1705                              'current_level',
1706                              'current_session',
1707                              'current_mode',
1708                              'current_verdict',
1709                              'previous_verdict',
1710                              )
1711                              },
1712                         # 'StudentStudyLevel':
1713                         #  {'id': 'current_level',
1714                         #   'title': '',
1715                         #   'wf_transition_return': 'open',
1716                         #   'wf_transition_admit': 'remain',
1717                         #   'fields':
1718                         #     ('verdict',
1719                         #      'session',
1720                         #      )
1721                         #      },
1722                         'PaymentsFolder':
1723                          {'id': 'payments',
1724                           'title': 'Payments',
1725                           'wf_transition_return': 'open',
1726                           'wf_transition_admit': 'open',
1727                           'fields':
1728                             ()
1729                              },
1730                         }
1731    ###)
1732
1733    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1734    def mass_create_student(self,mapping):
1735        "create a students record due import"
1736        logger = logging.getLogger('WAeUPTool.mass_create_student')
1737        students_folder = self.portal_url.getPortalObject().campus.students
1738        jamb_reg_no = mapping.get('jamb_reg_no',None)
1739        if jamb_reg_no:
1740            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1741            if res:
1742                return '',"jamb_reg_no exists"
1743        matric_no = mapping.get('matric_no',None)
1744        if matric_no:
1745            res = self.students_catalog(matric_no = matric_no)
1746            if res:
1747                return '',"matric_no exists"
1748        sid = self.waeup_tool.generateStudentId('?')
1749        students_folder.invokeFactory('Student', sid)
1750        student_obj = getattr(students_folder,sid)
1751        f2t = self.field2types_student
1752        d = {}
1753        transition = mapping.get('reg_transition','admit')
1754        if transition not in ('admit','return'):
1755            return '',"no valid transition provided"
1756        for pt in f2t.keys():
1757            student_obj.invokeFactory(pt,f2t[pt]['id'])
1758            sub_obj = getattr(student_obj,f2t[pt]['id'])
1759            sub_doc = sub_obj.getContent()
1760            d['Title'] = f2t[pt]['title']
1761            for field in f2t[pt]['fields']:
1762                d[field] = mapping.get(field,'')
1763
1764            if pt == "StudentApplication":
1765                #d['jamb_sex']  = 'M'
1766                #if mapping.get('sex'):
1767                #    d['jamb_sex']  = 'F'
1768                d['jamb_firstname'] = mapping.get('firstname',None)
1769                d['jamb_middlename'] = mapping.get('middlename',None)
1770                d['jamb_lastname'] = mapping.get('lastname',None)
1771
1772            # if pt == "StudyCourse":
1773            #     for von,zu in (('entry_mode','current_mode'),
1774            #                    ('entry_session','current_session')):
1775            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1776            #             d[zu] = mapping[von]
1777            sub_doc.edit(mapping = d)
1778
1779            #import pdb;pdb.set_trace()
1780            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1781            if new_state != "remain":
1782                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1783        self.portal_workflow.doActionFor(student_obj,transition)
1784        student_obj.manage_setLocalRoles(sid, ['Owner',])
1785        return sid,''
1786    ###)
1787
1788    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1789    def mass_edit_student(self,mapping):
1790        wftool = self.portal_workflow
1791        "edit a students record due import"
1792        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1793        students_folder = self.portal_url.getPortalObject().campus.students
1794        sid = mapping.get('id',None)
1795        jamb_reg_no = mapping.get('jamb_reg_no',None)
1796        matric_no = mapping.get('matric_no',None)
1797        editable_keys = mapping.keys()
1798        if sid:
1799            res = self.students_catalog(id = sid)
1800            if not res:
1801                return '',"no student with id %s" % sid
1802            if matric_no and res[0].matric_no and\
1803              matric_no != res[0].matric_no:
1804                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1805            if jamb_reg_no and res[0].jamb_reg_no and\
1806              jamb_reg_no != res[0].jamb_reg_no:
1807                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1808        elif jamb_reg_no:
1809            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1810            if not res:
1811                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1812            editable_keys.remove('jamb_reg_no')
1813        elif matric_no:
1814            res = self.students_catalog(matric_no = matric_no)
1815            if not res:
1816                return '',"no student with matric_no %s" % matric_no
1817            editable_keys.remove('matric_no')
1818
1819        ## included only to change wf state from admitted to returning
1820        #if res[0].review_state not in ('admitted','objection_raised'):
1821        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1822        ## end inclusion
1823
1824        sid = res[0].id
1825        student_obj = getattr(students_folder,sid)
1826        f2t = self.field2types_student
1827        d = {}
1828        #import pdb;pdb.set_trace()
1829        any_change = False
1830        #special treatment for StudentStudyLevel
1831        d['verdict']  = mapping.get('current_verdict','')
1832        d['session']  = mapping.get('current_session','')
1833        current_level = mapping.get('current_level','')
1834        while d['session'] and d['verdict'] and current_level:
1835            sub_obj = getattr(student_obj,'study_course',None)
1836            if sub_obj is None:
1837                break
1838            level_obj = getattr(sub_obj,current_level,None)
1839            if  level_obj is None:
1840                break
1841            any_change = True
1842            level_obj.getContent().edit(mapping = d)
1843            try:
1844                wftool.doActionFor(level_obj,'close')
1845            except:
1846                pass
1847            break
1848        for pt in f2t.keys():
1849            #if pt == "StudentApplication":
1850            #    d['jamb_sex']  = 'M'
1851            #    if mapping.get('sex'):
1852            #        d['jamb_sex']  = 'F'
1853            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1854            if intersect and pt not in ('StudentStudyLevel',):
1855                object_id = f2t[pt]['id']
1856                sub_obj = getattr(student_obj,object_id,None)
1857                if sub_obj is None:
1858                    try:
1859                        student_obj.invokeFactory(pt,object_id)
1860                    except:
1861                        continue
1862                    sub_obj = getattr(student_obj,object_id)
1863                    if f2t[pt]['title'] != '':
1864                        d['Title'] = f2t[pt]['title']
1865                sub_doc = sub_obj.getContent()
1866                for field in intersect:
1867                    changed = False
1868                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1869                        any_change = True
1870                        changed = True
1871                        d[field] = mapping.get(field,'')
1872                    if changed:
1873                        sub_doc.edit(mapping = d)
1874
1875
1876        ## included only to change wf state from admitted to returning
1877        #    if res[0].review_state in ('admitted','objection_raised'):
1878        #        new_state = f2t[pt]['wf_transition_return']
1879        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1880        #        if sub_obj and new_state != "remain":
1881        #            try:
1882        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1883        #            except:
1884        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1885        #                pass
1886        #if res[0].review_state in ('admitted','objection_raised'):
1887        #    wfaction = 'return'
1888        #    try:
1889        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1890        #        logger.info('%s, wf state changed' % sid)
1891        #        any_change = True
1892        #    except:
1893        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1894        #        pass
1895        ## end inclusion
1896
1897
1898        if any_change:
1899            return sid,''
1900        else:
1901            return sid,'not modified'
1902    ###)
1903
1904    security.declareProtected(ModifyPortalContent,"importData")###(
1905    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1906        """load data from CSV values"""
1907        import transaction
1908        import random
1909
1910        pm = self.portal_membership
1911        member = pm.getAuthenticatedMember()
1912
1913        logger = logging.getLogger('WAeUPTool.importData')
1914        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1915        students_folder = self.portal_url.getPortalObject().campus.students
1916        start = True
1917        tr_count = 0
1918        total_imported = 0
1919        total_not_imported = 0
1920        total = 0
1921        iname = "import_%s" % name
1922        if iname in ('import_application','import_course_result',):
1923            commit_after = 2000
1924        else:
1925            commit_after = 100
1926        stool = getToolByName(self, 'portal_schemas')
1927        ltool = getToolByName(self, 'portal_layouts')
1928        schema = stool._getOb(iname)
1929        if schema is None:
1930            em = 'No such schema %s' % iname
1931            logger.error('No such schema %s' % iname)
1932            return em
1933        layout = ltool._getOb(iname)
1934        if layout is None:
1935            em = 'No such layout %s' % iname
1936            logger.error(em)
1937            return em
1938        validators = {}
1939        for widget in layout.keys():
1940            validators[widget] = layout[widget].validate
1941        mode = "create"
1942        if edit:
1943            mode = "edit"
1944        importer_name = "mass_%(mode)s_%(name)s" % vars()
1945        importer = getattr(self, '%s' % importer_name,None)
1946        if importer is None:
1947            em = 'No importer function %s' % importer_name
1948            logger.error(em)
1949            return em
1950        not_imported = []
1951        imported = []
1952        try:
1953            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1954        except:
1955            em = 'Error reading %s.csv' % filename
1956            logger.error(em)
1957            return em
1958        for item in items:
1959            #import pdb;pdb.set_trace()
1960            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1961            if start:
1962                start = False
1963                adapters = [MappingStorageAdapter(schema, item)]
1964                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1965                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1966                import_keys = [k.strip() for k in attrs if not k.startswith('ignore')]
1967                diff2schema = set(import_keys).difference(set(schema.keys()))
1968                diff2layout = set(import_keys).difference(set(layout.keys()))
1969                if diff2schema:
1970                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1971                    return em
1972                if mode == "create":
1973                    required_keys = [layout.getIdUnprefixed(id)
1974                                     for id,widget in layout.objectItems()
1975                                     if widget.is_required]
1976                    if not set(required_keys).issubset(set(import_keys)):
1977                        diff2import = set(required_keys).difference(set(import_keys))
1978                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1979                        return em
1980                s = ','.join(['"%s"' % fn for fn in import_keys])
1981                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1982                s = '"id",' + s
1983                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1984                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1985                format_error = format + ',"%(Error)s"'
1986                format = '"%(id)s",'+ format
1987
1988            dm = DataModel(item, adapters,context=self)
1989            ds = DataStructure(data=item,datamodel=dm)
1990            error_string = ""
1991            total += 1
1992            error_count = 0
1993            for k in import_keys:
1994                if not validators[k](ds,mode=mode):
1995                    if error_count:
1996                        error_string += ' ++ '
1997                    error_string += "%s: %s" % (k,
1998                                                  self.translation_service(ds.getError(k),
1999                                                                           ds.getErrorMapping(k)))
2000                    error_count += 1
2001            if not error_string:
2002                temp_item = item.copy()
2003                temp_item.update(dm)
2004                temp_item['id'],error = importer(temp_item)
2005                if error:
2006                    error_string += error
2007                else:
2008                    item = temp_item
2009            if error_string:
2010                item['Error'] = error_string
2011                not_imported.append(format_error % item)
2012                total_not_imported += 1
2013            else:
2014                em = format % item
2015                imported.append(em)
2016                tr_count += 1
2017                total_imported += 1
2018                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
2019
2020            if total and not total % commit_after:
2021                transaction.commit()
2022                if len(not_imported) > 0:
2023                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2024                             '\n'.join(not_imported) + '\n')
2025                    not_imported = []
2026                if len(imported) > 0:
2027                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2028                             '\n'.join(imported) + '\n')
2029                    imported = []
2030                em = '%d transactions committed\n' % (tr_count)
2031                regs = []
2032                logger.info(em)
2033                tr_count = 0
2034                #if total > 100:
2035                #    return
2036        if len(imported) > 0:
2037            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2038                                                '\n'.join(imported))
2039        if len(not_imported) > 0:
2040            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2041                                                '\n'.join(not_imported))
2042        em = "Finished import from %s.csv: %d imported, %d not imported (of total %d)" % (filename,total_imported,total_not_imported,total)
2043        logger.info(em)
2044        return em
2045    ###)
2046
2047    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2048    def moveImagesToFS(self,student_id="O738726"):
2049        "move the images to the filesystem"
2050        images_dir = getImagesDir(student_id)
2051        #images_dir = os.path.join("%s" % images_base,student_id)
2052        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2053        stool = getToolByName(self, 'portal_schemas')
2054        schemas = ['student_application',
2055                   'student_clearance',
2056                   ]
2057        created = False
2058        for schema_id in schemas:
2059            schema = stool._getOb(schema_id)
2060            object = getattr(student_folder,schema_id[len('student_'):],None)
2061            if object is None:
2062                continue
2063            doc = object.getContent()
2064            for key in schema.keys():
2065                if schema[key].meta_type != "CPS Image Field":
2066                    continue
2067                #import pdb;pdb.set_trace()
2068                image = getattr(doc,key,None)
2069                if not image or not hasattr(image,"data"):
2070                    continue
2071                if not created:
2072                    if not os.path.exists(images_dir):
2073                        os.mkdir(images_dir)
2074                    created = True
2075                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2076                open(filename,"wb").write(str(image.data))
2077    ###)
2078
2079    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2080    def movePassportToFS(self,student_id="O738726"):
2081        "move the passports to the filesystem"
2082        images_dir = os.path.join("%s" % i_home,'passports')
2083        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2084        stool = getToolByName(self, 'portal_schemas')
2085        schemas = ['student_application',
2086                   #'student_clearance',
2087                   ]
2088        created = False
2089        for schema_id in schemas:
2090            schema = stool._getOb(schema_id)
2091            object = getattr(student_folder,schema_id[len('student_'):],None)
2092            if object is None:
2093                continue
2094            doc = object.getContent()
2095            for key in schema.keys():
2096                if schema[key].meta_type != "CPS Image Field":
2097                    continue
2098                #import pdb;pdb.set_trace()
2099                image = getattr(doc,key)
2100                if not hasattr(image,"data"):
2101                    continue
2102                if not created:
2103                    if not os.path.exists(images_dir):
2104                        os.mkdir(images_dir)
2105                    created = True
2106                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2107                open(filename,"wb").write(str(image.data))
2108    ###)
2109
2110InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.