source: WAeUP_SRP/base/WAeUPTool.py @ 3039

Last change on this file since 3039 was 3039, checked in by Henrik Bettermann, 17 years ago

minor changes
  • Property svn:keywords set to Id
File size: 91.9 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3039 2008-01-23 19:09:11Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64
65class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66    """WAeUP tool"""
67
68    id = 'waeup_tool'
69    meta_type = 'WAeUP Tool'
70    _actions = ()
71    security = ClassSecurityInfo()
72    security.declareObjectProtected(View)
73    manage_options = ( ActionProviderBase.manage_options
74                     + SimpleItem.manage_options
75                     )
76
77    security.declareProtected(View,'re_split') ###(
78    def re_split(self,split_string,string):
79        return re.split(split_string,string)
80    ###)
81
82    security.declareProtected(View,'difference') ###(
83    def difference(self,l1,l2):
84        return set(l1).difference(set(l2))
85    ###)
86
87    def rwrite(self,s): ###(
88        response = self.REQUEST.RESPONSE
89        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
90        response.write("%s<br />\r\n" % s)
91    ###)
92
93    def addtodict(self,d,key,item): ###(
94        d[key].append(item)
95        return d[key]
96    ###)
97
98    def sleep(self,secs): ###(
99        "sleep"
100        import time
101        time.sleep(secs)
102        return
103
104###)
105
106    security.declareProtected(View,'updateRoleMappingsFor') ###(
107    def updateRoleMappingsFor(self,wf_definition,ob):
108        "do so for public"
109        wf_def = getattr(self.portal_workflow,wf_definition)
110        wf_def.updateRoleMappingsFor(ob)
111    ###)
112
113    security.declareProtected(View,'getStatesLgas') ###(
114    def getStatesLgas(self):
115        """return lga info"""
116        voc = getattr(self.portal_vocabularies,'local_gov_areas')
117        states = []
118        lgas  = []
119        d = {}
120        wd = {}
121        for k,v in voc.items():
122            parts = v.split(' / ')
123            if len(parts) == 1:
124                state = parts[0].lower()
125                lga = ""
126            elif len(parts) == 2:
127                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
128                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
129            else:
130                continue
131            if state not in states:
132                states.append(state)
133            if lga not in lgas:
134                lgas.append(lga)
135            words = re.split('[^a-zA-Z0-9/]',k)
136            words.sort()
137            wd[k] = words
138            d[k] = v
139        mapping = {}
140        mapping['word_dict'] = wd
141        mapping['lga_dict'] = d
142        mapping['states'] = states
143        mapping['lgas'] = lgas
144        return mapping
145    ###)
146
147    security.declareProtected(View,'findLga') ###(
148    def findLga(self,words,words_dict):
149        words = re.split('[^a-zA-Z0-9/]',words)
150        lga_words = []
151        for word in words:
152            if word:
153                lga_words += word.strip().lower(),
154        lga_words.sort()
155        state_lga = ''
156        while not state_lga:
157            for k,l in words_dict.items():
158                if lga_words == l:
159                    state_lga = k
160                    break
161            break
162        return state_lga
163    ###)
164    security.declareProtected(View,'getAccessInfo') ###(
165    def getAccessInfo(self,context):
166        "return a dict with access_info"
167        logger = logging.getLogger('WAeUPTool.getAccessInfo')
168        mtool = self.portal_membership
169        member = mtool.getAuthenticatedMember()
170        member_id = str(member)
171        info = {}
172        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
173        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
174        is_staff = info['is_staff'] = not is_anonymous and not is_student
175        roles = member.getRolesInContext(context)
176        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
177                                                                            "SectionManager" in roles or
178                                                                            "Manager" in roles)
179        is_allowed = info['is_allowed'] = not is_anonymous
180        requested_id = context.getStudentId()
181        student_id  = None
182        if is_allowed:
183            if not is_student and requested_id:
184                student_id  = requested_id
185            elif is_anonymous or member_id != requested_id:
186                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
187            else:
188                student_id = member_id
189        info['student_id'] = student_id
190        return info
191    ###)
192
193    security.declareProtected(ModifyPortalContent,'openLog') ###(
194    def openLog(self,name):
195        """open a log file"""
196        version = 1
197        path = "%s/log/%s_%d.log" % (i_home,name,version)
198        while os.path.exists(path):
199            version += 1
200            path = "%s/log/%s_%d.log" % (i_home,name,version)
201        log = open(path,"w")
202        return log
203    ###)
204
205    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
206    def bypassQueueCatalog(self,enable=True):
207        """bypass the QueueCatalog by setting all indexes to process imediate,
208        if enable is True (default) the old settings are restored
209        """
210
211    ###)
212
213    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
214    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
215        """measure Object access Time"""
216        import random
217        if hasattr(self,'portal_catalog_real'):
218            aq_portal = self.portal_catalog_real.evalAdvancedQuery
219        else:
220            aq_portal = self.portal_catalog.evalAdvancedQuery
221        nr_pts = int(nr_pts)
222        probe = int(probe)
223        intervall = probe/10
224        objects = ("application","clearance","personal")
225        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
226        #i = random.randrange(num_objects)
227        count = 0
228        found = 0
229        not_found = 0
230        t_found = 0
231        t_not_found = 0
232        time_found = time_not_found = 0.0
233        t_time_found = t_time_not_found = 0.0
234        accessed = []
235        t_min = 1000
236        t_max = 0
237        #import pdb;pdb.set_trace()
238        students = self.portal_catalog(portal_type="Student")
239        num_students = len(students)
240        if method == "d":
241            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
242            res = aq_portal(query)
243            brains = {}
244            for r in res:
245                sid = r.relative_path.split('/')[-2]
246                if brains.has_key(sid):
247                    brains[sid][r.portal_type] = r
248                else:
249                    brains[sid] = {r.portal_type : r}
250            brains_list = brains.keys()
251            num_objects = len(brains_list)
252        else:
253            num_objects = num_students
254        print "="*40
255        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
256                                                                        probe,
257                                                                        nr_pts,
258                                                                        num_objects)
259        print "nr found/not time found/not min/max"
260        elapse = time.time()
261        i_elapse = time.time()
262        c_elapse = time.clock()
263        for c in range(1,probe + 1):
264            i = random.randrange(num_objects)
265            if method in ('a','b','c'):
266                student_brain = students[i]
267            elif method == "d":
268                #import pdb;pdb.set_trace()
269                student_brain = brains[brains_list[i]]
270            if method == "c":
271                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
272                res = aq_portal(query)
273                this_portal_types = [r.portal_type for r in res]
274            for i in range(nr_pts):
275                oid = objects[i]
276                if method == "a":
277                    try:
278                        student_path = student_brain.getPath()
279                        path = "%s/%s" % (student_path,oid)
280                        doc = self.unrestrictedTraverse(path).getContent()
281                        found += 1
282                        i_time = time.time() - i_elapse
283                        time_found += i_time
284                    except:
285                        not_found += 1
286                        i_time = time.time() - i_elapse
287                        time_not_found += i_time
288                        pass
289                elif method == "b":
290                    try:
291                        student_object = student_brain.getObject()
292                        doc = getattr(student_object,oid).getContent()
293                        found += 1
294                        i_time = time.time() - i_elapse
295                        time_found += i_time
296                    except:
297                        i_time = time.time() - i_elapse
298                        time_not_found += i_time
299                        not_found += 1
300                        pass
301                elif method == "c":
302                    if portal_types[i] in this_portal_types:
303                        found += 1
304                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
305                        i_time = time.time() - i_elapse
306                        time_found += i_time
307                    else:
308                        not_found += 1
309                        i_time = time.time() - i_elapse
310                        time_not_found += i_time
311                elif method == "d":
312                    if student_brain.has_key(portal_types[i]):
313                        found += 1
314                        doc = student_brain[portal_types[i]].getObject().getContent()
315                        i_time = time.time() - i_elapse
316                        time_found += i_time
317                    else:
318                        not_found += 1
319                        i_time = time.time() - i_elapse
320                        time_not_found += i_time
321                i_elapse = time.time()
322            if c and (c % intervall == 0):
323                #i_time = time.time() - i_elapse
324                t_per = 0.0
325                if found:
326                    t_per = time_found/found
327                if t_per > t_max:
328                    t_max = t_per
329                if t_per > 0.0 and t_per < t_min:
330                    t_min = t_per
331                itf = 0.0
332                if found:
333                    itf = time_found/found
334                itnf = 0.0
335                if not_found :
336                    itnf = time_not_found / not_found
337                interval_time = time_found + time_not_found
338                s = "%(c)d: %(found)d/%(not_found)d " % vars()
339                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
340                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
341                print s
342                t_found += found
343                t_not_found += not_found
344                t_time_found += time_found
345                t_time_not_found += time_not_found
346                time_found = time_not_found = 0.0
347                found = not_found = 0
348        # t_found += found
349        # t_not_found += not_found
350        elapse = time.time() - elapse
351        itf = 0.0
352        if t_found:
353            itf = t_time_found/t_found
354        itnf = 0.0
355        if t_not_found:
356            itnf = t_time_not_found / t_not_found
357        #c_elapse = time.clock() - c_elapse
358        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
359        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
360        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
361        print "-"*40
362        print s
363        rel_found = float(t_found)/probe
364        rel_not_found = float(t_not_found)/probe
365        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
366        print estimated_total_time
367    ###)
368
369    security.declareProtected(ModifyPortalContent,'writeLog') ###(
370    def writeLog(self,logfile,s):
371        """write to the log file"""
372        logfile.write(s)
373
374###)
375
376    def generateStudentId(self,letter): ###(
377        import random
378        r = random
379        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
380        if letter == '?':
381            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
382        sid = "%c%d" % (letter,r.randint(99999,1000000))
383        students = self.portal_url.getPortalObject().campus.students
384##        while hasattr(students, sid):
385##            sid = "%c%d" % (letter,r.randint(99999,1000000))
386        while self.students_catalog(id = sid):
387            sid = "%c%d" % (letter,r.randint(99999,1000000))
388        return sid
389    ###)
390
391    def generatePassword(self,s=None): ###(
392        import random
393        r = random
394        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
395        if s is None:
396            s = 'abcdefghklmnpqrstuvwxy23456789'
397        pw = ''
398        while len(pw) < 6:
399            pw += r.choice(s)
400        return pw
401    ###)
402
403    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
404    def dumpSchoolfeePayments(self):
405        "dump paid schoolfees"
406        mtool = self.portal_membership
407        member = mtool.getAuthenticatedMember()
408        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
409        aq_student = self.students_catalog.evalAdvancedQuery
410        query = In('review_state',('schoolfee_paid',
411                                   'courses_registered',
412                                   'courses_validated',
413                                   ))
414        res = aq_student(query)
415        #import pdb;pdb.set_trace()
416        l = []
417        logger.info("start for %d" % len(res))
418        count = 1
419        log_after = 100
420        for student in res:
421            if not count % log_after:
422                logger.info("processed %d total %d" % (log_after,count))
423            count += 1
424            fee_dict =self.getSchoolFee(student)
425            fulltime = student.mode.endswith('_ft')
426            d = {}
427            d['student_id'] = student.id
428            d['name'] = student.name
429            d['amount'] = fee_dict.get(new_returning)
430            l += d,
431        csv_name = self.dumpListToCSV(l,'payments')
432        logger.info('%s dumped payments to %s' % (member,export_file))
433    ###)
434
435    security.declarePublic('dumpListToCSV') ###(
436    def dumpListToCSV(self,l,filename,fields=None):
437        """dump a list of dicts to a CSV file"""
438        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
439        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
440        if fields is None:
441            fields = l[0].keys()
442        headline = ','.join(fields)
443        out = open(export_file,"wb")
444        out.write(headline +'\n')
445        out.close()
446        out = open(export_file,"a")
447        csv_writer = csv.DictWriter(out,fields,)
448        csv_writer.writerows(l)
449        return export_file
450    ###)
451
452    security.declareProtected(ManagePortal, 'listMembers') ###(
453    def listMembers(self):
454        "list all members"
455        mtool = self.portal_membership
456        member = mtool.getAuthenticatedMember()
457        logger = logging.getLogger('WAeUPTool.listMembers')
458        if str(member) not in ('admin','joachim'):
459            logger.info('%s tried to list members' % (member))
460            return None
461        members = self.portal_directories.members
462        all = members.listEntryIdsAndTitles()
463        l = []
464        for user_id,name in all:
465            d = {}
466            d['user_id'] = user_id
467            d['name'] = name
468            d['pw'] = getattr(getattr(members,user_id),'password')
469            d['email'] = getattr(getattr(members,user_id),'email')
470            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
471            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
472            l += d,
473        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
474        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
475        logger.info('%s dumped memberlist to %s' % (member,export_file))
476        fields = l[0].keys()
477        headline = ','.join(fields)
478        out = open(export_file,"wb")
479        out.write(headline +'\n')
480        out.close()
481        out = open(export_file,"a")
482        csv_writer = csv.DictWriter(out,fields,)
483        csv_writer.writerows(l)
484    ###)
485
486    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
487    def removeDeletedDocIds(self, max=1000):
488        """
489        remove deleted docids from repository commit after max
490        """
491        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
492        repository = getToolByName(self, 'portal_repository')
493        pxtool = getToolByName(self, 'portal_proxies')
494        logger.info('start')
495        pxtool_infos = pxtool.getRevisionsUsed()
496        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
497
498        nb_revs = 0
499        docids_d = {} # all docids
500        unused_docids_d = {} # all docids that are unused
501        ids_unused_revs_docids = [] # ids for revs of unused docids
502        unused_ids = [] # ids for unused revs
503        total = 0
504        idlist = repository.objectIds()
505        to_delete = 0
506        found = False
507        for id in idlist:
508            docid, rev = repository._splitId(id)
509            if docid is None:
510                logger.info("invalid doc_id %s" % docid)
511                continue
512            nb_revs += 1
513            if not pxtool_infos.has_key(docid):
514                found = True
515                to_delete += 1
516                unused_ids.append(id)
517            elif not pxtool_infos[docid].has_key(rev):
518                found = True
519                to_delete += 1
520                unused_ids.append(id)
521            if found and not to_delete % max:
522                found = False
523                #import pdb;pdb.set_trace()
524                repository.manage_delObjects(unused_ids)
525                transaction.commit()
526                logger.info('removed %d total %d unused docids ' % (max,to_delete))
527        else:
528            if unused_ids:
529                repository.manage_delObjects(unused_ids)
530                transaction.commit()
531        logger.info('finished removing %d unused docids ' % (to_delete))
532
533
534###)
535
536    security.declareProtected(View,'getCredential') ###(
537    def getCredential(self,student_id):
538        student_entry = getattr(self.portal_directories.students,student_id,None)
539        if not self.isStaff():
540            mtool = self.portal_membership
541            member = mtool.getAuthenticatedMember()
542            logger = logging.getLogger('WAeUPTool.getCredential')
543            logger.info('%s tried to access password of %s' % (member,student_id))
544            return None
545        if student_entry is None:
546            return None
547        return getattr(student_entry,"password","not set")
548    ###)
549
550    security.declarePublic('checkPassword') ###(
551    def checkPassword(self,student_id,password):
552        student_entry = getattr(self.portal_directories.students,student_id,None)
553        if student_entry is None:
554            return False
555        return getattr(student_entry,"password","not set") == password
556    ###)
557
558    security.declarePublic('checkGenericPassword') ###(
559    def checkGenericPassword(self,member_id):
560        member_entry = getattr(self.portal_directories.members,member_id,None)
561        if member_entry is None:
562            return False
563        ltool = getToolByName(self, 'portal_layouts')
564        unsecure_words = ltool._getOb('members')['w__password'].check_words
565        password = getattr(member_entry,"password","not set")
566        is_unsecure = password in unsecure_words
567        if is_unsecure:
568            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
569            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
570        return is_unsecure
571    ###)
572
573    security.declareProtected(ModifyPortalContent,'editPassword') ###(
574    def editPassword(self,student_id,password):
575        "edit a student password"
576        student_entry = getattr(self.portal_directories.students,student_id,None)
577        if student_entry is None:
578            return
579        setattr(student_entry,'password',password)
580    ###)
581
582    security.declareProtected(ModifyPortalContent,'doCommit') ###(
583    def doCommit(self,logger=None):
584        "commit some transactions"
585        transaction.commit()
586    ###)
587
588    security.declarePublic('loadStudentFoto') ###(
589    def loadStudentFoto(self,student,filename,folder):
590        "return a student passport picture"
591        #import pdb;pdb.set_trace()
592        picture ="%s/import/%s/%s" % (i_home,folder,filename)
593        student_id = student.getId()
594        images_dir = getImagesDir(student_id)
595        if not os.path.exists(images_dir):
596            os.mkdir(images_dir)
597        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
598        for extension in ('.jpg','.JPG'):
599            fullname = "%(picture)s%(extension)s" % vars()
600            if os.path.exists(fullname):
601                copy2(fullname,image_name)
602                return "successfully copied passport picture"
603        return "passport picture not found: %s.jpg or .JPG" % picture
604    ###)
605
606    def old____loadStudentFoto(self,student,filename,folder): ###(
607        "return a student passport picture"
608        app = student.application
609        app_doc = app.getContent()
610        #clear = student.clearance
611        #clear_doc = clear.getContent()
612        #matric_no = clear_doc.matric_no.upper()
613        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
614        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
615        #import pdb;pdb.set_trace()
616        if os.path.exists(picture1):
617            file = open(picture1)
618        elif os.path.exists(picture2):
619            file = open(picture2)
620        else:
621            return "passport picture not found %s" % picture1
622        reopened = False
623        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
624            self.portal_workflow.doActionFor(app,'open')
625            reopened = True
626        outfile = file.read()
627        app_doc.manage_addFile('passport',
628                               file=outfile,
629                               title="%s.jpg" % filename)
630        if reopened:
631            self.portal_workflow.doActionFor(app,'close')
632        return "successfully loaded passport picture"
633    ###)
634
635    security.declareProtected(ModifyPortalContent,'createOne') ###(
636    def createOne(self,students_folder,student_brain,letter,commit=False):
637        sid = self.waeup_tool.generateStudentId(letter)
638        students_folder.invokeFactory('Student', sid)
639        student = getattr(students_folder,sid)
640        self.portal_workflow.doActionFor(student,'return')
641        student.manage_setLocalRoles(sid, ['Owner',])
642        matric_no = student_brain.matric_no
643        jamb_reg_no = student_brain.Entryregno
644        self.students_catalog.addRecord(id = sid,
645                                           matric_no = matric_no,
646                                           jamb_reg_no = jamb_reg_no,
647                                           sex = student_brain.Sex == "F",
648                                           name = "%s %s %s" % (student_brain.Firstname,
649                                                                student_brain.Middlename,
650                                                                student_brain.Lastname)
651                                        )
652        if commit:
653            transaction.commit()
654        return sid,jamb_reg_no
655    ###)
656
657    security.declareProtected(ModifyPortalContent,'addStudent') ###(
658    def addStudent(self,dict):
659        students_folder = self.portal_url.getPortalObject().campus.students
660        sid = self.waeup_tool.generateStudentId('?')
661        students_folder.invokeFactory('Student', sid)
662        student_obj = getattr(students_folder,sid)
663        f2t = self.field2types_student
664        #from pdb import set_trace; set_trace()
665        d = {}
666        #d['jamb_sex']  = 'M'
667        #if dict.get('sex'):
668        #    d['jamb_sex']  = 'F'
669
670        entry_session = dict.get('entry_session')
671        if entry_session == self.getSessionId()[0]:
672            wfaction = 'admit'
673            wft = 'wf_transition_admit'
674            password = None
675        else:
676            wfaction = 'return'
677            wft = 'wf_transition_return'
678            password = self.generatePassword()
679            self.makeStudentMember(sid,password)
680
681        for pt in f2t.keys():
682            student_obj.invokeFactory(pt,f2t[pt]['id'])
683            sub_obj = getattr(student_obj,f2t[pt]['id'])
684            sub_doc = sub_obj.getContent()
685            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
686            d['Title'] = f2t[pt]['title']
687            for field in f2t[pt]['fields']:
688                d[field] = dict.get(field,'')
689            sub_doc.edit(mapping = d)
690            new_state = f2t[pt][wft]
691            if new_state != "remain":
692                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
693        self.portal_workflow.doActionFor(student_obj,wfaction)
694        student_obj.manage_setLocalRoles(sid, ['Owner',])
695        return sid,password
696    ###)
697
698    security.declarePublic('getCertificateBrain') ###(
699    def getCertificateBrain(self,cert_id):
700        "do it"
701        res = ZCatalog.searchResults(self.portal_catalog_real,
702                                {'portal_type':"Certificate",
703                                      'id': cert_id})
704        if res:
705            return res[0]
706        return None
707    ###)
708
709    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
710    def get_csv_filenames(self):
711        "do it"
712        files = [file for file in os.listdir("%s/import/" % (i_home))
713                 if file.endswith('.csv') and file.find('imported') == -1]
714        return files
715    ###)
716
717    security.declarePublic('findStudentByMatricelNo') ###(
718    def findStudentByMatricelNo(self,matric_no):
719        "do it"
720        res = ZCatalog.searchResults(self.portal_catalog_real,
721                                {'portal_type':"StudentClearance",
722                                 'SearchableText': matric_no})
723        if res:
724            return res[0]
725        return None
726    ###)
727
728    security.declarePublic('makeStudentMember') ###(
729    def makeStudentMember(self,sid,password='uNsEt'):
730        """make the student a member"""
731        membership = self.portal_membership
732        membership.addMember(sid,
733                             password ,
734                             roles=('Member',
735                                     'Student',
736                                     ),
737                             domains='',
738                             properties = {'memberareaCreationFlag': False,
739                                           'homeless': True},)
740        member = membership.getMemberById(sid)
741        self.portal_registration.afterAdd(member, sid, password, None)
742        #self.manage_setLocalRoles(sid, ['Owner',])
743    ###)
744
745    security.declareProtected(View,'makeStudentData') ###(
746    def makeStudentData(self,student_id,email=None,phone_nr=None):
747        "create Datastructure for a returning Student"
748        #import pdb;pdb.set_trace()
749        logger = logging.getLogger('WAeUPTool.makeStudentData')
750        students_folder = self.portal_url.getPortalObject().campus.students
751        #res = self.students_catalog(id=student_id)
752        #if res:
753        #    st = res[0]
754        #res = self.returning_import(matric_no = st.matric_no)
755        res = self.returning_import(id = student_id)
756        if res:
757            student = res[0]
758        else:
759            logger.info('Id %s not found in returning_import' % student_id)
760            return
761        logger.info('%s creates data structure' % student_id)
762        s_results = self.results_import(matric_no = student.matric_no)
763        if s_results:
764            lnr = self.getLevelFromResultsCosCode(s_results)
765            level = "%d00" % lnr
766            verdict,eligible = self.getVerdict(s_results[0].Verdict)
767            #if eligible:
768            #    level = "%d00" % (lnr + 1)
769        else:
770            logger.info('matric_no %s not found in results_import' % student.matric_no)
771            level = ''
772            verdict = ''
773        #student should not be allowed to perform this transition
774        #wftool = self.portal_workflow
775        #wftool.doActionFor(student,'return')
776        certcode_org = student.Coursemajorcode
777        certcode = makeCertificateCode(certcode_org)
778        certificate_brain = self.getCertificateBrain(certcode)
779        if not certificate_brain:
780            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
781            logger.info(em)
782        matric_no = student.matric_no
783        sid = student_id
784        student_obj = getattr(students_folder,sid)
785        student_obj.invokeFactory('StudentApplication','application')
786        application = student_obj.application
787        self.portal_workflow.doActionFor(application,'open',dest_container=application)
788        da = {'Title': 'Application Data'}
789        student_obj.invokeFactory('StudentPersonal','personal')
790        da['jamb_reg_no'] = student.Entryregno
791        em = self.getEntryMode(student.Entryregno)
792        da['entry_mode'] = em
793        personal = student_obj.personal
794        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
795        dp = {'Title': 'Personal Data'}
796        student_obj.invokeFactory('StudentClearance','clearance')
797        clearance = student_obj.clearance
798        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
799        dc = {'Title': 'Clearance/Eligibility Record'}
800        dc['matric_no'] = matric_no
801        state = student.State
802        lga = student.LGA
803        if state and lga:
804            lga =  state + ' / ' + lga
805        else:
806            lga = "None"
807        da['jamb_lga'] = dc['lga'] = lga
808        da['app_email'] = dp['email'] = email
809        da['app_mobile'] = dp['phone'] = phone_nr
810        dp['firstname'] = student.Firstname
811        dp['middlename'] = student.Middlename
812        dp['lastname'] = student.Lastname
813        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
814        da['jamb_sex'] = student.Sex
815        dp['sex'] = student.Sex == 'F'
816        dp['perm_address'] = student.Permanent_Address
817        application.getContent().edit(mapping=da)
818        self.portal_workflow.doActionFor(application,'close',dest_container=application)
819        personal.getContent().edit(mapping=dp)
820        clearance.getContent().edit(mapping=dc)
821        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
822        #
823        # Study Course
824        #
825        student_obj.invokeFactory('StudentStudyCourse','study_course')
826        studycourse = student_obj.study_course
827        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
828        dsc = {}
829        dsc['study_course'] = certcode
830        dsc['current_level'] = level
831        dsc['current_verdict'] = verdict
832        dsc['current_mode'] = em
833        dsc['current_session'] = '05'
834        studycourse.getContent().edit(mapping=dsc)
835        #
836        # Level
837        #
838##        l = getattr(studycourse,level,None)
839##        if l is None:
840##            studycourse.invokeFactory('StudentStudyLevel', level)
841##            l = getattr(studycourse, level)
842##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
843##            l.getContent().edit(mapping={'Title': "Level %s" % level})
844###)
845
846    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
847    def admitOneStudent(self,brain,entry_session,pin_password):
848        "create Datastructure for an admitted Student"
849        #import pdb;pdb.set_trace()
850        if not hasattr(self,"_v_certificates"):
851            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
852        students_folder = self.portal_url.getPortalObject().campus.students
853        logger = logging.getLogger('WAeUPTool.admitOneStudent')
854        if brain.status != "admitted":
855            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
856            return
857        pin_parts = brain.pin.split('-')
858        if pin_parts and len(pin_parts) != 3:
859            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
860            return
861        if brain.course_admitted not in self._v_certificates:
862            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
863            return
864        if brain.sex not in (True,False):
865            logger.info('sex of %s not available' % (brain.reg_no))
866            return
867        student_id = self.generateStudentId('?')
868        students_folder.invokeFactory('Student', student_id)
869        student_object = getattr(students_folder,student_id)
870        if pin_password:
871            password = pin_parts[2]
872            self.makeStudentMember(student_id,password = password)
873        student_object.manage_setLocalRoles(student_id, ['Owner',])
874        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
875        #
876        # application
877        #
878        student_object.invokeFactory('StudentApplication','application')
879        application = student_object.application
880        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
881        da = {'Title': 'Application Data'}
882        da['jamb_reg_no'] = brain.reg_no
883
884        sex = 'M'
885        if brain.sex:
886            sex = 'F'
887        da['jamb_sex'] = sex
888        da['jamb_age'] = brain.jamb_age
889        #da['app_ac_pin'] = brain.pin
890        da['jamb_lga'] = brain.jamb_lga
891        da['jamb_state'] = brain.jamb_state
892        da['jamb_score'] = brain.aggregate
893        da['app_email'] = brain.email
894        da['app_mobile'] = brain.phone
895        if brain.entry_mode:
896            da['entry_mode'] = brain.entry_mode
897        elif brain.screening_type == 'pume':
898            da['entry_mode'] = 'ume_ft'
899        elif brain.screening_type == 'pde':
900            da['entry_mode'] = 'de_ft'
901        elif brain.screening_type == 'pce':
902            da['entry_mode'] = 'pce'
903        elif brain.screening_type == 'prence':
904            da['entry_mode'] = 'prence'
905        else:
906            da['entry_mode'] = ''
907        da['entry_session'] = entry_session
908        da['jamb_lastname'] = brain.lastname
909        da['jamb_middlename'] = brain.middlenames   # different field names!
910        da['jamb_firstname'] = brain.firstname
911        da['screening_application_date'] = brain.application_date
912        da['date_of_birth'] = brain.date_of_birth
913        da['jamb_first_cos'] = brain.course1
914        da['jamb_second_cos'] = brain.course2
915        da['course3'] = brain.course3
916        da['screening_type'] = brain.screening_type
917        da['screening_score'] = brain.screening_score
918        da['screening_date'] = brain.screening_date
919        da['hq_type'] = brain.hq_type
920        da['hq_grade'] = brain.hq_grade
921        da['aos'] = brain.aos
922
923        application.getContent().edit(mapping=da)
924        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
925        #
926        # personal
927        #
928        student_object.invokeFactory('StudentPersonal','personal')
929        personal = student_object.personal
930        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
931        dp = {'Title': 'Personal Data'}
932        dp['sex'] = brain.sex
933        dp['email'] = brain.email
934        dp['phone'] = brain.phone
935        dp['lastname'] = brain.lastname
936        dp['middlename'] = brain.middlenames   # different field names!
937        dp['firstname'] = brain.firstname
938        personal.getContent().edit(mapping=dp)
939        #
940        # clearance
941        #
942        student_object.invokeFactory('StudentClearance','clearance')
943        clearance = student_object.clearance
944        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
945        dc = {'Title': 'Clearance/Eligibility Record'}
946        dc['lga'] = brain.lga
947        dc['birthday'] = brain.date_of_birth
948        clearance.getContent().edit(mapping=dc)
949        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
950        #
951        # study Course
952        #
953        student_object.invokeFactory('StudentStudyCourse','study_course')
954        studycourse = student_object.study_course
955        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
956        dsc = {}
957        dsc['study_course'] = brain.course_admitted
958        dsc['current_verdict'] = ''
959        dsc['current_mode'] = da['entry_mode']
960        if da['entry_mode'].startswith('de'):
961            dsc['current_level'] = '200'
962        else:
963            dsc['current_level'] = '100'
964        dsc['current_session'] = entry_session
965        studycourse.getContent().edit(mapping=dsc)
966        #
967        # payments folder
968        student_object.invokeFactory('PaymentsFolder','payments')
969        payments = getattr(student_object,'payments')
970        dpay = {}
971        dpay['Title'] = 'Payments'
972        payments.getContent().edit(mapping=dpay)
973        self.portal_workflow.doActionFor(payments,'open')
974        #
975        # passport foto
976        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
977                                                              brain.screening_type,
978                                                              brain.reg_no)
979        images_dir = getImagesDir(student_id)
980        #images_dir = os.path.join("%s" % images_base,student_id)
981        letter_dir,student_dir = os.path.split(images_dir)
982        if not os.path.exists(letter_dir):
983            os.mkdir(letter_dir)
984        if not os.path.exists(images_dir):
985            os.mkdir(images_dir)
986        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
987        if os.path.exists(app_picture):
988            copy2(app_picture,image_name)
989        else:
990            logger.info('passport of %s/%s not found: %s' % (student_id,
991                                                             brain.reg_no,
992                                                             app_picture))
993
994        return student_id
995    ###)
996
997    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
998    def makeStudentLevel(self,student_id):
999        "create the StudyLevel for a returning Student"
1000        #import pdb;pdb.set_trace()
1001        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1002        students_folder = self.portal_url.getPortalObject().campus.students
1003        res = self.students_catalog(id=student_id)
1004        if res:
1005            st = res[0]
1006        course = st.course
1007        matric_no = st.matric_no
1008        level = st.level
1009        res = self.results_import(matric_no = matric_no)
1010        if res:
1011            results = res
1012        logger.info('%s creating Level %s' % (student_id,level))
1013        #
1014        # Level
1015        #
1016        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1017        studycourse = getattr(student_obj,"study_course",None)
1018        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1019        l = getattr(studycourse,level,None)
1020        if l is None:
1021            studycourse.invokeFactory('StudentStudyLevel', level)
1022            l = getattr(studycourse, level)
1023            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1024            l.getContent().edit(mapping={'Title': "Level %s" % level})
1025        ###)
1026
1027    security.declarePublic('getHallInfo') ###(
1028    def getHallInfo(self,bed):
1029        """return Hall Info"""
1030        info = {}
1031        hall,block,room,letter = bed.split('_')
1032        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1033        if res and len(res) == 1:
1034            hall_brain = res[0]
1035            hall_doc = hall_brain.getObject().getContent()
1036        else:
1037            return info
1038        info['hall_title'] = hall_brain.Title
1039        info['maintenance_code'] = hall_doc.maintenance_code
1040        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1041        batch_doc = None
1042        for brain in res:
1043            if brain.id.startswith(info['maintenance_code']):
1044                batch_doc = brain.getObject().getContent()
1045                break
1046        if batch_doc is None:
1047            info['maintenance_fee'] = ''
1048        else:
1049            info['maintenance_fee'] = batch_doc.cost
1050        return info
1051    ###)
1052
1053    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1054    def removePictureFolder(self,student_id):
1055        """remove picture_folder by renaming it"""
1056        path = 'images'
1057        picture_path = os.path.join(i_home,path,student_id)
1058        if not os.path.exists(picture_path):
1059            return False
1060        os.rename(picture_path,picture_path + "_removed")
1061        return True
1062    ###)
1063
1064    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1065    def restorePictureFolder(self,student_id):
1066        """restore picture_folder by renaming it"""
1067        path = 'images'
1068        picture_path = os.path.join(i_home,path,student_id)
1069        if not os.path.exists(picture_path + "_removed"):
1070            return False
1071        os.rename(picture_path + "_removed",picture_path)
1072        return True
1073    ###)
1074
1075    security.declarePublic('picturesExist') ###(
1076    def picturesExist(self, ids,student_id=None):
1077        """check if pictures exist in the filesystem"""
1078        if student_id is None:
1079            student_id = self.getStudentId()
1080        if student_id is None:
1081            return False
1082        picture_path = getImagesDir(student_id)
1083        #picture_path = os.path.join(images_base,student_id)
1084        if not os.path.exists(picture_path):
1085            return False
1086        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1087        return set(ids).issubset(set(pictures))
1088    ###)
1089
1090    security.declarePublic('picturesList') ###(
1091    def picturesList(self):
1092        """check if pictures exist in the filesystem"""
1093        path = 'images'
1094        student_id = self.getStudentId()
1095        #picture_path = os.path.join(i_home,path,student_id)
1096        picture_path = getImagesDir(student_id)
1097        if not os.path.exists(picture_path):
1098            return []
1099        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1100    ###)
1101
1102    security.declarePublic('showFsPicture') ###(
1103    def showFsPicture(self,path):
1104        """return a picture from the filesystem"""
1105        #picture_path = os.path.join(i_home,path)
1106        picture_path = os.path.join(images_base,path)
1107        response = self.REQUEST.RESPONSE
1108        #import pdb;pdb.set_trace()
1109        registry = getToolByName(self, 'mimetypes_registry')
1110        mimetype = str(registry.lookupExtension(path.lower()) or
1111                    registry.lookupExtension('file.bin'))
1112        if os.path.exists(picture_path):
1113            response.setHeader('Content-type',mimetype)
1114            return open(picture_path).read()
1115        picture_path = os.path.join(i_home,'import',path)
1116        if os.path.exists(picture_path):
1117            return open(picture_path).read()
1118    ###)
1119
1120    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1121    def deleteAllCourses(self,department="All"):
1122        ''' delete the courses'''
1123        pm = self.portal_membership
1124        member = pm.getAuthenticatedMember()
1125
1126        if str(member) not in ("henrik","joachim"):
1127            return "not possible"
1128        if department == "All":
1129            res = self.portal_catalog({'meta_type': 'Department'})
1130        if len(res) < 1:
1131            return "No Departments found"
1132
1133        deleted = []
1134        for dep in res:
1135            cf = dep.getObject().courses
1136            if cf:
1137                cf.manage_delObjects(ids=cf.objectIds())
1138                deleted.append("deleted Courses in %s" % dep.getId)
1139        return "\r".join(deleted)
1140    ###)
1141
1142    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1143    def getLogfileLines(self,filename="event.log",numlines=20):
1144        """Get last NUMLINES lines of logfile FILENAME.
1145
1146        Return last lines' of a file in the instances logfile directory as
1147        a list. The number of returned lines equals `numlines' or less. If
1148        less than `numlines' lines are available, the whole file ist
1149        returned. If the file can not be opened or some other error
1150        occurs, empty list is returend.
1151        """
1152        result = []
1153        lines_hit = 0
1154
1155        # We only handle files in instances' log directory...
1156        logpath = os.path.join(i_home, "log")
1157        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1158        if not filename.startswith( logpath ):
1159            # Attempt to access file outside log-dir...
1160            return []
1161
1162        try:
1163            fd = file( filename, "rb" )
1164        except IOError:
1165            return []
1166        if not fd:
1167            return []
1168
1169        if os.linesep == None:
1170            linesep = '\n'
1171        else:
1172            linesep = os.linesep
1173
1174        # Try to find 'numlines' times a lineseparator, searching from end
1175        # and moving to the beginning of file...
1176        fd.seek( 0, 2) # Move to end of file...
1177        while lines_hit < numlines:
1178            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1179                                          # one step forward...
1180                lines_hit += 1
1181            try:
1182                fd.seek( -2, 1) # Go two bytes back from current pos...
1183            except IOError:
1184                # We cannot go back two bytes. Maybe the file is too small...
1185                break
1186        fd.seek(2,1)
1187
1188        # Read all lines from current position...
1189        result = fd.readlines()
1190        # Remove line endings...
1191        result = [x.strip() for x in result]
1192        fd.close()
1193        return result
1194    ###)
1195
1196    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1197    def getCallbacksFromLog(self,filename):
1198        """fix Online Payment Transactions from Z2.log entries"""
1199        import transaction
1200        import random
1201        from cgi import parse_qs
1202        from urlparse import urlparse
1203        #from pdb import set_trace
1204        wftool = self.portal_workflow
1205        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1206        students_folder = self.portal_url.getPortalObject().campus.students
1207        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1208        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1209        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1210        data = re.compile(s)
1211        start = True
1212        tr_count = 1
1213        total = 0
1214        #name = 'pume_results'
1215        #name = 'epaymentsuccessful_z2log2'
1216        name = filename
1217        no_import = []
1218        imported = []
1219        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1220        try:
1221            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1222        except:
1223            logger.error('Error reading %s' % name)
1224            return
1225        tas = []
1226        for line in transactions:
1227            dict = {}
1228            items = data.search(line)
1229            dict['idict'] = idict = items.groupdict()
1230            #print idict
1231            #from pdb import set_trace;set_trace()
1232            urlparsed = urlparse(idict['get'][4:])
1233            #print urlparsed
1234            path = urlparsed[2].split('/')
1235            dict['student_id'] = student_id = path[8]
1236            dict['payment_id'] = payment_id = path[10]
1237            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1238            tas.append(dict)
1239            tr_count += 1
1240        return tas
1241    ###)
1242
1243    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1244    def importOnlinePaymentTransactions(self):
1245        """load Online Payment Transactions from CSV values"""
1246        import transaction
1247        import random
1248        #from pdb import set_trace
1249        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1250        opt = self.online_payments_import
1251        students_folder = self.portal_url.getPortalObject().campus.students
1252        start = True
1253        tr_count = 1
1254        total = 0
1255        #name = 'pume_results'
1256        name = 'OnlineTransactions'
1257        no_import = []
1258        imported = []
1259        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1260        try:
1261            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1262        except:
1263            logger.error('Error reading %s.csv' % name)
1264            return
1265        for pay_transaction in transactions:
1266            if start:
1267                start = False
1268                logger.info('Start loading from %s.csv' % name)
1269                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1270                no_import.append('%s,"Error"' % s)
1271                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1272                format_error = format + ',"%(Error)s"'
1273            data = {}
1274
1275            # format of the first file sent by Tayo
1276            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1277            #data['student_id'] = student_id = pay_transaction['Payer ID']
1278            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1279            #data['response_code'] = response_code = pay_transaction['Resp Code']
1280            #data['amount'] = amount = pay_transaction['Amount']
1281
1282            # format of the second file sent by Tayo
1283            #data['datetime'] = date = 0
1284            #data['student_id'] = student_id = pay_transaction['Payer ID']
1285            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1286            #data['response_code'] = response_code = '00'
1287            #data['amount'] = amount = pay_transaction['Amount']
1288
1289            # format of the third file sent by Kehinde
1290            data['datetime'] = date = 0
1291            data['student_id'] = student_id = pay_transaction['customer_id']
1292            data['order_id'] = order_id = pay_transaction['merchant_reference']
1293            data['response_code'] = response_code = '00'
1294            data['amount'] = amount = pay_transaction['Amount']
1295
1296            dup = False
1297            if response_code == "12":
1298                continue
1299            try:
1300                opt.addRecord(**data)
1301            except ValueError:
1302                dup = True
1303            #from pdb import set_trace;set_trace()
1304            if dup:
1305                if response_code == "00":
1306                    try:
1307                        opt.modifyRecord(**data)
1308                    except:
1309                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1310                        continue
1311                else:
1312                    pay_transaction['Error'] = "Duplicate order_id"
1313                    no_import.append( format_error % pay_transaction)
1314                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1315                    continue
1316            tr_count += 1
1317            if tr_count > 1000:
1318                if len(no_import) > 0:
1319                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1320                             '\n'.join(no_import) + '\n')
1321                    no_import = []
1322                em = '%d transactions committed\n' % (tr_count)
1323                transaction.commit()
1324                regs = []
1325                logger.info(em)
1326                total += tr_count
1327                tr_count = 0
1328        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1329                                                '\n'.join(no_import))
1330        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1331    ###)
1332
1333    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1334    def mass_create_faculty(self,mapping):
1335        "create a faculty"
1336        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1337        academics_folder = self.portal_url.getPortalObject().campus.academics
1338        fid = mapping['code']
1339        if getattr(academics_folder,fid,None) is not None:
1340            return '', "Faculty with ID: %s exists" % fid
1341        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1342        try:
1343            academics_folder.invokeFactory('Faculty', fid)
1344        except BadRequest,E:
1345            return '', "%s" % E
1346        f = getattr(academics_folder,fid,None)
1347        f.getContent().edit(mapping=mapping)
1348        return fid,''
1349    ###)
1350
1351    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1352    def mass_edit_faculty(self,mapping):
1353        "edit a faculty"
1354        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1355        academics_folder = self.portal_url.getPortalObject().campus.academics
1356        fid = mapping['code']
1357        f = getattr(academics_folder,fid,None)
1358        if f is None:
1359            return '', "Faculty with ID: %s does not exist" % fid
1360        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1361        f.getContent().edit(mapping=mapping)
1362        return fid,''
1363    ###)
1364
1365    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1366    def mass_create_department(self,mapping):
1367        "create a department in the correct faculty"
1368        logger = logging.getLogger('WAeUPTool.mass_create_department')
1369        fid = mapping['faculty_code']
1370        if getattr(self,'_v_faculties',None) is None:
1371            res = self.portal_catalog(portal_type = "Faculty")
1372            self._v_faculties = {}
1373            for f in res:
1374                self._v_faculties[f.getId] = f.getObject()
1375        f = self._v_faculties.get(fid,None)
1376        if f is None:
1377            return '', "No Faculty with ID: %s" % fid
1378        else:
1379            did = mapping.get('code')
1380            d = getattr(f,did,None)
1381            if d is None or d.portal_type == "Faculty":
1382                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1383                try:
1384                    f.invokeFactory('Department', did)
1385                except BadRequest,E:
1386                    return '', "%s" % E
1387                d = getattr(f,did)
1388                d.invokeFactory('CoursesFolder','courses')
1389                courses = getattr(d,'courses')
1390                dict = {'Title': 'Courses'}
1391                courses.getContent().edit(mapping=dict)
1392                d.invokeFactory('CertificatesFolder','certificates')
1393                certificates = getattr(d,'certificates')
1394                dict = {'Title': 'Certificates'}
1395                certificates.getContent().edit(mapping=dict)
1396            d.getContent().edit(mapping=mapping)
1397        return did,''
1398    ###)
1399
1400    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1401    def mass_edit_department(self,mapping):
1402        "create a department in the correct faculty"
1403        logger = logging.getLogger('WAeUPTool.mass_create_department')
1404        academics_folder = self.portal_url.getPortalObject().campus.academics
1405        fid = mapping['faculty_code']
1406        did = mapping.get('code')
1407        try:
1408            d = getattr(getattr(academics_folder,fid),did,None)
1409        except KeyError:
1410            return '', "Department %s or Faculty %s wrong" % (did,fid)
1411        else:
1412            if d is None or d.portal_type == "Faculty":
1413                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1414            d.getContent().edit(mapping=mapping)
1415        return did,''
1416    ###)
1417
1418    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1419    def mass_create_course(self,mapping):
1420        #import pdb;pdb.set_trace()
1421        if getattr(self,'_v_course_list',None) is None:
1422            self._v_course_list = []
1423        if getattr(self,'_v_departments',None) is None:
1424            res = self.portal_catalog(portal_type = "Department")
1425            self._v_department_courses = {}
1426            for d in res:
1427                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1428        did = mapping['department_code']
1429        d = self._v_department_courses.get(did,None)
1430        if d is None:
1431            return '', "No Department with ID: %s" % did
1432        course_id = mapping.get('code')
1433        if course_id in self._v_course_list:
1434            return '', "Duplicate Course ID: %s" % did
1435        c = getattr(d,course_id,None)
1436        if c is not None:
1437            return '', "Duplicate Course ID: %s" % did
1438        try:
1439            d.invokeFactory('Course', course_id)
1440        except BadRequest,E:
1441            return '', "%s" % E
1442        self._v_course_list.append(course_id)
1443        c = getattr(d,course_id)
1444        c.getContent().edit(mapping=mapping)
1445        return course_id,''
1446    ###)
1447
1448    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1449    def mass_edit_course(self,mapping):
1450        #import pdb;pdb.set_trace()
1451        course_id = mapping.get('code')
1452        res = self.portal_catalog(id=course_id)
1453        if not res:
1454            return '', "No Course with ID: %s" % course_id
1455        c = res[0].getObject()
1456        c.getContent().edit(mapping=mapping)
1457        return course_id,''
1458    ###)
1459
1460    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1461    def mass_create_certificate(self,mapping):
1462        if getattr(self,'_v_certificate_list',None) is None:
1463            self._v_certificate_list = []
1464        if getattr(self,'_v_department_certificates',None) is None:
1465            res = self.portal_catalog(portal_type = "Department")
1466            self._v_department_certificates = {}
1467            for d in res:
1468                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1469        did = mapping['department_code']
1470        d = self._v_department_certificates.get(did,None)
1471        if d is None:
1472            return '', "No Department with ID: %s" % did
1473        certificate_id = mapping.get('code')
1474        if certificate_id in self._v_certificate_list:
1475            return '', "Duplicate Certificate ID: %s" % did
1476        c = getattr(d,certificate_id,None)
1477        if c is not None:
1478            return '', "Duplicate Certificate ID: %s" % did
1479        try:
1480            d.invokeFactory('Certificate', certificate_id)
1481        except BadRequest,E:
1482            return '', "%s" % E
1483        self._v_certificate_list.append(certificate_id)
1484        c = getattr(d,certificate_id)
1485        c.getContent().edit(mapping=mapping)
1486        return certificate_id,''
1487    ###)
1488
1489    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1490    def mass_edit_certificate(self,mapping):
1491        #import pdb;pdb.set_trace()
1492        certificate_id = mapping.get('code')
1493        res = self.portal_catalog(id=certificate_id)
1494        if not res:
1495            return '', "No Certificate with ID: %s" % did
1496        c = res[0].getObject()
1497        c.getContent().edit(mapping=mapping)
1498        return certificate_id,''
1499    ###)
1500
1501    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1502    def mass_create_application(self,mapping):
1503        #import pdb;pdb.set_trace()
1504        reg_no = mapping.get('reg_no')
1505        try:
1506            self.applicants_catalog.addRecord(**mapping)
1507        except ValueError:
1508            return '', "applicant record with reg_no %s already exists" % reg_no
1509        return reg_no,''
1510    ###)
1511
1512    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1513    def mass_edit_application(self,mapping):
1514        #import pdb;pdb.set_trace()
1515        reg_no = mapping.get('reg_no')
1516        status = mapping.get('status')
1517        res = self.applicants_catalog(reg_no = reg_no)
1518        if len(res):
1519            if res[0].status == 'created' and status != 'created':
1520                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1521            elif status == 'created' and res[0].status != 'created':
1522                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1523            else:
1524                self.applicants_catalog.modifyRecord(**mapping)
1525                return reg_no,''
1526        else:
1527            return '', "applicant record with reg_no %s does not exist" % reg_no
1528
1529    ###)
1530
1531    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1532    def mass_create_course_result(self,mapping):
1533        #import pdb;pdb.set_trace()
1534        students_folder = self.portal_url.getPortalObject().campus.students
1535        if getattr(self,'_v_courses',None) is None:
1536            res = self.courses_catalog()
1537            self._v_courses = {}
1538            for brain in res:
1539                self._v_courses[brain.code] = brain
1540        if getattr(self,'_v_level_created',None) is None:
1541            self._v_level_created = []
1542        course_id = mapping.get('code')
1543        if course_id not in self._v_courses.keys():
1544            return '', "No course with ID: %s" % course_id
1545        id_key = ''
1546        for id_key in ('student_id','matric_no'):
1547            id_field = mapping.get(id_key,None)
1548            if id_field is not None:
1549                student_id = id_field
1550                break
1551        query = Eq(id_key,id_field)
1552        res = self.students_catalog.evalAdvancedQuery(query)
1553        if not res:
1554            return '', "no student with %(id_key)s %(id_field)s" % vars()
1555        student_rec = res[0]
1556        if id_field != "student_id":
1557            mapping['student_id'] = student_rec.id
1558        student_id = student_rec.id
1559        level_id = mapping['level_id']
1560        code = mapping['code']
1561        if student_id not in self._v_level_created:
1562            try:
1563                context = getattr(getattr(students_folder,
1564                                          "%(student_id)s" % vars()),
1565                                  'study_course')
1566            except:
1567                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1568            if level_id not in context.objectIds():
1569                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1570                level = getattr(context,"%(level_id)s" % vars())
1571                self.portal_workflow.doActionFor(level,'open')
1572                # the session string must not be copied into the level object
1573                current_verdict = getattr(student_rec,'current_verdict','')
1574                current_session = getattr(student_rec,'current_session','')
1575                if current_verdict and student_rec.current_level == level_id:
1576                    level.getContent().edit(mapping={'verdict': "%s" %
1577                                                     current_verdict,
1578                                                     'session': "%s" %
1579                                                     current_session,
1580                                                    })
1581                    self.portal_workflow.doActionFor(level,'close')
1582            self._v_level_created += student_id,
1583        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1584        for k in ('semester','credits',):
1585            mapping[k] = getattr(self._v_courses[course_id],k)
1586        try:
1587            self.course_results.addRecord(**mapping)
1588        except ValueError:
1589            return '', "course result already exists: %s" % key
1590        return key,''
1591    ###)
1592
1593    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1594    def mass_edit_course_result(self,mapping):
1595        #import pdb;pdb.set_trace()
1596        # if getattr(self,'_v_courses',None) is None:
1597        #     res = self.courses_catalog()
1598        #     self._v_courses = {}
1599        #     for brain in res:
1600        #         self._v_courses[brain.code] = brain
1601        # course_id = mapping.get('code')
1602        # if course_id not in self._v_courses.keys():
1603        #     return '', "No course with ID: %s" % did
1604        id_key = ''
1605        for id_key in ('student_id','matric_no'):
1606            id_field = mapping.get(id_key,None)
1607            if id_field is not None:
1608                student_id = id_field
1609                break
1610        query = Eq(id_key,id_field)
1611        res = self.students_catalog.evalAdvancedQuery(query)
1612        if not res:
1613            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1614        if id_field != "student_id":
1615            mapping['student_id'] = res[0].id
1616        if not res:
1617            return '', "no student with %(id_key)s %(id_field)s" % vars()
1618        student_rec = res[0]
1619        student_id = student_rec.id
1620        level_id = mapping['level_id']
1621        code = mapping['code']
1622        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1623        # for k in ('semester',):
1624        #     mapping[k] = getattr(self._v_courses[course_id],k)
1625        try:
1626            self.course_results.modifyRecord(**mapping)
1627        except KeyError:
1628            return '', "No course result to edit: %s" % key
1629        return key,''
1630    ###)
1631
1632    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1633    def mass_create_certificate_course(self,mapping):
1634        if getattr(self,'_v_courses',None) is None:
1635            res = self.courses_catalog()
1636            self._v_courses= [course.code for course in res]
1637        if getattr(self,'_v_ceritficates',None) is None:
1638            res = self.portal_catalog(portal_type = "Certificate")
1639            self._v_certificates = {}
1640            for cert in res:
1641                self._v_certificates[cert.getId] = cert.getObject()
1642        certificate_course_id = mapping.get('code')
1643        if certificate_course_id not in self._v_courses:
1644            return '', "No Course with ID: %s" % certificate_course_id
1645        cert_id = mapping['certificate_code']
1646        cert = self._v_certificates.get(cert_id,None)
1647        if cert is None:
1648            return '', "No Certificate with ID: %s" % cert_id
1649        level_id = mapping.get('level')
1650        level = getattr(cert,level_id,None)
1651        if level is None:
1652            cert.invokeFactory('StudyLevel', level_id)
1653            level = getattr(cert,level_id,None)
1654        elif hasattr(level,certificate_course_id):
1655            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1656            (certificate_course_id,cert_id,level_id)
1657        level.invokeFactory('CertificateCourse', certificate_course_id)
1658        c = getattr(level,certificate_course_id)
1659        c.getContent().edit(mapping=mapping)
1660        return certificate_course_id,''
1661    ###)
1662
1663    field2types_student = {   ###(
1664                      'StudentApplication':
1665                          {'id': 'application',
1666                           'title': 'Application Data',
1667                           'wf_transition_return': 'close',
1668                           'wf_transition_admit': 'remain',
1669                           'fields':
1670                             ('jamb_reg_no',
1671                              'entry_mode',
1672                              'entry_session',
1673                              'jamb_score',
1674                              'app_email',
1675                              'jamb_age',
1676                              'jamb_state',
1677                              'jamb_lga',
1678                              'jamb_sex',
1679                              )
1680                              },
1681                      #'StudentPume':
1682                      #    {'id': 'pume',
1683                      #     'title': 'Pume Data',
1684                      #     'wf_transition_return': 'close',
1685                      #     'wf_transition_admit': 'close',
1686                      #     'fields':
1687                      #       ('pume_score',
1688                      #        )
1689                      #        },
1690                      'StudentClearance':
1691                          {'id': 'clearance',
1692                           'title': 'Clearance/Eligibility Record',
1693                           'wf_transition_return': 'close',
1694                           'wf_transition_admit': 'remain',
1695                           'fields':
1696                             ('matric_no',
1697                              'nationality',
1698                              'lga',
1699                              'birthday',
1700                              )
1701                              },
1702                         'StudentPersonal':
1703                          {'id': 'personal',
1704                           'title': 'Personal Data',
1705                           'wf_transition_return': 'open',
1706                           'wf_transition_admit': 'remain',
1707                           'fields':
1708                             ('firstname',
1709                              'middlename',
1710                              'lastname',
1711                              'sex',
1712                              'email',
1713                              'phone',
1714                              'perm_address',
1715                              )
1716                              },
1717                         'StudentStudyCourse':
1718                          {'id': 'study_course',
1719                           'title': 'Study Course',
1720                           'wf_transition_return': 'open',
1721                           'wf_transition_admit': 'remain',
1722                           'fields':
1723                             ('study_course',
1724                              'current_level',
1725                              'current_session',
1726                              'current_mode',
1727                              'current_verdict',
1728                              'previous_verdict',
1729                              )
1730                              },
1731                         # 'StudentStudyLevel':
1732                         #  {'id': 'current_level',
1733                         #   'title': '',
1734                         #   'wf_transition_return': 'open',
1735                         #   'wf_transition_admit': 'remain',
1736                         #   'fields':
1737                         #     ('verdict',
1738                         #      'session',
1739                         #      )
1740                         #      },
1741                         'PaymentsFolder':
1742                          {'id': 'payments',
1743                           'title': 'Payments',
1744                           'wf_transition_return': 'open',
1745                           'wf_transition_admit': 'open',
1746                           'fields':
1747                             ()
1748                              },
1749                         }
1750    ###)
1751
1752    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1753    def mass_create_student(self,mapping):
1754        "create student records due import"
1755        logger = logging.getLogger('WAeUPTool.mass_create_student')
1756        students_folder = self.portal_url.getPortalObject().campus.students
1757        jamb_reg_no = mapping.get('jamb_reg_no',None)
1758        if jamb_reg_no:
1759            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1760            if res:
1761                return '',"jamb_reg_no exists"
1762        matric_no = mapping.get('matric_no',None)
1763        if matric_no:
1764            res = self.students_catalog(matric_no = matric_no)
1765            if res:
1766                return '',"matric_no exists"
1767        sid = self.waeup_tool.generateStudentId('?')
1768        students_folder.invokeFactory('Student', sid)
1769        student_obj = getattr(students_folder,sid)
1770        f2t = self.field2types_student
1771        d = {}
1772        transition = mapping.get('reg_transition','admit')
1773        if transition not in ('admit','return'):
1774            return '',"no valid transition provided"
1775        for pt in f2t.keys():
1776            student_obj.invokeFactory(pt,f2t[pt]['id'])
1777            sub_obj = getattr(student_obj,f2t[pt]['id'])
1778            sub_doc = sub_obj.getContent()
1779            d['Title'] = f2t[pt]['title']
1780            for field in f2t[pt]['fields']:
1781                d[field] = mapping.get(field,'')
1782
1783            if pt == "StudentApplication":
1784                #d['jamb_sex']  = 'M'
1785                #if mapping.get('sex'):
1786                #    d['jamb_sex']  = 'F'
1787                d['jamb_firstname'] = mapping.get('firstname',None)
1788                d['jamb_middlename'] = mapping.get('middlename',None)
1789                d['jamb_lastname'] = mapping.get('lastname',None)
1790
1791            # if pt == "StudyCourse":
1792            #     for von,zu in (('entry_mode','current_mode'),
1793            #                    ('entry_session','current_session')):
1794            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1795            #             d[zu] = mapping[von]
1796            sub_doc.edit(mapping = d)
1797
1798            #import pdb;pdb.set_trace()
1799            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1800            if new_state != "remain":
1801                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1802        self.portal_workflow.doActionFor(student_obj,transition)
1803        student_obj.manage_setLocalRoles(sid, ['Owner',])
1804        return sid,''
1805    ###)
1806
1807    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1808    def mass_edit_student(self,mapping):
1809        wftool = self.portal_workflow
1810        "edit student records due import"
1811        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1812        students_folder = self.portal_url.getPortalObject().campus.students
1813        sid = mapping.get('id',None)
1814        jamb_reg_no = mapping.get('jamb_reg_no',None)
1815        matric_no = mapping.get('matric_no',None)
1816        editable_keys = mapping.keys()
1817        if sid:
1818            res = self.students_catalog(id = sid)
1819            if not res:
1820                return '',"no student with id %s" % sid
1821            if matric_no and res[0].matric_no and\
1822              matric_no != res[0].matric_no:
1823                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1824            if jamb_reg_no and res[0].jamb_reg_no and\
1825              jamb_reg_no != res[0].jamb_reg_no:
1826                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1827        elif jamb_reg_no:
1828            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1829            if not res:
1830                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1831            editable_keys.remove('jamb_reg_no')
1832        elif matric_no:
1833            res = self.students_catalog(matric_no = matric_no)
1834            if not res:
1835                return '',"no student with matric_no %s" % matric_no
1836            editable_keys.remove('matric_no')
1837
1838        ## included only to change wf state from admitted to returning
1839        #if res[0].review_state not in ('admitted','objection_raised'):
1840        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1841        ## end inclusion
1842
1843        sid = res[0].id
1844        student_obj = getattr(students_folder,sid)
1845        f2t = self.field2types_student
1846        d = {}
1847        #import pdb;pdb.set_trace()
1848        any_change = False
1849        #special treatment for StudentStudyLevel
1850        d['verdict']  = mapping.get('current_verdict','')
1851        d['session']  = mapping.get('current_session','')
1852        current_level = mapping.get('current_level','')
1853        while d['session'] and d['verdict'] and current_level:
1854            sub_obj = getattr(student_obj,'study_course',None)
1855            if sub_obj is None:
1856                break
1857            level_obj = getattr(sub_obj,current_level,None)
1858            if  level_obj is None:
1859                break
1860            any_change = True
1861            level_obj.getContent().edit(mapping = d)
1862            try:
1863                wftool.doActionFor(level_obj,'close')
1864            except:
1865                pass
1866            break
1867        for pt in f2t.keys():
1868            #if pt == "StudentApplication":
1869            #    d['jamb_sex']  = 'M'
1870            #    if mapping.get('sex'):
1871            #        d['jamb_sex']  = 'F'
1872            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1873            if intersect and pt not in ('StudentStudyLevel',):
1874                object_id = f2t[pt]['id']
1875                sub_obj = getattr(student_obj,object_id,None)
1876                if sub_obj is None:
1877                    try:
1878                        student_obj.invokeFactory(pt,object_id)
1879                    except:
1880                        continue
1881                    sub_obj = getattr(student_obj,object_id)
1882                    if f2t[pt]['title'] != '':
1883                        d['Title'] = f2t[pt]['title']
1884                sub_doc = sub_obj.getContent()
1885                for field in intersect:
1886                    changed = False
1887                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1888                        any_change = True
1889                        changed = True
1890                        d[field] = mapping.get(field,'')
1891                    if changed:
1892                        sub_doc.edit(mapping = d)
1893
1894
1895        ## included only to change wf state from admitted to returning
1896        #    if res[0].review_state in ('admitted','objection_raised'):
1897        #        new_state = f2t[pt]['wf_transition_return']
1898        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1899        #        if sub_obj and new_state != "remain":
1900        #            try:
1901        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1902        #            except:
1903        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1904        #                pass
1905        #if res[0].review_state in ('admitted','objection_raised'):
1906        #    wfaction = 'return'
1907        #    try:
1908        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1909        #        logger.info('%s, wf state changed' % sid)
1910        #        any_change = True
1911        #    except:
1912        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1913        #        pass
1914        ## end inclusion
1915
1916
1917        if any_change:
1918            return sid,''
1919        else:
1920            return sid,'not modified'
1921    ###)
1922
1923    security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
1924    def mass_edit_verdict(self,mapping):
1925        wftool = self.portal_workflow
1926        "edit student verdicts"
1927        logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
1928        students_folder = self.portal_url.getPortalObject().campus.students
1929        student_id = mapping.get('id',None)
1930        matric_no = mapping.get('matric_no',None)
1931        editable_keys = mapping.keys()
1932        if student_id:
1933            student_record = self.students_catalog.getRecordById(student_id)
1934            if student_record is None:
1935                return '',"no student with id %s" % student_id
1936            if matric_no and student_record.matric_no and\
1937              matric_no != student_record.matric_no:
1938                return '', 'student %s matric_no %s does not match %s' % (student_record.id,
1939                                                                          student_record.matric_no,
1940                                                                          matric_no)
1941        elif matric_no:
1942            res = self.students_catalog(matric_no = matric_no)
1943            if not res:
1944                return '',"no student with matric_no %s" % matric_no
1945            student_record = res[0]
1946            editable_keys.remove('matric_no')
1947
1948        student_id = student_record.id
1949        student_review_state =  student_record.review_state
1950        if student_review_state not in ('courses_validated','returning'):
1951            return '',"student %s in wrong review_state %s" % (student_id, student_review_state)
1952        student_obj = getattr(students_folder,student_id)
1953        f2t = self.field2types_student
1954        d = {}
1955        #import pdb;pdb.set_trace()
1956        any_change = False
1957        #special treatment for StudentStudyLevel
1958        current_session = mapping.get('current_session','')
1959        if current_session and student_record.session != current_session:
1960            return '','student_id: %s imported session %s does not match current_session %s' % (student_id,
1961                                                                                        current_session,
1962                                                                                        student_record.session)
1963
1964        current_level = mapping.get('current_level','')
1965        if current_level and student_record.level != current_level:
1966            return '','student_id %s: imported level %s does not match current_level %s' % (student_id,
1967                                                                                    current_level,
1968                                                                                    student_record.level)
1969        study_course_obj = getattr(student_obj,'study_course',None)
1970        if study_course_obj is None:
1971            return '','student_id %s: no study_course object' % student_id
1972        level_obj = getattr(study_course_obj,current_level,None)
1973        if  level_obj is None:
1974            return '','student_id %s: no study_level object for level %s' % (student_id,
1975                                                                             current_level)
1976        verdict = d['verdict'] = d['current_verdict']  = mapping.get('current_verdict','')
1977        if verdict == student_record.verdict:
1978            return '','student_id %s: verdict already set to %s' % (student_id,
1979                                                                    verdict)
1980
1981        level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
1982        if level_review_state != "closed":
1983            return '','student_id %s: level %s is not closed' % (student_id,
1984                                                                 current_level)
1985
1986        study_course_obj.getContent().edit(mapping = d)
1987        level_obj.getContent().edit(mapping = d)
1988        if student_review_state != "returning":
1989            wftool.doActionFor(student_obj,'return')
1990        try:
1991            wftool.doActionFor(level_obj,'close')
1992        except:
1993            pass
1994        return student_id,''
1995    ###)
1996
1997    security.declareProtected(ModifyPortalContent,"importData")###(
1998    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1999        """load data from CSV values"""
2000        import transaction
2001        import random
2002
2003        pm = self.portal_membership
2004        member = pm.getAuthenticatedMember()
2005
2006        logger = logging.getLogger('WAeUPTool.importData')
2007        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2008        students_folder = self.portal_url.getPortalObject().campus.students
2009        start = True
2010        tr_count = 0
2011        total_imported = 0
2012        total_not_imported = 0
2013        total = 0
2014        iname = "import_%s" % name
2015        if iname in ('import_application','import_course_result',):
2016            commit_after = 2000
2017        else:
2018            commit_after = 100
2019        stool = getToolByName(self, 'portal_schemas')
2020        ltool = getToolByName(self, 'portal_layouts')
2021        schema = stool._getOb(iname)
2022        if schema is None:
2023            em = 'No such schema %s' % iname
2024            logger.error('No such schema %s' % iname)
2025            return em
2026        layout = ltool._getOb(iname)
2027        if layout is None:
2028            em = 'No such layout %s' % iname
2029            logger.error(em)
2030            return em
2031        validators = {}
2032        for widget in layout.keys():
2033            validators[widget] = layout[widget].validate
2034        mode = "create"
2035        if edit:
2036            mode = "edit"
2037        importer_name = "mass_%(mode)s_%(name)s" % vars()
2038        importer = getattr(self, '%s' % importer_name,None)
2039        if importer is None:
2040            em = 'No importer function %s' % importer_name
2041            logger.error(em)
2042            return em
2043        not_imported = []
2044        imported = []
2045        try:
2046            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
2047        except:
2048            em = 'Error reading %s.csv' % filename
2049            logger.error(em)
2050            return em
2051        for item in items:
2052            #import pdb;pdb.set_trace()
2053            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2054            if start:
2055                start = False
2056                adapters = [MappingStorageAdapter(schema, item)]
2057                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2058                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
2059                import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
2060                diff2schema = set(import_keys).difference(set(schema.keys()))
2061                diff2layout = set(import_keys).difference(set(layout.keys()))
2062                if diff2schema:
2063                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2064                    return em
2065                if mode == "create":
2066                    required_keys = [layout.getIdUnprefixed(id)
2067                                     for id,widget in layout.objectItems()
2068                                     if widget.is_required]
2069                    if not set(required_keys).issubset(set(import_keys)):
2070                        diff2import = set(required_keys).difference(set(import_keys))
2071                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2072                        return em
2073                s = ','.join(['"%s"' % fn for fn in import_keys])
2074                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
2075                s = '"id",' + s
2076                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
2077                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2078                format_error = format + ',"%(Error)s"'
2079                format = '"%(id)s",'+ format
2080
2081            dm = DataModel(item, adapters,context=self)
2082            ds = DataStructure(data=item,datamodel=dm)
2083            error_string = ""
2084            total += 1
2085            error_count = 0
2086            for k in import_keys:
2087                if not validators[k](ds,mode=mode):
2088                    if error_count:
2089                        error_string += ' ++ '
2090                    error_string += "%s: %s" % (k,
2091                                                  self.translation_service(ds.getError(k),
2092                                                                           ds.getErrorMapping(k)))
2093                    error_count += 1
2094            if not error_string:
2095                temp_item = item.copy()
2096                temp_item.update(dm)
2097                temp_item['id'],error = importer(temp_item)
2098                if error:
2099                    error_string += error
2100                else:
2101                    item = temp_item
2102            if error_string:
2103                item['Error'] = error_string
2104                not_imported.append(format_error % item)
2105                total_not_imported += 1
2106            else:
2107                em = format % item
2108                imported.append(em)
2109                tr_count += 1
2110                total_imported += 1
2111                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
2112
2113            if total and not total % commit_after:
2114                transaction.commit()
2115                if len(not_imported) > 0:
2116                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2117                             '\n'.join(not_imported) + '\n')
2118                    not_imported = []
2119                if len(imported) > 0:
2120                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2121                             '\n'.join(imported) + '\n')
2122                    imported = []
2123                em = '%d transactions committed\n' % (tr_count)
2124                regs = []
2125                logger.info(em)
2126                tr_count = 0
2127                #if total > 100:
2128                #    return
2129        if len(imported) > 0:
2130            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2131                                                '\n'.join(imported))
2132        if len(not_imported) > 0:
2133            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2134                                                '\n'.join(not_imported))
2135        em = "Finished import from %s.csv: %d imported, %d not imported (of total %d)" % (filename,total_imported,total_not_imported,total)
2136        logger.info(em)
2137        return em
2138    ###)
2139
2140    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2141    def moveImagesToFS(self,student_id="O738726"):
2142        "move the images to the filesystem"
2143        images_dir = getImagesDir(student_id)
2144        #images_dir = os.path.join("%s" % images_base,student_id)
2145        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2146        stool = getToolByName(self, 'portal_schemas')
2147        schemas = ['student_application',
2148                   'student_clearance',
2149                   ]
2150        created = False
2151        for schema_id in schemas:
2152            schema = stool._getOb(schema_id)
2153            object = getattr(student_folder,schema_id[len('student_'):],None)
2154            if object is None:
2155                continue
2156            doc = object.getContent()
2157            for key in schema.keys():
2158                if schema[key].meta_type != "CPS Image Field":
2159                    continue
2160                #import pdb;pdb.set_trace()
2161                image = getattr(doc,key,None)
2162                if not image or not hasattr(image,"data"):
2163                    continue
2164                if not created:
2165                    if not os.path.exists(images_dir):
2166                        os.mkdir(images_dir)
2167                    created = True
2168                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2169                open(filename,"wb").write(str(image.data))
2170    ###)
2171
2172    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2173    def movePassportToFS(self,student_id="O738726"):
2174        "move the passports to the filesystem"
2175        images_dir = os.path.join("%s" % i_home,'passports')
2176        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2177        stool = getToolByName(self, 'portal_schemas')
2178        schemas = ['student_application',
2179                   #'student_clearance',
2180                   ]
2181        created = False
2182        for schema_id in schemas:
2183            schema = stool._getOb(schema_id)
2184            object = getattr(student_folder,schema_id[len('student_'):],None)
2185            if object is None:
2186                continue
2187            doc = object.getContent()
2188            for key in schema.keys():
2189                if schema[key].meta_type != "CPS Image Field":
2190                    continue
2191                #import pdb;pdb.set_trace()
2192                image = getattr(doc,key)
2193                if not hasattr(image,"data"):
2194                    continue
2195                if not created:
2196                    if not os.path.exists(images_dir):
2197                        os.mkdir(images_dir)
2198                    created = True
2199                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2200                open(filename,"wb").write(str(image.data))
2201    ###)
2202
2203InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.