source: WAeUP_SRP/base/WAeUPTool.py @ 3079

Last change on this file since 3079 was 3071, checked in by Henrik Bettermann, 17 years ago

fix #74 fceokene
  • Property svn:keywords set to Id
File size: 92.9 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3071 2008-01-31 07:07:53Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64
65class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66    """WAeUP tool"""
67
68    id = 'waeup_tool'
69    meta_type = 'WAeUP Tool'
70    _actions = ()
71    security = ClassSecurityInfo()
72    security.declareObjectProtected(View)
73    manage_options = ( ActionProviderBase.manage_options
74                     + SimpleItem.manage_options
75                     )
76
77    security.declareProtected(View,'re_split') ###(
78    def re_split(self,split_string,string):
79        return re.split(split_string,string)
80    ###)
81
82    security.declareProtected(View,'difference') ###(
83    def difference(self,l1,l2):
84        return set(l1).difference(set(l2))
85    ###)
86
87    def rwrite(self,s): ###(
88        response = self.REQUEST.RESPONSE
89        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
90        response.write("%s<br />\r\n" % s)
91    ###)
92
93    def addtodict(self,d,key,item): ###(
94        d[key].append(item)
95        return d[key]
96    ###)
97
98    def sleep(self,secs): ###(
99        "sleep"
100        import time
101        time.sleep(secs)
102        return
103
104###)
105
106    security.declareProtected(View,'updateRoleMappingsFor') ###(
107    def updateRoleMappingsFor(self,wf_definition,ob):
108        "do so for public"
109        wf_def = getattr(self.portal_workflow,wf_definition)
110        wf_def.updateRoleMappingsFor(ob)
111    ###)
112
113    security.declareProtected(View,'getStatesLgas') ###(
114    def getStatesLgas(self):
115        """return lga info"""
116        voc = getattr(self.portal_vocabularies,'local_gov_areas')
117        states = []
118        lgas  = []
119        d = {}
120        wd = {}
121        for k,v in voc.items():
122            parts = v.split(' / ')
123            if len(parts) == 1:
124                state = parts[0].lower()
125                lga = ""
126            elif len(parts) == 2:
127                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
128                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
129            else:
130                continue
131            if state not in states:
132                states.append(state)
133            if lga not in lgas:
134                lgas.append(lga)
135            words = re.split('[^a-zA-Z0-9/]',k)
136            words.sort()
137            wd[k] = words
138            d[k] = v
139        mapping = {}
140        mapping['word_dict'] = wd
141        mapping['lga_dict'] = d
142        mapping['states'] = states
143        mapping['lgas'] = lgas
144        return mapping
145    ###)
146
147    security.declareProtected(View,'findLga') ###(
148    def findLga(self,words,words_dict):
149        words = re.split('[^a-zA-Z0-9/]',words)
150        lga_words = []
151        for word in words:
152            if word:
153                lga_words += word.strip().lower(),
154        lga_words.sort()
155        state_lga = ''
156        while not state_lga:
157            for k,l in words_dict.items():
158                if lga_words == l:
159                    state_lga = k
160                    break
161            break
162        return state_lga
163    ###)
164    security.declareProtected(View,'getAccessInfo') ###(
165    def getAccessInfo(self,context):
166        "return a dict with access_info"
167        logger = logging.getLogger('WAeUPTool.getAccessInfo')
168        mtool = self.portal_membership
169        member = mtool.getAuthenticatedMember()
170        member_id = str(member)
171        info = {}
172        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
173        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
174        is_staff = info['is_staff'] = not is_anonymous and not is_student
175        roles = member.getRolesInContext(context)
176        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
177                                                                            "SectionManager" in roles or
178                                                                            "Manager" in roles)
179        is_allowed = info['is_allowed'] = not is_anonymous
180        requested_id = context.getStudentId()
181        student_id  = None
182        if is_allowed:
183            if not is_student and requested_id:
184                student_id  = requested_id
185            elif is_anonymous or member_id != requested_id:
186                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
187            else:
188                student_id = member_id
189        info['student_id'] = student_id
190        return info
191    ###)
192
193    security.declareProtected(ModifyPortalContent,'openLog') ###(
194    def openLog(self,name):
195        """open a log file"""
196        version = 1
197        path = "%s/log/%s_%d.log" % (i_home,name,version)
198        while os.path.exists(path):
199            version += 1
200            path = "%s/log/%s_%d.log" % (i_home,name,version)
201        log = open(path,"w")
202        return log
203    ###)
204
205    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
206    def bypassQueueCatalog(self,enable=True):
207        """bypass the QueueCatalog by setting all indexes to process imediate,
208        if enable is True (default) the old settings are restored
209        """
210
211    ###)
212
213    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
214    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
215        """measure Object access Time"""
216        import random
217        if hasattr(self,'portal_catalog_real'):
218            aq_portal = self.portal_catalog_real.evalAdvancedQuery
219        else:
220            aq_portal = self.portal_catalog.evalAdvancedQuery
221        nr_pts = int(nr_pts)
222        probe = int(probe)
223        intervall = probe/10
224        objects = ("application","clearance","personal")
225        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
226        #i = random.randrange(num_objects)
227        count = 0
228        found = 0
229        not_found = 0
230        t_found = 0
231        t_not_found = 0
232        time_found = time_not_found = 0.0
233        t_time_found = t_time_not_found = 0.0
234        accessed = []
235        t_min = 1000
236        t_max = 0
237        #import pdb;pdb.set_trace()
238        students = self.portal_catalog(portal_type="Student")
239        num_students = len(students)
240        if method == "d":
241            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
242            res = aq_portal(query)
243            brains = {}
244            for r in res:
245                sid = r.relative_path.split('/')[-2]
246                if brains.has_key(sid):
247                    brains[sid][r.portal_type] = r
248                else:
249                    brains[sid] = {r.portal_type : r}
250            brains_list = brains.keys()
251            num_objects = len(brains_list)
252        else:
253            num_objects = num_students
254        print "="*40
255        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
256                                                                        probe,
257                                                                        nr_pts,
258                                                                        num_objects)
259        print "nr found/not time found/not min/max"
260        elapse = time.time()
261        i_elapse = time.time()
262        c_elapse = time.clock()
263        for c in range(1,probe + 1):
264            i = random.randrange(num_objects)
265            if method in ('a','b','c'):
266                student_brain = students[i]
267            elif method == "d":
268                #import pdb;pdb.set_trace()
269                student_brain = brains[brains_list[i]]
270            if method == "c":
271                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
272                res = aq_portal(query)
273                this_portal_types = [r.portal_type for r in res]
274            for i in range(nr_pts):
275                oid = objects[i]
276                if method == "a":
277                    try:
278                        student_path = student_brain.getPath()
279                        path = "%s/%s" % (student_path,oid)
280                        doc = self.unrestrictedTraverse(path).getContent()
281                        found += 1
282                        i_time = time.time() - i_elapse
283                        time_found += i_time
284                    except:
285                        not_found += 1
286                        i_time = time.time() - i_elapse
287                        time_not_found += i_time
288                        pass
289                elif method == "b":
290                    try:
291                        student_object = student_brain.getObject()
292                        doc = getattr(student_object,oid).getContent()
293                        found += 1
294                        i_time = time.time() - i_elapse
295                        time_found += i_time
296                    except:
297                        i_time = time.time() - i_elapse
298                        time_not_found += i_time
299                        not_found += 1
300                        pass
301                elif method == "c":
302                    if portal_types[i] in this_portal_types:
303                        found += 1
304                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
305                        i_time = time.time() - i_elapse
306                        time_found += i_time
307                    else:
308                        not_found += 1
309                        i_time = time.time() - i_elapse
310                        time_not_found += i_time
311                elif method == "d":
312                    if student_brain.has_key(portal_types[i]):
313                        found += 1
314                        doc = student_brain[portal_types[i]].getObject().getContent()
315                        i_time = time.time() - i_elapse
316                        time_found += i_time
317                    else:
318                        not_found += 1
319                        i_time = time.time() - i_elapse
320                        time_not_found += i_time
321                i_elapse = time.time()
322            if c and (c % intervall == 0):
323                #i_time = time.time() - i_elapse
324                t_per = 0.0
325                if found:
326                    t_per = time_found/found
327                if t_per > t_max:
328                    t_max = t_per
329                if t_per > 0.0 and t_per < t_min:
330                    t_min = t_per
331                itf = 0.0
332                if found:
333                    itf = time_found/found
334                itnf = 0.0
335                if not_found :
336                    itnf = time_not_found / not_found
337                interval_time = time_found + time_not_found
338                s = "%(c)d: %(found)d/%(not_found)d " % vars()
339                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
340                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
341                print s
342                t_found += found
343                t_not_found += not_found
344                t_time_found += time_found
345                t_time_not_found += time_not_found
346                time_found = time_not_found = 0.0
347                found = not_found = 0
348        # t_found += found
349        # t_not_found += not_found
350        elapse = time.time() - elapse
351        itf = 0.0
352        if t_found:
353            itf = t_time_found/t_found
354        itnf = 0.0
355        if t_not_found:
356            itnf = t_time_not_found / t_not_found
357        #c_elapse = time.clock() - c_elapse
358        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
359        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
360        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
361        print "-"*40
362        print s
363        rel_found = float(t_found)/probe
364        rel_not_found = float(t_not_found)/probe
365        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
366        print estimated_total_time
367    ###)
368
369    security.declareProtected(ModifyPortalContent,'writeLog') ###(
370    def writeLog(self,logfile,s):
371        """write to the log file"""
372        logfile.write(s)
373
374###)
375
376    def generateStudentId(self,letter): ###(
377        import random
378        r = random
379        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
380        if letter == '?':
381            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
382        sid = "%c%d" % (letter,r.randint(99999,1000000))
383        students = self.portal_url.getPortalObject().campus.students
384##        while hasattr(students, sid):
385##            sid = "%c%d" % (letter,r.randint(99999,1000000))
386        while self.students_catalog(id = sid):
387            sid = "%c%d" % (letter,r.randint(99999,1000000))
388        return sid
389    ###)
390
391    def generatePassword(self,s=None): ###(
392        import random
393        r = random
394        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
395        if s is None:
396            s = 'abcdefghklmnpqrstuvwxy23456789'
397        pw = ''
398        while len(pw) < 6:
399            pw += r.choice(s)
400        return pw
401    ###)
402
403    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
404    def dumpSchoolfeePayments(self):
405        "dump paid schoolfees"
406        mtool = self.portal_membership
407        member = mtool.getAuthenticatedMember()
408        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
409        aq_student = self.students_catalog.evalAdvancedQuery
410        query = In('review_state',('schoolfee_paid',
411                                   'courses_registered',
412                                   'courses_validated',
413                                   ))
414        res = aq_student(query)
415        #import pdb;pdb.set_trace()
416        l = []
417        logger.info("start for %d" % len(res))
418        count = 1
419        log_after = 100
420        for student in res:
421            if not count % log_after:
422                logger.info("processed %d total %d" % (log_after,count))
423            count += 1
424            fee_dict =self.getSchoolFee(student)
425            fulltime = student.mode.endswith('_ft')
426            d = {}
427            d['student_id'] = student.id
428            d['name'] = student.name
429            d['amount'] = fee_dict.get(new_returning)
430            l += d,
431        csv_name = self.dumpListToCSV(l,'payments')
432        logger.info('%s dumped payments to %s' % (member,export_file))
433    ###)
434
435    security.declarePublic('dumpListToCSV') ###(
436    def dumpListToCSV(self,l,filename,fields=None):
437        """dump a list of dicts to a CSV file"""
438        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
439        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
440        if fields is None:
441            fields = l[0].keys()
442        headline = ','.join(fields)
443        out = open(export_file,"wb")
444        out.write(headline +'\n')
445        out.close()
446        out = open(export_file,"a")
447        csv_writer = csv.DictWriter(out,fields,)
448        csv_writer.writerows(l)
449        return export_file
450    ###)
451
452    security.declareProtected(ManagePortal, 'listMembers') ###(
453    def listMembers(self):
454        "list all members"
455        mtool = self.portal_membership
456        member = mtool.getAuthenticatedMember()
457        logger = logging.getLogger('WAeUPTool.listMembers')
458        if str(member) not in ('admin','joachim'):
459            logger.info('%s tried to list members' % (member))
460            return None
461        members = self.portal_directories.members
462        all = members.listEntryIdsAndTitles()
463        l = []
464        for user_id,name in all:
465            d = {}
466            d['user_id'] = user_id
467            d['name'] = name
468            d['pw'] = getattr(getattr(members,user_id),'password')
469            d['email'] = getattr(getattr(members,user_id),'email')
470            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
471            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
472            l += d,
473        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
474        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
475        logger.info('%s dumped memberlist to %s' % (member,export_file))
476        fields = l[0].keys()
477        headline = ','.join(fields)
478        out = open(export_file,"wb")
479        out.write(headline +'\n')
480        out.close()
481        out = open(export_file,"a")
482        csv_writer = csv.DictWriter(out,fields,)
483        csv_writer.writerows(l)
484    ###)
485
486    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
487    def removeDeletedDocIds(self, max=1000):
488        """
489        remove deleted docids from repository commit after max
490        """
491        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
492        repository = getToolByName(self, 'portal_repository')
493        pxtool = getToolByName(self, 'portal_proxies')
494        logger.info('start')
495        pxtool_infos = pxtool.getRevisionsUsed()
496        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
497
498        nb_revs = 0
499        docids_d = {} # all docids
500        unused_docids_d = {} # all docids that are unused
501        ids_unused_revs_docids = [] # ids for revs of unused docids
502        unused_ids = [] # ids for unused revs
503        total = 0
504        idlist = repository.objectIds()
505        to_delete = 0
506        found = False
507        for id in idlist:
508            docid, rev = repository._splitId(id)
509            if docid is None:
510                logger.info("invalid doc_id %s" % docid)
511                continue
512            nb_revs += 1
513            if not pxtool_infos.has_key(docid):
514                found = True
515                to_delete += 1
516                unused_ids.append(id)
517            elif not pxtool_infos[docid].has_key(rev):
518                found = True
519                to_delete += 1
520                unused_ids.append(id)
521            if found and not to_delete % max:
522                found = False
523                #import pdb;pdb.set_trace()
524                repository.manage_delObjects(unused_ids)
525                transaction.commit()
526                logger.info('removed %d total %d unused docids ' % (max,to_delete))
527        else:
528            if unused_ids:
529                repository.manage_delObjects(unused_ids)
530                transaction.commit()
531        logger.info('finished removing %d unused docids ' % (to_delete))
532
533
534###)
535
536    security.declareProtected(View,'getCredential') ###(
537    def getCredential(self,student_id):
538        student_entry = getattr(self.portal_directories.students,student_id,None)
539        if not self.isStaff():
540            mtool = self.portal_membership
541            member = mtool.getAuthenticatedMember()
542            logger = logging.getLogger('WAeUPTool.getCredential')
543            logger.info('%s tried to access password of %s' % (member,student_id))
544            return None
545        if student_entry is None:
546            return None
547        return getattr(student_entry,"password","not set")
548    ###)
549
550    security.declarePublic('checkPassword') ###(
551    def checkPassword(self,student_id,password):
552        student_entry = getattr(self.portal_directories.students,student_id,None)
553        if student_entry is None:
554            return False
555        return getattr(student_entry,"password","not set") == password
556    ###)
557
558    security.declarePublic('checkGenericPassword') ###(
559    def checkGenericPassword(self,member_id):
560        member_entry = getattr(self.portal_directories.members,member_id,None)
561        if member_entry is None:
562            return False
563        ltool = getToolByName(self, 'portal_layouts')
564        unsecure_words = ltool._getOb('members')['w__password'].check_words
565        password = getattr(member_entry,"password","not set")
566        is_unsecure = password in unsecure_words
567        if is_unsecure:
568            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
569            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
570        return is_unsecure
571    ###)
572
573    security.declareProtected(ModifyPortalContent,'editPassword') ###(
574    def editPassword(self,student_id,password):
575        "edit a student password"
576        student_entry = getattr(self.portal_directories.students,student_id,None)
577        if student_entry is None:
578            return
579        setattr(student_entry,'password',password)
580    ###)
581
582    security.declareProtected(ModifyPortalContent,'doCommit') ###(
583    def doCommit(self,logger=None):
584        "commit some transactions"
585        transaction.commit()
586    ###)
587
588    security.declarePublic('loadStudentFoto') ###(
589    def loadStudentFoto(self,student,filename,folder):
590        "return a student passport picture"
591        #import pdb;pdb.set_trace()
592        picture ="%s/import/%s/%s" % (i_home,folder,filename)
593        student_id = student.getId()
594        images_dir = getImagesDir(student_id)
595        if not os.path.exists(images_dir):
596            os.mkdir(images_dir)
597        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
598        for extension in ('.jpg','.JPG'):
599            fullname = "%(picture)s%(extension)s" % vars()
600            if os.path.exists(fullname):
601                copy2(fullname,image_name)
602                return "successfully copied passport picture"
603        return "passport picture not found: %s.jpg or .JPG" % picture
604    ###)
605
606    def old____loadStudentFoto(self,student,filename,folder): ###(
607        "return a student passport picture"
608        app = student.application
609        app_doc = app.getContent()
610        #clear = student.clearance
611        #clear_doc = clear.getContent()
612        #matric_no = clear_doc.matric_no.upper()
613        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
614        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
615        #import pdb;pdb.set_trace()
616        if os.path.exists(picture1):
617            file = open(picture1)
618        elif os.path.exists(picture2):
619            file = open(picture2)
620        else:
621            return "passport picture not found %s" % picture1
622        reopened = False
623        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
624            self.portal_workflow.doActionFor(app,'open')
625            reopened = True
626        outfile = file.read()
627        app_doc.manage_addFile('passport',
628                               file=outfile,
629                               title="%s.jpg" % filename)
630        if reopened:
631            self.portal_workflow.doActionFor(app,'close')
632        return "successfully loaded passport picture"
633    ###)
634
635    security.declareProtected(ModifyPortalContent,'createOne') ###(
636    def createOne(self,students_folder,student_brain,letter,commit=False):
637        sid = self.waeup_tool.generateStudentId(letter)
638        students_folder.invokeFactory('Student', sid)
639        student = getattr(students_folder,sid)
640        self.portal_workflow.doActionFor(student,'return')
641        student.manage_setLocalRoles(sid, ['Owner',])
642        matric_no = student_brain.matric_no
643        jamb_reg_no = student_brain.Entryregno
644        self.students_catalog.addRecord(id = sid,
645                                           matric_no = matric_no,
646                                           jamb_reg_no = jamb_reg_no,
647                                           sex = student_brain.Sex == "F",
648                                           name = "%s %s %s" % (student_brain.Firstname,
649                                                                student_brain.Middlename,
650                                                                student_brain.Lastname)
651                                        )
652        if commit:
653            transaction.commit()
654        return sid,jamb_reg_no
655    ###)
656
657    security.declareProtected(ModifyPortalContent,'addStudent') ###(
658    def addStudent(self,dict):
659        students_folder = self.portal_url.getPortalObject().campus.students
660        sid = self.waeup_tool.generateStudentId('?')
661        students_folder.invokeFactory('Student', sid)
662        student_obj = getattr(students_folder,sid)
663        f2t = self.field2types_student
664        #from pdb import set_trace; set_trace()
665        d = {}
666        #d['jamb_sex']  = 'M'
667        #if dict.get('sex'):
668        #    d['jamb_sex']  = 'F'
669
670        entry_session = dict.get('entry_session')
671        if entry_session == self.getSessionId()[0]:
672            wfaction = 'admit'
673            wft = 'wf_transition_admit'
674            password = None
675        else:
676            wfaction = 'return'
677            wft = 'wf_transition_return'
678            password = self.generatePassword()
679            self.makeStudentMember(sid,password)
680
681        for pt in f2t.keys():
682            student_obj.invokeFactory(pt,f2t[pt]['id'])
683            sub_obj = getattr(student_obj,f2t[pt]['id'])
684            sub_doc = sub_obj.getContent()
685            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
686            d['Title'] = f2t[pt]['title']
687            for field in f2t[pt]['fields']:
688                d[field] = dict.get(field,'')
689            sub_doc.edit(mapping = d)
690            new_state = f2t[pt][wft]
691            if new_state != "remain":
692                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
693        self.portal_workflow.doActionFor(student_obj,wfaction)
694        student_obj.manage_setLocalRoles(sid, ['Owner',])
695        return sid,password
696    ###)
697
698    security.declarePublic('getCertificateBrain') ###(
699    def getCertificateBrain(self,cert_id):
700        "do it"
701        res = ZCatalog.searchResults(self.portal_catalog_real,
702                                {'portal_type':"Certificate",
703                                      'id': cert_id})
704        if res:
705            return res[0]
706        return None
707    ###)
708
709    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
710    def get_csv_filenames(self):
711        "do it"
712        files = [file for file in os.listdir("%s/import/" % (i_home))
713                 if file.endswith('.csv') and file.find('imported') == -1]
714        return files
715    ###)
716
717    security.declarePublic('findStudentByMatricelNo') ###(
718    def findStudentByMatricelNo(self,matric_no):
719        "do it"
720        res = ZCatalog.searchResults(self.portal_catalog_real,
721                                {'portal_type':"StudentClearance",
722                                 'SearchableText': matric_no})
723        if res:
724            return res[0]
725        return None
726    ###)
727
728    security.declarePublic('makeStudentMember') ###(
729    def makeStudentMember(self,sid,password='uNsEt'):
730        """make the student a member"""
731        membership = self.portal_membership
732        membership.addMember(sid,
733                             password ,
734                             roles=('Member',
735                                     'Student',
736                                     ),
737                             domains='',
738                             properties = {'memberareaCreationFlag': False,
739                                           'homeless': True},)
740        member = membership.getMemberById(sid)
741        self.portal_registration.afterAdd(member, sid, password, None)
742        #self.manage_setLocalRoles(sid, ['Owner',])
743    ###)
744
745    security.declareProtected(View,'makeStudentData') ###(
746    def makeStudentData(self,student_id,email=None,phone_nr=None):
747        "create Datastructure for a returning Student"
748        #import pdb;pdb.set_trace()
749        logger = logging.getLogger('WAeUPTool.makeStudentData')
750        students_folder = self.portal_url.getPortalObject().campus.students
751        #res = self.students_catalog(id=student_id)
752        #if res:
753        #    st = res[0]
754        #res = self.returning_import(matric_no = st.matric_no)
755        res = self.returning_import(id = student_id)
756        if res:
757            student = res[0]
758        else:
759            logger.info('Id %s not found in returning_import' % student_id)
760            return
761        logger.info('%s creates data structure' % student_id)
762        s_results = self.results_import(matric_no = student.matric_no)
763        if s_results:
764            lnr = self.getLevelFromResultsCosCode(s_results)
765            level = "%d00" % lnr
766            verdict,eligible = self.getVerdict(s_results[0].Verdict)
767            #if eligible:
768            #    level = "%d00" % (lnr + 1)
769        else:
770            logger.info('matric_no %s not found in results_import' % student.matric_no)
771            level = ''
772            verdict = ''
773        #student should not be allowed to perform this transition
774        #wftool = self.portal_workflow
775        #wftool.doActionFor(student,'return')
776        certcode_org = student.Coursemajorcode
777        certcode = makeCertificateCode(certcode_org)
778        certificate_brain = self.getCertificateBrain(certcode)
779        if not certificate_brain:
780            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
781            logger.info(em)
782        matric_no = student.matric_no
783        sid = student_id
784        student_obj = getattr(students_folder,sid)
785        student_obj.invokeFactory('StudentApplication','application')
786        application = student_obj.application
787        self.portal_workflow.doActionFor(application,'open',dest_container=application)
788        da = {'Title': 'Application Data'}
789        student_obj.invokeFactory('StudentPersonal','personal')
790        da['jamb_reg_no'] = student.Entryregno
791        em = self.getEntryMode(student.Entryregno)
792        da['entry_mode'] = em
793        personal = student_obj.personal
794        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
795        dp = {'Title': 'Personal Data'}
796        student_obj.invokeFactory('StudentClearance','clearance')
797        clearance = student_obj.clearance
798        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
799        dc = {'Title': 'Clearance/Eligibility Record'}
800        dc['matric_no'] = matric_no
801        state = student.State
802        lga = student.LGA
803        if state and lga:
804            lga =  state + ' / ' + lga
805        else:
806            lga = "None"
807        da['jamb_lga'] = dc['lga'] = lga
808        da['app_email'] = dp['email'] = email
809        da['app_mobile'] = dp['phone'] = phone_nr
810        dp['firstname'] = student.Firstname
811        dp['middlename'] = student.Middlename
812        dp['lastname'] = student.Lastname
813        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
814        da['jamb_sex'] = student.Sex
815        dp['sex'] = student.Sex == 'F'
816        dp['perm_address'] = student.Permanent_Address
817        application.getContent().edit(mapping=da)
818        self.portal_workflow.doActionFor(application,'close',dest_container=application)
819        personal.getContent().edit(mapping=dp)
820        clearance.getContent().edit(mapping=dc)
821        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
822        #
823        # Study Course
824        #
825        student_obj.invokeFactory('StudentStudyCourse','study_course')
826        studycourse = student_obj.study_course
827        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
828        dsc = {}
829        dsc['study_course'] = certcode
830        dsc['current_level'] = level
831        dsc['current_verdict'] = verdict
832        dsc['current_mode'] = em
833        dsc['current_session'] = '05'
834        studycourse.getContent().edit(mapping=dsc)
835        #
836        # Level
837        #
838##        l = getattr(studycourse,level,None)
839##        if l is None:
840##            studycourse.invokeFactory('StudentStudyLevel', level)
841##            l = getattr(studycourse, level)
842##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
843##            l.getContent().edit(mapping={'Title': "Level %s" % level})
844###)
845
846    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
847    def admitOneStudent(self,brain,entry_session,pin_password):
848        "create Datastructure for an admitted Student"
849        #import pdb;pdb.set_trace()
850        if not hasattr(self,"_v_certificates"):
851            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
852        students_folder = self.portal_url.getPortalObject().campus.students
853        logger = logging.getLogger('WAeUPTool.admitOneStudent')
854        if brain.status != "admitted":
855            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
856            return
857        pin_parts = brain.pin.split('-')
858        if pin_parts and len(pin_parts) != 3:
859            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
860            return
861        if brain.course_admitted not in self._v_certificates:
862            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
863            return
864        if brain.sex not in (True,False):
865            logger.info('sex of %s not available' % (brain.reg_no))
866            return
867        student_id = self.generateStudentId('?')
868        students_folder.invokeFactory('Student', student_id)
869        student_object = getattr(students_folder,student_id)
870        if pin_password:
871            password = pin_parts[2]
872            self.makeStudentMember(student_id,password = password)
873        student_object.manage_setLocalRoles(student_id, ['Owner',])
874        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
875        #
876        # application
877        #
878        student_object.invokeFactory('StudentApplication','application')
879        application = student_object.application
880        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
881        da = {'Title': 'Application Data'}
882        da['jamb_reg_no'] = brain.reg_no
883
884        sex = 'M'
885        if brain.sex:
886            sex = 'F'
887        da['jamb_sex'] = sex
888        da['jamb_age'] = brain.jamb_age
889        #da['app_ac_pin'] = brain.pin
890        da['jamb_lga'] = brain.jamb_lga
891        da['jamb_state'] = brain.jamb_state
892        da['jamb_score'] = brain.aggregate
893        da['app_email'] = brain.email
894        da['app_mobile'] = brain.phone
895        if brain.entry_mode:
896            da['entry_mode'] = brain.entry_mode
897        elif brain.screening_type == 'pume':
898            da['entry_mode'] = 'ume_ft'
899        elif brain.screening_type == 'pde':
900            da['entry_mode'] = 'de_ft'
901        elif brain.screening_type == 'pce':
902            da['entry_mode'] = 'pce'
903        elif brain.screening_type == 'prence':
904            da['entry_mode'] = 'prence'
905        else:
906            da['entry_mode'] = ''
907        da['entry_session'] = entry_session
908        da['jamb_lastname'] = brain.lastname
909        da['jamb_middlename'] = brain.middlenames   # different field names!
910        da['jamb_firstname'] = brain.firstname
911        da['screening_application_date'] = brain.application_date
912        da['date_of_birth'] = brain.date_of_birth
913        da['jamb_first_cos'] = brain.course1
914        da['jamb_second_cos'] = brain.course2
915        da['course3'] = brain.course3
916        da['screening_type'] = brain.screening_type
917        da['screening_score'] = brain.screening_score
918        da['screening_date'] = brain.screening_date
919        da['hq_type'] = brain.hq_type
920        da['hq_grade'] = brain.hq_grade
921        da['aos'] = brain.aos
922
923        application.getContent().edit(mapping=da)
924        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
925        #
926        # personal
927        #
928        student_object.invokeFactory('StudentPersonal','personal')
929        personal = student_object.personal
930        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
931        dp = {'Title': 'Personal Data'}
932        dp['sex'] = brain.sex
933        dp['email'] = brain.email
934        dp['phone'] = brain.phone
935        dp['lastname'] = brain.lastname
936        dp['middlename'] = brain.middlenames   # different field names!
937        dp['firstname'] = brain.firstname
938        personal.getContent().edit(mapping=dp)
939        #
940        # clearance
941        #
942        student_object.invokeFactory('StudentClearance','clearance')
943        clearance = student_object.clearance
944        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
945        dc = {'Title': 'Clearance/Eligibility Record'}
946        dc['lga'] = brain.lga
947        dc['birthday'] = brain.date_of_birth
948        clearance.getContent().edit(mapping=dc)
949        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
950        #
951        # study Course
952        #
953        student_object.invokeFactory('StudentStudyCourse','study_course')
954        studycourse = student_object.study_course
955        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
956        dsc = {}
957        dsc['study_course'] = brain.course_admitted
958        dsc['current_verdict'] = ''
959        dsc['current_mode'] = da['entry_mode']
960        if da['entry_mode'].startswith('de'):
961            dsc['current_level'] = '200'
962        elif da['entry_mode'].startswith('pre'):
963            dsc['current_level'] = '000'
964        else:
965            dsc['current_level'] = '100'
966        dsc['current_session'] = entry_session
967        studycourse.getContent().edit(mapping=dsc)
968        #
969        # payments folder
970        student_object.invokeFactory('PaymentsFolder','payments')
971        payments = getattr(student_object,'payments')
972        dpay = {}
973        dpay['Title'] = 'Payments'
974        payments.getContent().edit(mapping=dpay)
975        self.portal_workflow.doActionFor(payments,'open')
976        #
977        # passport foto
978        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
979                                                              brain.screening_type,
980                                                              brain.reg_no)
981        images_dir = getImagesDir(student_id)
982        #images_dir = os.path.join("%s" % images_base,student_id)
983        letter_dir,student_dir = os.path.split(images_dir)
984        if not os.path.exists(letter_dir):
985            os.mkdir(letter_dir)
986        if not os.path.exists(images_dir):
987            os.mkdir(images_dir)
988        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
989        if os.path.exists(app_picture):
990            copy2(app_picture,image_name)
991        else:
992            logger.info('passport of %s/%s not found: %s' % (student_id,
993                                                             brain.reg_no,
994                                                             app_picture))
995
996        return student_id
997    ###)
998
999    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1000    def makeStudentLevel(self,student_id):
1001        "create the StudyLevel for a returning Student"
1002        #import pdb;pdb.set_trace()
1003        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1004        students_folder = self.portal_url.getPortalObject().campus.students
1005        res = self.students_catalog(id=student_id)
1006        if res:
1007            st = res[0]
1008        course = st.course
1009        matric_no = st.matric_no
1010        level = st.level
1011        res = self.results_import(matric_no = matric_no)
1012        if res:
1013            results = res
1014        logger.info('%s creating Level %s' % (student_id,level))
1015        #
1016        # Level
1017        #
1018        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1019        studycourse = getattr(student_obj,"study_course",None)
1020        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1021        l = getattr(studycourse,level,None)
1022        if l is None:
1023            studycourse.invokeFactory('StudentStudyLevel', level)
1024            l = getattr(studycourse, level)
1025            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1026            l.getContent().edit(mapping={'Title': "Level %s" % level})
1027        ###)
1028
1029    security.declarePublic('getHallInfo') ###(
1030    def getHallInfo(self,bed):
1031        """return Hall Info"""
1032        info = {}
1033        hall,block,room,letter = bed.split('_')
1034        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1035        if res and len(res) == 1:
1036            hall_brain = res[0]
1037            hall_doc = hall_brain.getObject().getContent()
1038        else:
1039            return info
1040        info['hall_title'] = hall_brain.Title
1041        info['maintenance_code'] = hall_doc.maintenance_code
1042        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1043        batch_doc = None
1044        for brain in res:
1045            if brain.id.startswith(info['maintenance_code']):
1046                batch_doc = brain.getObject().getContent()
1047                break
1048        if batch_doc is None:
1049            info['maintenance_fee'] = ''
1050        else:
1051            info['maintenance_fee'] = batch_doc.cost
1052        return info
1053    ###)
1054
1055    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1056    def removePictureFolder(self,student_id):
1057        """remove picture_folder by renaming it"""
1058        path = 'images'
1059        picture_path = os.path.join(i_home,path,student_id)
1060        if not os.path.exists(picture_path):
1061            return False
1062        os.rename(picture_path,picture_path + "_removed")
1063        return True
1064    ###)
1065
1066    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1067    def restorePictureFolder(self,student_id):
1068        """restore picture_folder by renaming it"""
1069        path = 'images'
1070        picture_path = os.path.join(i_home,path,student_id)
1071        if not os.path.exists(picture_path + "_removed"):
1072            return False
1073        os.rename(picture_path + "_removed",picture_path)
1074        return True
1075    ###)
1076
1077    security.declarePublic('picturesExist') ###(
1078    def picturesExist(self, ids,student_id=None):
1079        """check if pictures exist in the filesystem"""
1080        if student_id is None:
1081            student_id = self.getStudentId()
1082        if student_id is None:
1083            return False
1084        picture_path = getImagesDir(student_id)
1085        #picture_path = os.path.join(images_base,student_id)
1086        if not os.path.exists(picture_path):
1087            return False
1088        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1089        return set(ids).issubset(set(pictures))
1090    ###)
1091
1092    security.declarePublic('picturesList') ###(
1093    def picturesList(self):
1094        """check if pictures exist in the filesystem"""
1095        path = 'images'
1096        student_id = self.getStudentId()
1097        #picture_path = os.path.join(i_home,path,student_id)
1098        picture_path = getImagesDir(student_id)
1099        if not os.path.exists(picture_path):
1100            return []
1101        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1102    ###)
1103
1104    security.declarePublic('showFsPicture') ###(
1105    def showFsPicture(self,path):
1106        """return a picture from the filesystem"""
1107        #picture_path = os.path.join(i_home,path)
1108        picture_path = os.path.join(images_base,path)
1109        response = self.REQUEST.RESPONSE
1110        #import pdb;pdb.set_trace()
1111        registry = getToolByName(self, 'mimetypes_registry')
1112        mimetype = str(registry.lookupExtension(path.lower()) or
1113                    registry.lookupExtension('file.bin'))
1114        if os.path.exists(picture_path):
1115            response.setHeader('Content-type',mimetype)
1116            return open(picture_path).read()
1117        picture_path = os.path.join(i_home,'import',path)
1118        if os.path.exists(picture_path):
1119            return open(picture_path).read()
1120    ###)
1121
1122    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1123    def deleteAllCourses(self,department="All"):
1124        ''' delete the courses'''
1125        pm = self.portal_membership
1126        member = pm.getAuthenticatedMember()
1127
1128        if str(member) not in ("henrik","joachim"):
1129            return "not possible"
1130        if department == "All":
1131            res = self.portal_catalog({'meta_type': 'Department'})
1132        if len(res) < 1:
1133            return "No Departments found"
1134
1135        deleted = []
1136        for dep in res:
1137            cf = dep.getObject().courses
1138            if cf:
1139                cf.manage_delObjects(ids=cf.objectIds())
1140                deleted.append("deleted Courses in %s" % dep.getId)
1141        return "\r".join(deleted)
1142    ###)
1143
1144    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1145    def getLogfileLines(self,filename="event.log",numlines=20):
1146        """Get last NUMLINES lines of logfile FILENAME.
1147
1148        Return last lines' of a file in the instances logfile directory as
1149        a list. The number of returned lines equals `numlines' or less. If
1150        less than `numlines' lines are available, the whole file ist
1151        returned. If the file can not be opened or some other error
1152        occurs, empty list is returend.
1153        """
1154        result = []
1155        lines_hit = 0
1156
1157        # We only handle files in instances' log directory...
1158        logpath = os.path.join(i_home, "log")
1159        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1160        if not filename.startswith( logpath ):
1161            # Attempt to access file outside log-dir...
1162            return []
1163
1164        try:
1165            fd = file( filename, "rb" )
1166        except IOError:
1167            return []
1168        if not fd:
1169            return []
1170
1171        if os.linesep == None:
1172            linesep = '\n'
1173        else:
1174            linesep = os.linesep
1175
1176        # Try to find 'numlines' times a lineseparator, searching from end
1177        # and moving to the beginning of file...
1178        fd.seek( 0, 2) # Move to end of file...
1179        while lines_hit < numlines:
1180            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1181                                          # one step forward...
1182                lines_hit += 1
1183            try:
1184                fd.seek( -2, 1) # Go two bytes back from current pos...
1185            except IOError:
1186                # We cannot go back two bytes. Maybe the file is too small...
1187                break
1188        fd.seek(2,1)
1189
1190        # Read all lines from current position...
1191        result = fd.readlines()
1192        # Remove line endings...
1193        result = [x.strip() for x in result]
1194        fd.close()
1195        return result
1196    ###)
1197
1198    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1199    def getCallbacksFromLog(self,filename):
1200        """fix Online Payment Transactions from Z2.log entries"""
1201        import transaction
1202        import random
1203        from cgi import parse_qs
1204        from urlparse import urlparse
1205        #from pdb import set_trace
1206        wftool = self.portal_workflow
1207        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1208        students_folder = self.portal_url.getPortalObject().campus.students
1209        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1210        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1211        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1212        data = re.compile(s)
1213        start = True
1214        tr_count = 1
1215        total = 0
1216        #name = 'pume_results'
1217        #name = 'epaymentsuccessful_z2log2'
1218        name = filename
1219        no_import = []
1220        imported = []
1221        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1222        try:
1223            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1224        except:
1225            logger.error('Error reading %s' % name)
1226            return
1227        tas = []
1228        for line in transactions:
1229            dict = {}
1230            items = data.search(line)
1231            dict['idict'] = idict = items.groupdict()
1232            #print idict
1233            #from pdb import set_trace;set_trace()
1234            urlparsed = urlparse(idict['get'][4:])
1235            #print urlparsed
1236            path = urlparsed[2].split('/')
1237            dict['student_id'] = student_id = path[8]
1238            dict['payment_id'] = payment_id = path[10]
1239            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1240            tas.append(dict)
1241            tr_count += 1
1242        return tas
1243    ###)
1244
1245    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1246    def importOnlinePaymentTransactions(self):
1247        """load Online Payment Transactions from CSV values"""
1248        import transaction
1249        import random
1250        #from pdb import set_trace
1251        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1252        opt = self.online_payments_import
1253        students_folder = self.portal_url.getPortalObject().campus.students
1254        start = True
1255        tr_count = 1
1256        total = 0
1257        #name = 'pume_results'
1258        name = 'OnlineTransactions'
1259        no_import = []
1260        imported = []
1261        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1262        try:
1263            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1264        except:
1265            logger.error('Error reading %s.csv' % name)
1266            return
1267        for pay_transaction in transactions:
1268            if start:
1269                start = False
1270                logger.info('Start loading from %s.csv' % name)
1271                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1272                no_import.append('%s,"Error"' % s)
1273                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1274                format_error = format + ',"%(Error)s"'
1275            data = {}
1276
1277            # format of the first file sent by Tayo
1278            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1279            #data['student_id'] = student_id = pay_transaction['Payer ID']
1280            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1281            #data['response_code'] = response_code = pay_transaction['Resp Code']
1282            #data['amount'] = amount = pay_transaction['Amount']
1283
1284            # format of the second file sent by Tayo
1285            #data['datetime'] = date = 0
1286            #data['student_id'] = student_id = pay_transaction['Payer ID']
1287            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1288            #data['response_code'] = response_code = '00'
1289            #data['amount'] = amount = pay_transaction['Amount']
1290
1291            # format of the third file sent by Kehinde
1292            data['datetime'] = date = 0
1293            data['student_id'] = student_id = pay_transaction['customer_id']
1294            data['order_id'] = order_id = pay_transaction['merchant_reference']
1295            data['response_code'] = response_code = '00'
1296            data['amount'] = amount = pay_transaction['Amount']
1297
1298            dup = False
1299            if response_code == "12":
1300                continue
1301            try:
1302                opt.addRecord(**data)
1303            except ValueError:
1304                dup = True
1305            #from pdb import set_trace;set_trace()
1306            if dup:
1307                if response_code == "00":
1308                    try:
1309                        opt.modifyRecord(**data)
1310                    except:
1311                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1312                        continue
1313                else:
1314                    pay_transaction['Error'] = "Duplicate order_id"
1315                    no_import.append( format_error % pay_transaction)
1316                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1317                    continue
1318            tr_count += 1
1319            if tr_count > 1000:
1320                if len(no_import) > 0:
1321                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1322                             '\n'.join(no_import) + '\n')
1323                    no_import = []
1324                em = '%d transactions committed\n' % (tr_count)
1325                transaction.commit()
1326                regs = []
1327                logger.info(em)
1328                total += tr_count
1329                tr_count = 0
1330        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1331                                                '\n'.join(no_import))
1332        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1333    ###)
1334
1335    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1336    def mass_create_faculty(self,mapping):
1337        "create a faculty"
1338        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1339        academics_folder = self.portal_url.getPortalObject().campus.academics
1340        fid = mapping['code']
1341        if getattr(academics_folder,fid,None) is not None:
1342            return '', "Faculty with ID: %s exists" % fid
1343        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1344        try:
1345            academics_folder.invokeFactory('Faculty', fid)
1346        except BadRequest,E:
1347            return '', "%s" % E
1348        f = getattr(academics_folder,fid,None)
1349        f.getContent().edit(mapping=mapping)
1350        return fid,''
1351    ###)
1352
1353    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1354    def mass_edit_faculty(self,mapping):
1355        "edit a faculty"
1356        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1357        academics_folder = self.portal_url.getPortalObject().campus.academics
1358        fid = mapping['code']
1359        f = getattr(academics_folder,fid,None)
1360        if f is None:
1361            return '', "Faculty with ID: %s does not exist" % fid
1362        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1363        f.getContent().edit(mapping=mapping)
1364        return fid,''
1365    ###)
1366
1367    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1368    def mass_create_department(self,mapping):
1369        "create a department in the correct faculty"
1370        logger = logging.getLogger('WAeUPTool.mass_create_department')
1371        fid = mapping['faculty_code']
1372        if getattr(self,'_v_faculties',None) is None:
1373            res = self.portal_catalog(portal_type = "Faculty")
1374            self._v_faculties = {}
1375            for f in res:
1376                self._v_faculties[f.getId] = f.getObject()
1377        f = self._v_faculties.get(fid,None)
1378        if f is None:
1379            return '', "No Faculty with ID: %s" % fid
1380        else:
1381            did = mapping.get('code')
1382            d = getattr(f,did,None)
1383            if d is None or d.portal_type == "Faculty":
1384                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1385                try:
1386                    f.invokeFactory('Department', did)
1387                except BadRequest,E:
1388                    return '', "%s" % E
1389                d = getattr(f,did)
1390                d.invokeFactory('CoursesFolder','courses')
1391                courses = getattr(d,'courses')
1392                dict = {'Title': 'Courses'}
1393                courses.getContent().edit(mapping=dict)
1394                d.invokeFactory('CertificatesFolder','certificates')
1395                certificates = getattr(d,'certificates')
1396                dict = {'Title': 'Certificates'}
1397                certificates.getContent().edit(mapping=dict)
1398            d.getContent().edit(mapping=mapping)
1399        return did,''
1400    ###)
1401
1402    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1403    def mass_edit_department(self,mapping):
1404        "create a department in the correct faculty"
1405        logger = logging.getLogger('WAeUPTool.mass_create_department')
1406        academics_folder = self.portal_url.getPortalObject().campus.academics
1407        fid = mapping['faculty_code']
1408        did = mapping.get('code')
1409        try:
1410            d = getattr(getattr(academics_folder,fid),did,None)
1411        except KeyError:
1412            return '', "Department %s or Faculty %s wrong" % (did,fid)
1413        else:
1414            if d is None or d.portal_type == "Faculty":
1415                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1416            d.getContent().edit(mapping=mapping)
1417        return did,''
1418    ###)
1419
1420    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1421    def mass_create_course(self,mapping):
1422        #import pdb;pdb.set_trace()
1423        if getattr(self,'_v_course_list',None) is None:
1424            self._v_course_list = []
1425        if getattr(self,'_v_departments',None) is None:
1426            res = self.portal_catalog(portal_type = "Department")
1427            self._v_department_courses = {}
1428            for d in res:
1429                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1430        did = mapping['department_code']
1431        d = self._v_department_courses.get(did,None)
1432        if d is None:
1433            return '', "No Department with ID: %s" % did
1434        course_id = mapping.get('code')
1435        if course_id in self._v_course_list:
1436            return '', "Duplicate Course ID: %s" % did
1437        c = getattr(d,course_id,None)
1438        if c is not None:
1439            return '', "Duplicate Course ID: %s" % did
1440        try:
1441            d.invokeFactory('Course', course_id)
1442        except BadRequest,E:
1443            return '', "%s" % E
1444        self._v_course_list.append(course_id)
1445        c = getattr(d,course_id)
1446        c.getContent().edit(mapping=mapping)
1447        return course_id,''
1448    ###)
1449
1450    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1451    def mass_edit_course(self,mapping):
1452        #import pdb;pdb.set_trace()
1453        course_id = mapping.get('code')
1454        res = self.portal_catalog(id=course_id)
1455        if not res:
1456            return '', "No Course with ID: %s" % course_id
1457        c = res[0].getObject()
1458        c.getContent().edit(mapping=mapping)
1459        return course_id,''
1460    ###)
1461
1462    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1463    def mass_create_certificate(self,mapping):
1464        if getattr(self,'_v_certificate_list',None) is None:
1465            self._v_certificate_list = []
1466        if getattr(self,'_v_department_certificates',None) is None:
1467            res = self.portal_catalog(portal_type = "Department")
1468            self._v_department_certificates = {}
1469            for d in res:
1470                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1471        did = mapping['department_code']
1472        d = self._v_department_certificates.get(did,None)
1473        if d is None:
1474            return '', "No Department with ID: %s" % did
1475        certificate_id = mapping.get('code')
1476        if certificate_id in self._v_certificate_list:
1477            return '', "Duplicate Certificate ID: %s" % did
1478        c = getattr(d,certificate_id,None)
1479        if c is not None:
1480            return '', "Duplicate Certificate ID: %s" % did
1481        try:
1482            d.invokeFactory('Certificate', certificate_id)
1483        except BadRequest,E:
1484            return '', "%s" % E
1485        self._v_certificate_list.append(certificate_id)
1486        c = getattr(d,certificate_id)
1487        c.getContent().edit(mapping=mapping)
1488        return certificate_id,''
1489    ###)
1490
1491    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1492    def mass_edit_certificate(self,mapping):
1493        #import pdb;pdb.set_trace()
1494        certificate_id = mapping.get('code')
1495        res = self.portal_catalog(id=certificate_id)
1496        if not res:
1497            return '', "No Certificate with ID: %s" % certificate_id
1498        c = res[0].getObject()
1499        c.getContent().edit(mapping=mapping)
1500        return certificate_id,''
1501    ###)
1502
1503    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1504    def mass_create_application(self,mapping):
1505        #import pdb;pdb.set_trace()
1506        reg_no = mapping.get('reg_no')
1507        try:
1508            self.applicants_catalog.addRecord(**mapping)
1509        except ValueError:
1510            return '', "applicant record with reg_no %s already exists" % reg_no
1511        return reg_no,''
1512    ###)
1513
1514    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1515    def mass_edit_application(self,mapping):
1516        #import pdb;pdb.set_trace()
1517        reg_no = mapping.get('reg_no')
1518        status = mapping.get('status')
1519        res = self.applicants_catalog(reg_no = reg_no)
1520        if len(res):
1521            if res[0].status == 'created' and status != 'created':
1522                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1523            elif status == 'created' and res[0].status != 'created':
1524                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1525            else:
1526                self.applicants_catalog.modifyRecord(**mapping)
1527                return reg_no,''
1528        else:
1529            return '', "applicant record with reg_no %s does not exist" % reg_no
1530
1531    ###)
1532
1533    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1534    def mass_create_course_result(self,mapping):
1535        #import pdb;pdb.set_trace()
1536        students_folder = self.portal_url.getPortalObject().campus.students
1537        if getattr(self,'_v_courses',None) is None:
1538            res = self.courses_catalog()
1539            self._v_courses = {}
1540            for brain in res:
1541                self._v_courses[brain.code] = brain
1542        if getattr(self,'_v_level_created',None) is None:
1543            self._v_level_created = []
1544        course_id = mapping.get('code')
1545        if course_id not in self._v_courses.keys():
1546            return '', "No course with ID: %s" % course_id
1547        id_key = ''
1548        for id_key in ('student_id','matric_no'):
1549            id_field = mapping.get(id_key,None)
1550            if id_field is not None:
1551                student_id = id_field
1552                break
1553        query = Eq(id_key,id_field)
1554        res = self.students_catalog.evalAdvancedQuery(query)
1555        if not res:
1556            return '', "no student with %(id_key)s %(id_field)s" % vars()
1557        student_rec = res[0]
1558        if id_field != "student_id":
1559            mapping['student_id'] = student_rec.id
1560        student_id = student_rec.id
1561        level_id = mapping['level_id']
1562        code = mapping['code']
1563        if student_id not in self._v_level_created:
1564            try:
1565                context = getattr(getattr(students_folder,
1566                                          "%(student_id)s" % vars()),
1567                                  'study_course')
1568            except:
1569                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1570            if level_id not in context.objectIds():
1571                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1572                level = getattr(context,"%(level_id)s" % vars())
1573                self.portal_workflow.doActionFor(level,'open')
1574                # the session string must not be copied into the level object
1575                current_verdict = getattr(student_rec,'current_verdict','')
1576                current_session = getattr(student_rec,'current_session','')
1577                if current_verdict and student_rec.current_level == level_id:
1578                    level.getContent().edit(mapping={'verdict': "%s" %
1579                                                     current_verdict,
1580                                                     'session': "%s" %
1581                                                     current_session,
1582                                                    })
1583                    self.portal_workflow.doActionFor(level,'close')
1584            self._v_level_created += student_id,
1585        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1586        for k in ('semester','credits',):
1587            mapping[k] = getattr(self._v_courses[course_id],k)
1588        try:
1589            self.course_results.addRecord(**mapping)
1590        except ValueError:
1591            return '', "course result already exists: %s" % key
1592        return key,''
1593    ###)
1594
1595    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1596    def mass_edit_course_result(self,mapping):
1597        id_key = ''
1598        for id_key in ('student_id','matric_no'):
1599            id_field = mapping.get(id_key,None)
1600            if id_field is not None:
1601                student_id = id_field
1602                break
1603        query = Eq(id_key,id_field)
1604        res = self.students_catalog.evalAdvancedQuery(query)
1605        if not res:
1606            return '', "No student with %(id_key)s %(id_field)s" % vars()
1607        if id_field != "student_id":
1608            mapping['student_id'] = res[0].id
1609        if not res:
1610            return '', "No student with %(id_key)s %(id_field)s" % vars()
1611        student_rec = res[0]
1612        student_id = student_rec.id
1613        level_id = mapping['level_id']
1614        code = mapping['code']
1615        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1616        try:
1617            self.course_results.modifyRecord(**mapping)
1618        except KeyError:
1619            return '', "No course result to edit: %s" % key
1620        return key,''
1621    ###)
1622
1623    security.declareProtected(ModifyPortalContent,'mass_delete_course_result') ###(
1624    def mass_delete_course_result(self,mapping):
1625        id_key = ''
1626        for id_key in ('student_id','matric_no'):
1627            id_field = mapping.get(id_key,None)
1628            if id_field is not None:
1629                student_id = id_field
1630                break
1631        query = Eq(id_key,id_field)
1632        res = self.students_catalog.evalAdvancedQuery(query)
1633        if not res:
1634            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1635        if id_field != "student_id":
1636            mapping['student_id'] = res[0].id
1637        if not res:
1638            return '', "no student with %(id_key)s %(id_field)s" % vars()
1639        student_rec = res[0]
1640        student_id = student_rec.id
1641        level_id = mapping['level_id']
1642        code = mapping['code']
1643        key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1644        # for k in ('semester',):
1645        #     mapping[k] = getattr(self._v_courses[course_id],k)
1646        if self.course_results.getRecordByKey(key) is None:
1647            return '', "no course-result with %(key)s" % vars()
1648        self.course_results.deleteRecord(key)
1649        return key,''
1650    ###)
1651
1652    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1653    def mass_create_certificate_course(self,mapping):
1654        if getattr(self,'_v_courses',None) is None:
1655            res = self.courses_catalog()
1656            self._v_courses= [course.code for course in res]
1657        if getattr(self,'_v_ceritficates',None) is None:
1658            res = self.portal_catalog(portal_type = "Certificate")
1659            self._v_certificates = {}
1660            for cert in res:
1661                self._v_certificates[cert.getId] = cert.getObject()
1662        certificate_course_id = mapping.get('code')
1663        if certificate_course_id not in self._v_courses:
1664            return '', "No Course with ID: %s" % certificate_course_id
1665        cert_id = mapping['certificate_code']
1666        cert = self._v_certificates.get(cert_id,None)
1667        if cert is None:
1668            return '', "No Certificate with ID: %s" % cert_id
1669        level_id = mapping.get('level')
1670        level = getattr(cert,level_id,None)
1671        if level is None:
1672            cert.invokeFactory('StudyLevel', level_id)
1673            level = getattr(cert,level_id,None)
1674        elif hasattr(level,certificate_course_id):
1675            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1676            (certificate_course_id,cert_id,level_id)
1677        level.invokeFactory('CertificateCourse', certificate_course_id)
1678        c = getattr(level,certificate_course_id)
1679        c.getContent().edit(mapping=mapping)
1680        return certificate_course_id,''
1681    ###)
1682
1683    field2types_student = {   ###(
1684                      'StudentApplication':
1685                          {'id': 'application',
1686                           'title': 'Application Data',
1687                           'wf_transition_return': 'close',
1688                           'wf_transition_admit': 'remain',
1689                           'fields':
1690                             ('jamb_reg_no',
1691                              'entry_mode',
1692                              'entry_session',
1693                              'jamb_score',
1694                              'app_email',
1695                              'jamb_age',
1696                              'jamb_state',
1697                              'jamb_lga',
1698                              'jamb_sex',
1699                              )
1700                              },
1701                      #'StudentPume':
1702                      #    {'id': 'pume',
1703                      #     'title': 'Pume Data',
1704                      #     'wf_transition_return': 'close',
1705                      #     'wf_transition_admit': 'close',
1706                      #     'fields':
1707                      #       ('pume_score',
1708                      #        )
1709                      #        },
1710                      'StudentClearance':
1711                          {'id': 'clearance',
1712                           'title': 'Clearance/Eligibility Record',
1713                           'wf_transition_return': 'close',
1714                           'wf_transition_admit': 'remain',
1715                           'fields':
1716                             ('matric_no',
1717                              'nationality',
1718                              'lga',
1719                              'birthday',
1720                              )
1721                              },
1722                         'StudentPersonal':
1723                          {'id': 'personal',
1724                           'title': 'Personal Data',
1725                           'wf_transition_return': 'open',
1726                           'wf_transition_admit': 'remain',
1727                           'fields':
1728                             ('firstname',
1729                              'middlename',
1730                              'lastname',
1731                              'sex',
1732                              'email',
1733                              'phone',
1734                              'perm_address',
1735                              )
1736                              },
1737                         'StudentStudyCourse':
1738                          {'id': 'study_course',
1739                           'title': 'Study Course',
1740                           'wf_transition_return': 'open',
1741                           'wf_transition_admit': 'remain',
1742                           'fields':
1743                             ('study_course',
1744                              'current_level',
1745                              'current_session',
1746                              'current_mode',
1747                              'current_verdict',
1748                              'previous_verdict',
1749                              )
1750                              },
1751                         # 'StudentStudyLevel':
1752                         #  {'id': 'current_level',
1753                         #   'title': '',
1754                         #   'wf_transition_return': 'open',
1755                         #   'wf_transition_admit': 'remain',
1756                         #   'fields':
1757                         #     ('verdict',
1758                         #      'session',
1759                         #      )
1760                         #      },
1761                         'PaymentsFolder':
1762                          {'id': 'payments',
1763                           'title': 'Payments',
1764                           'wf_transition_return': 'open',
1765                           'wf_transition_admit': 'open',
1766                           'fields':
1767                             ()
1768                              },
1769                         }
1770    ###)
1771
1772    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1773    def mass_create_student(self,mapping):
1774        "create student records due import"
1775        logger = logging.getLogger('WAeUPTool.mass_create_student')
1776        students_folder = self.portal_url.getPortalObject().campus.students
1777        jamb_reg_no = mapping.get('jamb_reg_no',None)
1778        if jamb_reg_no:
1779            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1780            if res:
1781                return '',"jamb_reg_no exists"
1782        matric_no = mapping.get('matric_no',None)
1783        if matric_no:
1784            res = self.students_catalog(matric_no = matric_no)
1785            if res:
1786                return '',"matric_no exists"
1787        sid = self.waeup_tool.generateStudentId('?')
1788        students_folder.invokeFactory('Student', sid)
1789        student_obj = getattr(students_folder,sid)
1790        f2t = self.field2types_student
1791        d = {}
1792        transition = mapping.get('reg_transition','admit')
1793        if transition not in ('admit','return'):
1794            return '',"no valid transition provided"
1795        for pt in f2t.keys():
1796            student_obj.invokeFactory(pt,f2t[pt]['id'])
1797            sub_obj = getattr(student_obj,f2t[pt]['id'])
1798            sub_doc = sub_obj.getContent()
1799            d['Title'] = f2t[pt]['title']
1800            for field in f2t[pt]['fields']:
1801                d[field] = mapping.get(field,'')
1802
1803            if pt == "StudentApplication":
1804                #d['jamb_sex']  = 'M'
1805                #if mapping.get('sex'):
1806                #    d['jamb_sex']  = 'F'
1807                d['jamb_firstname'] = mapping.get('firstname',None)
1808                d['jamb_middlename'] = mapping.get('middlename',None)
1809                d['jamb_lastname'] = mapping.get('lastname',None)
1810
1811            # if pt == "StudyCourse":
1812            #     for von,zu in (('entry_mode','current_mode'),
1813            #                    ('entry_session','current_session')):
1814            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1815            #             d[zu] = mapping[von]
1816            sub_doc.edit(mapping = d)
1817
1818            #import pdb;pdb.set_trace()
1819            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1820            if new_state != "remain":
1821                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1822        self.portal_workflow.doActionFor(student_obj,transition)
1823        student_obj.manage_setLocalRoles(sid, ['Owner',])
1824        return sid,''
1825    ###)
1826
1827    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1828    def mass_edit_student(self,mapping):
1829        wftool = self.portal_workflow
1830        "edit student records due import"
1831        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1832        students_folder = self.portal_url.getPortalObject().campus.students
1833        sid = mapping.get('id',None)
1834        jamb_reg_no = mapping.get('jamb_reg_no',None)
1835        matric_no = mapping.get('matric_no',None)
1836        editable_keys = mapping.keys()
1837        if sid:
1838            res = self.students_catalog(id = sid)
1839            if not res:
1840                return '',"no student with id %s" % sid
1841            if matric_no and res[0].matric_no and\
1842              matric_no != res[0].matric_no:
1843                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1844            if jamb_reg_no and res[0].jamb_reg_no and\
1845              jamb_reg_no != res[0].jamb_reg_no:
1846                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1847        elif jamb_reg_no:
1848            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1849            if not res:
1850                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1851            editable_keys.remove('jamb_reg_no')
1852        elif matric_no:
1853            res = self.students_catalog(matric_no = matric_no)
1854            if not res:
1855                return '',"no student with matric_no %s" % matric_no
1856            editable_keys.remove('matric_no')
1857
1858        ## included only to change wf state from admitted to returning
1859        #if res[0].review_state not in ('admitted','objection_raised'):
1860        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1861        ## end inclusion
1862
1863        sid = res[0].id
1864        student_obj = getattr(students_folder,sid)
1865        f2t = self.field2types_student
1866        d = {}
1867        #import pdb;pdb.set_trace()
1868        any_change = False
1869        #special treatment for StudentStudyLevel
1870        d['verdict']  = mapping.get('current_verdict','')
1871        d['session']  = mapping.get('current_session','')
1872        current_level = mapping.get('current_level','')
1873        while d['session'] and d['verdict'] and current_level:
1874            sub_obj = getattr(student_obj,'study_course',None)
1875            if sub_obj is None:
1876                break
1877            level_obj = getattr(sub_obj,current_level,None)
1878            if  level_obj is None:
1879                break
1880            any_change = True
1881            level_obj.getContent().edit(mapping = d)
1882            try:
1883                wftool.doActionFor(level_obj,'close')
1884            except:
1885                pass
1886            break
1887        for pt in f2t.keys():
1888            #if pt == "StudentApplication":
1889            #    d['jamb_sex']  = 'M'
1890            #    if mapping.get('sex'):
1891            #        d['jamb_sex']  = 'F'
1892            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1893            if intersect and pt not in ('StudentStudyLevel',):
1894                object_id = f2t[pt]['id']
1895                sub_obj = getattr(student_obj,object_id,None)
1896                if sub_obj is None:
1897                    try:
1898                        student_obj.invokeFactory(pt,object_id)
1899                    except:
1900                        continue
1901                    sub_obj = getattr(student_obj,object_id)
1902                    if f2t[pt]['title'] != '':
1903                        d['Title'] = f2t[pt]['title']
1904                sub_doc = sub_obj.getContent()
1905                for field in intersect:
1906                    changed = False
1907                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1908                        any_change = True
1909                        changed = True
1910                        d[field] = mapping.get(field,'')
1911                    if changed:
1912                        sub_doc.edit(mapping = d)
1913
1914
1915        ## included only to change wf state from admitted to returning
1916        #    if res[0].review_state in ('admitted','objection_raised'):
1917        #        new_state = f2t[pt]['wf_transition_return']
1918        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1919        #        if sub_obj and new_state != "remain":
1920        #            try:
1921        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1922        #            except:
1923        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1924        #                pass
1925        #if res[0].review_state in ('admitted','objection_raised'):
1926        #    wfaction = 'return'
1927        #    try:
1928        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1929        #        logger.info('%s, wf state changed' % sid)
1930        #        any_change = True
1931        #    except:
1932        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1933        #        pass
1934        ## end inclusion
1935
1936
1937        if any_change:
1938            return sid,''
1939        else:
1940            return sid,'not modified'
1941    ###)
1942
1943    security.declareProtected(ModifyPortalContent,'mass_edit_verdict') ###(
1944    def mass_edit_verdict(self,mapping):
1945        wftool = self.portal_workflow
1946        "edit student verdicts"
1947        logger = logging.getLogger('WAeUPTool.mass_edit_verdict')
1948        students_folder = self.portal_url.getPortalObject().campus.students
1949        student_id = mapping.get('id',None)
1950        matric_no = mapping.get('matric_no',None)
1951        editable_keys = mapping.keys()
1952        if student_id:
1953            student_record = self.students_catalog.getRecordById(student_id)
1954            if student_record is None:
1955                return '',"no student with id %s" % student_id
1956            if matric_no and student_record.matric_no and\
1957              matric_no != student_record.matric_no:
1958                return '', 'student %s: matric_no %s does not match %s' % (student_record.id,
1959                                                                          student_record.matric_no,
1960                                                                          matric_no)
1961        elif matric_no:
1962            res = self.students_catalog(matric_no = matric_no)
1963            if not res:
1964                return '',"no student with matric_no %s" % matric_no
1965            student_record = res[0]
1966            editable_keys.remove('matric_no')
1967
1968        student_id = student_record.id
1969        student_review_state =  student_record.review_state
1970        if student_review_state not in ('courses_validated','returning'):
1971            return '',"student %s in wrong review_state %s" % (student_id, student_review_state)
1972        student_obj = getattr(students_folder,student_id)
1973        f2t = self.field2types_student
1974        d = {}
1975        #import pdb;pdb.set_trace()
1976        any_change = False
1977        #special treatment for StudentStudyLevel
1978        current_session = d['session'] = mapping.get('current_session','')
1979        if current_session and student_record.session != current_session:
1980            return '','student %s: imported session %s does not match current_session %s' % (student_id,
1981                                                                                        current_session,
1982                                                                                        student_record.session)
1983
1984        current_level = mapping.get('current_level','')
1985        if current_level and student_record.level != current_level:
1986            return '','student %s: imported level %s does not match current_level %s' % (student_id,
1987                                                                                    current_level,
1988                                                                                    student_record.level)
1989        study_course_obj = getattr(student_obj,'study_course',None)
1990        if study_course_obj is None:
1991            return '','student %s: no study_course object' % student_id
1992        level_obj = getattr(study_course_obj,current_level,None)
1993        if  level_obj is None:
1994            return '','student %s: no study_level object for level %s' % (student_id,
1995                                                                             current_level)
1996        verdict = d['verdict'] = d['current_verdict']  = mapping.get('current_verdict','')
1997
1998        #if verdict == student_record.verdict:
1999        #    return '','student %s: verdict already set to %s' % (student_id,
2000        #                                                            verdict)
2001
2002        level_review_state = wftool.getInfoFor(level_obj,'review_state',None)
2003        if level_review_state != "closed":
2004            wftool.doActionFor(level_obj,'close')
2005            # return '','student %s: level %s is not closed' % (student_id,
2006            #                                                      current_level)
2007
2008        study_course_obj.getContent().edit(mapping = d)
2009        level_obj.getContent().edit(mapping = d)
2010        if student_review_state != "returning":
2011            wftool.doActionFor(student_obj,'return')
2012        # try:
2013        #     wftool.doActionFor(level_obj,'close')
2014        # except:
2015        #     pass
2016        return student_id,''
2017    ###)
2018
2019    security.declareProtected(ModifyPortalContent,"importData")###(
2020    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
2021        """load data from CSV values"""
2022        import transaction
2023        import random
2024
2025        pm = self.portal_membership
2026        member = pm.getAuthenticatedMember()
2027
2028        logger = logging.getLogger('WAeUPTool.importData')
2029        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
2030        students_folder = self.portal_url.getPortalObject().campus.students
2031        start = True
2032        tr_count = 0
2033        total_imported = 0
2034        total_not_imported = 0
2035        total = 0
2036        iname = "import_%s" % name
2037        if iname in ('import_application','import_course_result',):
2038            commit_after = 2000
2039        else:
2040            commit_after = 100
2041        stool = getToolByName(self, 'portal_schemas')
2042        ltool = getToolByName(self, 'portal_layouts')
2043        schema = stool._getOb(iname)
2044        if schema is None:
2045            em = 'No such schema %s' % iname
2046            logger.error('No such schema %s' % iname)
2047            return em
2048        layout = ltool._getOb(iname)
2049        if layout is None:
2050            em = 'No such layout %s' % iname
2051            logger.error(em)
2052            return em
2053        validators = {}
2054        for widget in layout.keys():
2055            validators[widget] = layout[widget].validate
2056        mode = "create"
2057        if edit:
2058            if filename.endswith('_toDelete'):
2059                mode = "delete"
2060            else:
2061                mode = "edit"
2062        importer_name = "mass_%(mode)s_%(name)s" % vars()
2063        importer = getattr(self, '%s' % importer_name,None)
2064        if importer is None:
2065            em = 'No importer function %s' % importer_name
2066            logger.error(em)
2067            return em
2068        not_imported = []
2069        imported = []
2070        try:
2071            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
2072        except:
2073            em = 'Error reading %s.csv' % filename
2074            logger.error(em)
2075            return em
2076        for item in items:
2077            #import pdb;pdb.set_trace()
2078            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
2079            if start:
2080                start = False
2081                adapters = [MappingStorageAdapter(schema, item)]
2082                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
2083                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
2084                import_keys = [k.strip() for k in attrs if not k.strip().startswith('ignore')]
2085                diff2schema = set(import_keys).difference(set(schema.keys()))
2086                diff2layout = set(import_keys).difference(set(layout.keys()))
2087                if diff2schema:
2088                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
2089                    return em
2090                if mode == "create":
2091                    required_keys = [layout.getIdUnprefixed(id)
2092                                     for id,widget in layout.objectItems()
2093                                     if widget.is_required]
2094                    if not set(required_keys).issubset(set(import_keys)):
2095                        diff2import = set(required_keys).difference(set(import_keys))
2096                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
2097                        return em
2098                s = ','.join(['"%s"' % fn for fn in import_keys])
2099                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
2100                s = '"id",' + s
2101                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
2102                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2103                format_error = format + ',"%(Error)s"'
2104                format = '"%(id)s",'+ format
2105
2106            dm = DataModel(item, adapters,context=self)
2107            ds = DataStructure(data=item,datamodel=dm)
2108            error_string = ""
2109            total += 1
2110            error_count = 0
2111            for k in import_keys:
2112                if not validators[k](ds,mode=mode):
2113                    if error_count:
2114                        error_string += ' ++ '
2115                    error_string += "%s: %s" % (k,
2116                                                  self.translation_service(ds.getError(k),
2117                                                                           ds.getErrorMapping(k)))
2118                    error_count += 1
2119            if not error_string:
2120                temp_item = item.copy()
2121                temp_item.update(dm)
2122                temp_item['id'],error = importer(temp_item)
2123                if error:
2124                    error_string += error
2125                else:
2126                    item = temp_item
2127            if error_string:
2128                item['Error'] = error_string
2129                not_imported.append(format_error % item)
2130                total_not_imported += 1
2131            else:
2132                em = format % item
2133                imported.append(em)
2134                tr_count += 1
2135                total_imported += 1
2136                logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(em)s" % vars())
2137
2138            if total and not total % commit_after:
2139                transaction.commit()
2140                if len(not_imported) > 0:
2141                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2142                             '\n'.join(not_imported) + '\n')
2143                    not_imported = []
2144                if len(imported) > 0:
2145                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2146                             '\n'.join(imported) + '\n')
2147                    imported = []
2148                em = '%d transactions committed\n' % (tr_count)
2149                regs = []
2150                logger.info(em)
2151                tr_count = 0
2152                #if total > 100:
2153                #    return
2154        if len(imported) > 0:
2155            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2156                                                '\n'.join(imported))
2157        if len(not_imported) > 0:
2158            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2159                                                '\n'.join(not_imported))
2160        #if mode.endswith('e'):
2161        #    mode = mode[:-1]
2162        em = "Finished importing in %(mode)s mode from %(filename)s.csv: %(total_imported)d imported, %(total_not_imported)d not imported (of total %(total)d)" % vars()
2163        logger.info(em)
2164        return em
2165    ###)
2166
2167    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2168    def moveImagesToFS(self,student_id="O738726"):
2169        "move the images to the filesystem"
2170        images_dir = getImagesDir(student_id)
2171        #images_dir = os.path.join("%s" % images_base,student_id)
2172        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2173        stool = getToolByName(self, 'portal_schemas')
2174        schemas = ['student_application',
2175                   'student_clearance',
2176                   ]
2177        created = False
2178        for schema_id in schemas:
2179            schema = stool._getOb(schema_id)
2180            object = getattr(student_folder,schema_id[len('student_'):],None)
2181            if object is None:
2182                continue
2183            doc = object.getContent()
2184            for key in schema.keys():
2185                if schema[key].meta_type != "CPS Image Field":
2186                    continue
2187                #import pdb;pdb.set_trace()
2188                image = getattr(doc,key,None)
2189                if not image or not hasattr(image,"data"):
2190                    continue
2191                if not created:
2192                    if not os.path.exists(images_dir):
2193                        os.mkdir(images_dir)
2194                    created = True
2195                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2196                open(filename,"wb").write(str(image.data))
2197    ###)
2198
2199    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2200    def movePassportToFS(self,student_id="O738726"):
2201        "move the passports to the filesystem"
2202        images_dir = os.path.join("%s" % i_home,'passports')
2203        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2204        stool = getToolByName(self, 'portal_schemas')
2205        schemas = ['student_application',
2206                   #'student_clearance',
2207                   ]
2208        created = False
2209        for schema_id in schemas:
2210            schema = stool._getOb(schema_id)
2211            object = getattr(student_folder,schema_id[len('student_'):],None)
2212            if object is None:
2213                continue
2214            doc = object.getContent()
2215            for key in schema.keys():
2216                if schema[key].meta_type != "CPS Image Field":
2217                    continue
2218                #import pdb;pdb.set_trace()
2219                image = getattr(doc,key)
2220                if not hasattr(image,"data"):
2221                    continue
2222                if not created:
2223                    if not os.path.exists(images_dir):
2224                        os.mkdir(images_dir)
2225                    created = True
2226                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2227                open(filename,"wb").write(str(image.data))
2228    ###)
2229
2230InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.