source: WAeUP_SRP/base/WAeUPTool.py @ 3016

Last change on this file since 3016 was 3014, checked in by joachim, 17 years ago

WAeUP LGA Widget uses new algorithm
ti_441_resolve_part1 + part2 also
part2 can now be used to rename lgas, a list of all not found lgas is
printed at the end.
  • Property svn:keywords set to Id
File size: 87.8 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3014 2008-01-11 20:18:17Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45import DateTime,time
46import logging
47import transaction
48import csv,re,os,sys
49from shutil import copy2
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52p_home = package_home(globals())
53i_home = INSTANCE_HOME
54images_base = os.path.join(i_home,"images")
55
56def getImagesDir(student_id):
57    return os.path.join("%s" % images_base,student_id[0],student_id)
58
59def getObject(object,name):
60    if object.hasObject(name):
61        return getattr(object,name)
62    return None
63
64
65class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
66    """WAeUP tool"""
67
68    id = 'waeup_tool'
69    meta_type = 'WAeUP Tool'
70    _actions = ()
71    security = ClassSecurityInfo()
72    security.declareObjectProtected(View)
73    manage_options = ( ActionProviderBase.manage_options
74                     + SimpleItem.manage_options
75                     )
76
77    security.declareProtected(View,'re_split') ###(
78    def re_split(self,split_string,string):
79        return re.split(split_string,string)
80    ###)
81
82    security.declareProtected(View,'difference') ###(
83    def difference(self,l1,l2):
84        return set(l1).difference(set(l2))
85    ###)
86
87    def rwrite(self,s): ###(
88        response = self.REQUEST.RESPONSE
89        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
90        response.write("%s<br />\r\n" % s)
91    ###)
92
93    def addtodict(self,d,key,item): ###(
94        d[key].append(item)
95        return d[key]
96    ###)
97
98    def sleep(self,secs): ###(
99        "sleep"
100        import time
101        time.sleep(secs)
102        return
103
104###)
105
106    security.declareProtected(View,'updateRoleMappingsFor') ###(
107    def updateRoleMappingsFor(self,wf_definition,ob):
108        "do so for public"
109        wf_def = getattr(self.portal_workflow,wf_definition)
110        wf_def.updateRoleMappingsFor(ob)
111    ###)
112
113    security.declareProtected(View,'getStatesLgas') ###(
114    def getStatesLgas(self):
115        """return lga info"""
116        voc = getattr(self.portal_vocabularies,'local_gov_areas')
117        states = []
118        lgas  = []
119        d = {}
120        wd = {}
121        for k,v in voc.items():
122            parts = v.split(' / ')
123            if len(parts) == 1:
124                state = parts[0].lower()
125                lga = ""
126            elif len(parts) == 2:
127                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
128                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
129            else:
130                continue
131            if state not in states:
132                states.append(state)
133            if lga not in lgas:
134                lgas.append(lga)
135            words = re.split('[^a-zA-Z0-9/]',k)
136            words.sort()
137            wd[k] = words
138            d[k] = v
139        mapping = {}
140        mapping['word_dict'] = wd
141        mapping['lga_dict'] = d
142        mapping['states'] = states
143        mapping['lgas'] = lgas
144        return mapping
145    ###)
146
147    security.declareProtected(View,'findLga') ###(
148    def findLga(self,words,words_dict):
149        words = re.split('[^a-zA-Z0-9/]',words)
150        lga_words = []
151        for word in words:
152            if word:
153                lga_words += word.strip().lower(),
154        lga_words.sort()
155        state_lga = ''
156        while not state_lga:
157            for k,l in words_dict.items():
158                if lga_words == l:
159                    state_lga = k
160                    break
161            break
162        return state_lga
163    ###)
164    security.declareProtected(View,'getAccessInfo') ###(
165    def getAccessInfo(self,context):
166        "return a dict with access_info"
167        logger = logging.getLogger('WAeUPTool.getAccessInfo')
168        mtool = self.portal_membership
169        member = mtool.getAuthenticatedMember()
170        member_id = str(member)
171        info = {}
172        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
173        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
174        is_staff = info['is_staff'] = not is_anonymous and not is_student
175        roles = member.getRolesInContext(context)
176        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
177                                                                            "SectionManager" in roles or
178                                                                            "Manager" in roles)
179        is_allowed = info['is_allowed'] = not is_anonymous
180        requested_id = context.getStudentId()
181        student_id  = None
182        if not is_student and requested_id:
183            student_id  = requested_id
184        elif is_anonymous or member_id != requested_id:
185            logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
186        else:
187            student_id = member_id
188        info['student_id'] = student_id
189        return info
190    ###)
191
192    security.declareProtected(ModifyPortalContent,'openLog') ###(
193    def openLog(self,name):
194        """open a log file"""
195        version = 1
196        path = "%s/log/%s_%d.log" % (i_home,name,version)
197        while os.path.exists(path):
198            version += 1
199            path = "%s/log/%s_%d.log" % (i_home,name,version)
200        log = open(path,"w")
201        return log
202    ###)
203
204    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
205    def bypassQueueCatalog(self,enable=True):
206        """bypass the QueueCatalog by setting all indexes to process imediate,
207        if enable is True (default) the old settings are restored
208        """
209
210    ###)
211
212    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
213    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
214        """measure Object access Time"""
215        import random
216        if hasattr(self,'portal_catalog_real'):
217            aq_portal = self.portal_catalog_real.evalAdvancedQuery
218        else:
219            aq_portal = self.portal_catalog.evalAdvancedQuery
220        nr_pts = int(nr_pts)
221        probe = int(probe)
222        intervall = probe/10
223        objects = ("application","clearance","personal")
224        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
225        #i = random.randrange(num_objects)
226        count = 0
227        found = 0
228        not_found = 0
229        t_found = 0
230        t_not_found = 0
231        time_found = time_not_found = 0.0
232        t_time_found = t_time_not_found = 0.0
233        accessed = []
234        t_min = 1000
235        t_max = 0
236        #import pdb;pdb.set_trace()
237        students = self.portal_catalog(portal_type="Student")
238        num_students = len(students)
239        if method == "d":
240            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
241            res = aq_portal(query)
242            brains = {}
243            for r in res:
244                sid = r.relative_path.split('/')[-2]
245                if brains.has_key(sid):
246                    brains[sid][r.portal_type] = r
247                else:
248                    brains[sid] = {r.portal_type : r}
249            brains_list = brains.keys()
250            num_objects = len(brains_list)
251        else:
252            num_objects = num_students
253        print "="*40
254        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
255                                                                        probe,
256                                                                        nr_pts,
257                                                                        num_objects)
258        print "nr found/not time found/not min/max"
259        elapse = time.time()
260        i_elapse = time.time()
261        c_elapse = time.clock()
262        for c in range(1,probe + 1):
263            i = random.randrange(num_objects)
264            if method in ('a','b','c'):
265                student_brain = students[i]
266            elif method == "d":
267                #import pdb;pdb.set_trace()
268                student_brain = brains[brains_list[i]]
269            if method == "c":
270                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
271                res = aq_portal(query)
272                this_portal_types = [r.portal_type for r in res]
273            for i in range(nr_pts):
274                oid = objects[i]
275                if method == "a":
276                    try:
277                        student_path = student_brain.getPath()
278                        path = "%s/%s" % (student_path,oid)
279                        doc = self.unrestrictedTraverse(path).getContent()
280                        found += 1
281                        i_time = time.time() - i_elapse
282                        time_found += i_time
283                    except:
284                        not_found += 1
285                        i_time = time.time() - i_elapse
286                        time_not_found += i_time
287                        pass
288                elif method == "b":
289                    try:
290                        student_object = student_brain.getObject()
291                        doc = getattr(student_object,oid).getContent()
292                        found += 1
293                        i_time = time.time() - i_elapse
294                        time_found += i_time
295                    except:
296                        i_time = time.time() - i_elapse
297                        time_not_found += i_time
298                        not_found += 1
299                        pass
300                elif method == "c":
301                    if portal_types[i] in this_portal_types:
302                        found += 1
303                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
304                        i_time = time.time() - i_elapse
305                        time_found += i_time
306                    else:
307                        not_found += 1
308                        i_time = time.time() - i_elapse
309                        time_not_found += i_time
310                elif method == "d":
311                    if student_brain.has_key(portal_types[i]):
312                        found += 1
313                        doc = student_brain[portal_types[i]].getObject().getContent()
314                        i_time = time.time() - i_elapse
315                        time_found += i_time
316                    else:
317                        not_found += 1
318                        i_time = time.time() - i_elapse
319                        time_not_found += i_time
320                i_elapse = time.time()
321            if c and (c % intervall == 0):
322                #i_time = time.time() - i_elapse
323                t_per = 0.0
324                if found:
325                    t_per = time_found/found
326                if t_per > t_max:
327                    t_max = t_per
328                if t_per > 0.0 and t_per < t_min:
329                    t_min = t_per
330                itf = 0.0
331                if found:
332                    itf = time_found/found
333                itnf = 0.0
334                if not_found :
335                    itnf = time_not_found / not_found
336                interval_time = time_found + time_not_found
337                s = "%(c)d: %(found)d/%(not_found)d " % vars()
338                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
339                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
340                print s
341                t_found += found
342                t_not_found += not_found
343                t_time_found += time_found
344                t_time_not_found += time_not_found
345                time_found = time_not_found = 0.0
346                found = not_found = 0
347        # t_found += found
348        # t_not_found += not_found
349        elapse = time.time() - elapse
350        itf = 0.0
351        if t_found:
352            itf = t_time_found/t_found
353        itnf = 0.0
354        if t_not_found:
355            itnf = t_time_not_found / t_not_found
356        #c_elapse = time.clock() - c_elapse
357        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
358        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
359        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
360        print "-"*40
361        print s
362        rel_found = float(t_found)/probe
363        rel_not_found = float(t_not_found)/probe
364        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
365        print estimated_total_time
366    ###)
367
368    security.declareProtected(ModifyPortalContent,'writeLog') ###(
369    def writeLog(self,logfile,s):
370        """write to the log file"""
371        logfile.write(s)
372
373###)
374
375    def generateStudentId(self,letter): ###(
376        import random
377        r = random
378        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
379        if letter == '?':
380            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
381        sid = "%c%d" % (letter,r.randint(99999,1000000))
382        students = self.portal_url.getPortalObject().campus.students
383##        while hasattr(students, sid):
384##            sid = "%c%d" % (letter,r.randint(99999,1000000))
385        while self.students_catalog(id = sid):
386            sid = "%c%d" % (letter,r.randint(99999,1000000))
387        return sid
388    ###)
389
390    def generatePassword(self,s=None): ###(
391        import random
392        r = random
393        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
394        if s is None:
395            s = 'abcdefghklmnpqrstuvwxy23456789'
396        pw = ''
397        while len(pw) < 6:
398            pw += r.choice(s)
399        return pw
400    ###)
401
402    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
403    def dumpSchoolfeePayments(self):
404        "dump paid schoolfees"
405        mtool = self.portal_membership
406        member = mtool.getAuthenticatedMember()
407        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
408        aq_student = self.students_catalog.evalAdvancedQuery
409        query = In('review_state',('schoolfee_paid',
410                                   'courses_registered',
411                                   'courses_validated',
412                                   ))
413        res = aq_student(query)
414        #import pdb;pdb.set_trace()
415        l = []
416        logger.info("start for %d" % len(res))
417        count = 1
418        log_after = 100
419        for student in res:
420            if not count % log_after:
421                logger.info("processed %d total %d" % (log_after,count))
422            count += 1
423            fee_dict =self.getSchoolFee(student)
424            fulltime = student.mode.endswith('_ft')
425            d = {}
426            d['student_id'] = student.id
427            d['name'] = student.name
428            d['amount'] = fee_dict.get(new_returning)
429            l += d,
430        csv_name = self.dumpListToCSV(l,'payments')
431        logger.info('%s dumped payments to %s' % (member,export_file))
432    ###)
433
434    security.declarePublic('dumpListToCSV') ###(
435    def dumpListToCSV(self,l,filename,fields=None):
436        """dump a list of dicts to a CSV file"""
437        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
438        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
439        if fields is None:
440            fields = l[0].keys()
441        headline = ','.join(fields)
442        out = open(export_file,"wb")
443        out.write(headline +'\n')
444        out.close()
445        out = open(export_file,"a")
446        csv_writer = csv.DictWriter(out,fields,)
447        csv_writer.writerows(l)
448        return export_file
449    ###)
450
451    security.declareProtected(ManagePortal, 'listMembers') ###(
452    def listMembers(self):
453        "list all members"
454        mtool = self.portal_membership
455        member = mtool.getAuthenticatedMember()
456        logger = logging.getLogger('WAeUPTool.listMembers')
457        if str(member) not in ('admin','joachim'):
458            logger.info('%s tried to list members' % (member))
459            return None
460        members = self.portal_directories.members
461        all = members.listEntryIdsAndTitles()
462        l = []
463        for user_id,name in all:
464            d = {}
465            d['user_id'] = user_id
466            d['name'] = name
467            d['pw'] = getattr(getattr(members,user_id),'password')
468            d['email'] = getattr(getattr(members,user_id),'email')
469            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
470            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
471            l += d,
472        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
473        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
474        logger.info('%s dumped memberlist to %s' % (member,export_file))
475        fields = l[0].keys()
476        headline = ','.join(fields)
477        out = open(export_file,"wb")
478        out.write(headline +'\n')
479        out.close()
480        out = open(export_file,"a")
481        csv_writer = csv.DictWriter(out,fields,)
482        csv_writer.writerows(l)
483    ###)
484
485    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
486    def removeDeletedDocIds(self, max=1000):
487        """
488        remove deleted docids from repository commit after max
489        """
490        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
491        repository = getToolByName(self, 'portal_repository')
492        pxtool = getToolByName(self, 'portal_proxies')
493        logger.info('start')
494        pxtool_infos = pxtool.getRevisionsUsed()
495        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
496
497        nb_revs = 0
498        docids_d = {} # all docids
499        unused_docids_d = {} # all docids that are unused
500        ids_unused_revs_docids = [] # ids for revs of unused docids
501        unused_ids = [] # ids for unused revs
502        total = 0
503        idlist = repository.objectIds()
504        to_delete = 0
505        found = False
506        for id in idlist:
507            docid, rev = repository._splitId(id)
508            if docid is None:
509                logger.info("invalid doc_id %s" % docid)
510                continue
511            nb_revs += 1
512            if not pxtool_infos.has_key(docid):
513                found = True
514                to_delete += 1
515                unused_ids.append(id)
516            elif not pxtool_infos[docid].has_key(rev):
517                found = True
518                to_delete += 1
519                unused_ids.append(id)
520            if found and not to_delete % max:
521                found = False
522                #import pdb;pdb.set_trace()
523                repository.manage_delObjects(unused_ids)
524                transaction.commit()
525                logger.info('removed %d total %d unused docids ' % (max,to_delete))
526        else:
527            if unused_ids:
528                repository.manage_delObjects(unused_ids)
529                transaction.commit()
530        logger.info('finished removing %d unused docids ' % (to_delete))
531
532
533###)
534
535    security.declareProtected(View,'getCredential') ###(
536    def getCredential(self,student_id):
537        student_entry = getattr(self.portal_directories.students,student_id,None)
538        if not self.isStaff():
539            mtool = self.portal_membership
540            member = mtool.getAuthenticatedMember()
541            logger = logging.getLogger('WAeUPTool.getCredential')
542            logger.info('%s tried to access password of %s' % (member,student_id))
543            return None
544        if student_entry is None:
545            return None
546        return getattr(student_entry,"password","not set")
547    ###)
548
549    security.declarePublic('checkPassword') ###(
550    def checkPassword(self,student_id,password):
551        student_entry = getattr(self.portal_directories.students,student_id,None)
552        if student_entry is None:
553            return False
554        return getattr(student_entry,"password","not set") == password
555    ###)
556
557    security.declarePublic('checkGenericPassword') ###(
558    def checkGenericPassword(self,member_id):
559        member_entry = getattr(self.portal_directories.members,member_id,None)
560        if member_entry is None:
561            return False
562        ltool = getToolByName(self, 'portal_layouts')
563        unsecure_words = ltool._getOb('members')['w__password'].check_words
564        password = getattr(member_entry,"password","not set")
565        is_unsecure = password in unsecure_words
566        if is_unsecure:
567            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
568            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
569        return is_unsecure
570    ###)
571
572    security.declareProtected(ModifyPortalContent,'editPassword') ###(
573    def editPassword(self,student_id,password):
574        "edit a student password"
575        student_entry = getattr(self.portal_directories.students,student_id,None)
576        if student_entry is None:
577            return
578        setattr(student_entry,'password',password)
579    ###)
580
581    security.declareProtected(ModifyPortalContent,'doCommit') ###(
582    def doCommit(self,logger=None):
583        "commit some transactions"
584        transaction.commit()
585    ###)
586
587    security.declarePublic('loadStudentFoto') ###(
588    def loadStudentFoto(self,student,filename,folder):
589        "return a student passport picture"
590        #import pdb;pdb.set_trace()
591        picture ="%s/import/%s/%s" % (i_home,folder,filename)
592        student_id = student.getId()
593        images_dir = getImagesDir(student_id)
594        if not os.path.exists(images_dir):
595            os.mkdir(images_dir)
596        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
597        for extension in ('.jpg','.JPG'):
598            fullname = "%(picture)s%(extension)s" % vars()
599            if os.path.exists(fullname):
600                copy2(fullname,image_name)
601                return "successfully copied passport picture"
602        return "passport picture not found: %s.jpg or .JPG" % picture
603    ###)
604
605    def old____loadStudentFoto(self,student,filename,folder): ###(
606        "return a student passport picture"
607        app = student.application
608        app_doc = app.getContent()
609        #clear = student.clearance
610        #clear_doc = clear.getContent()
611        #matric_no = clear_doc.matric_no.upper()
612        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
613        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
614        #import pdb;pdb.set_trace()
615        if os.path.exists(picture1):
616            file = open(picture1)
617        elif os.path.exists(picture2):
618            file = open(picture2)
619        else:
620            return "passport picture not found %s" % picture1
621        reopened = False
622        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
623            self.portal_workflow.doActionFor(app,'open')
624            reopened = True
625        outfile = file.read()
626        app_doc.manage_addFile('passport',
627                               file=outfile,
628                               title="%s.jpg" % filename)
629        if reopened:
630            self.portal_workflow.doActionFor(app,'close')
631        return "successfully loaded passport picture"
632    ###)
633
634    security.declareProtected(ModifyPortalContent,'createOne') ###(
635    def createOne(self,students_folder,student_brain,letter,commit=False):
636        sid = self.waeup_tool.generateStudentId(letter)
637        students_folder.invokeFactory('Student', sid)
638        student = getattr(students_folder,sid)
639        self.portal_workflow.doActionFor(student,'return')
640        student.manage_setLocalRoles(sid, ['Owner',])
641        matric_no = student_brain.matric_no
642        jamb_reg_no = student_brain.Entryregno
643        self.students_catalog.addRecord(id = sid,
644                                           matric_no = matric_no,
645                                           jamb_reg_no = jamb_reg_no,
646                                           sex = student_brain.Sex == "F",
647                                           name = "%s %s %s" % (student_brain.Firstname,
648                                                                student_brain.Middlename,
649                                                                student_brain.Lastname)
650                                        )
651        if commit:
652            transaction.commit()
653        return sid,jamb_reg_no
654    ###)
655
656    security.declareProtected(ModifyPortalContent,'addStudent') ###(
657    def addStudent(self,dict):
658        students_folder = self.portal_url.getPortalObject().campus.students
659        sid = self.waeup_tool.generateStudentId('?')
660        students_folder.invokeFactory('Student', sid)
661        student_obj = getattr(students_folder,sid)
662        f2t = self.student_field2types
663        #from pdb import set_trace; set_trace()
664        d = {}
665        #d['jamb_sex']  = 'M'
666        #if dict.get('sex'):
667        #    d['jamb_sex']  = 'F'
668
669        entry_session = dict.get('entry_session')
670        if entry_session == self.getSessionId()[0]:
671            wfaction = 'admit'
672            wft = 'wf_transition_admit'
673            password = None
674        else:
675            wfaction = 'return'
676            wft = 'wf_transition_return'
677            password = self.generatePassword()
678            self.makeStudentMember(sid,password)
679
680        for pt in f2t.keys():
681            student_obj.invokeFactory(pt,f2t[pt]['id'])
682            sub_obj = getattr(student_obj,f2t[pt]['id'])
683            sub_doc = sub_obj.getContent()
684            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
685            d['Title'] = f2t[pt]['title']
686            for field in f2t[pt]['fields']:
687                d[field] = dict.get(field,'')
688            sub_doc.edit(mapping = d)
689            new_state = f2t[pt][wft]
690            if new_state != "remain":
691                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
692        self.portal_workflow.doActionFor(student_obj,wfaction)
693        student_obj.manage_setLocalRoles(sid, ['Owner',])
694        return sid,password
695    ###)
696
697    security.declarePublic('getCertificateBrain') ###(
698    def getCertificateBrain(self,cert_id):
699        "do it"
700        res = ZCatalog.searchResults(self.portal_catalog_real,
701                                {'portal_type':"Certificate",
702                                      'id': cert_id})
703        if res:
704            return res[0]
705        return None
706    ###)
707
708    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
709    def get_csv_filenames(self):
710        "do it"
711        files = [file for file in os.listdir("%s/import/" % (i_home))
712                 if file.endswith('.csv') and file.find('imported') == -1]
713        return files
714    ###)
715
716    security.declarePublic('findStudentByMatricelNo') ###(
717    def findStudentByMatricelNo(self,matric_no):
718        "do it"
719        res = ZCatalog.searchResults(self.portal_catalog_real,
720                                {'portal_type':"StudentClearance",
721                                 'SearchableText': matric_no})
722        if res:
723            return res[0]
724        return None
725    ###)
726
727    security.declarePublic('makeStudentMember') ###(
728    def makeStudentMember(self,sid,password='uNsEt'):
729        """make the student a member"""
730        membership = self.portal_membership
731        membership.addMember(sid,
732                             password ,
733                             roles=('Member',
734                                     'Student',
735                                     ),
736                             domains='',
737                             properties = {'memberareaCreationFlag': False,
738                                           'homeless': True},)
739        member = membership.getMemberById(sid)
740        self.portal_registration.afterAdd(member, sid, password, None)
741        #self.manage_setLocalRoles(sid, ['Owner',])
742    ###)
743
744    security.declareProtected(View,'makeStudentData') ###(
745    def makeStudentData(self,student_id,email=None,phone_nr=None):
746        "create Datastructure for a returning Student"
747        #import pdb;pdb.set_trace()
748        logger = logging.getLogger('WAeUPTool.makeStudentData')
749        students_folder = self.portal_url.getPortalObject().campus.students
750        #res = self.students_catalog(id=student_id)
751        #if res:
752        #    st = res[0]
753        #res = self.returning_import(matric_no = st.matric_no)
754        res = self.returning_import(id = student_id)
755        if res:
756            student = res[0]
757        else:
758            logger.info('Id %s not found in returning_import' % student_id)
759            return
760        logger.info('%s creates data structure' % student_id)
761        s_results = self.results_import(matric_no = student.matric_no)
762        if s_results:
763            lnr = self.getLevelFromResultsCosCode(s_results)
764            level = "%d00" % lnr
765            verdict,eligible = self.getVerdict(s_results[0].Verdict)
766            #if eligible:
767            #    level = "%d00" % (lnr + 1)
768        else:
769            logger.info('matric_no %s not found in results_import' % student.matric_no)
770            level = ''
771            verdict = ''
772        #student should not be allowed to perform this transition
773        #wftool = self.portal_workflow
774        #wftool.doActionFor(student,'return')
775        certcode_org = student.Coursemajorcode
776        certcode = makeCertificateCode(certcode_org)
777        certificate_brain = self.getCertificateBrain(certcode)
778        if not certificate_brain:
779            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
780            logger.info(em)
781        matric_no = student.matric_no
782        sid = student_id
783        student_obj = getattr(students_folder,sid)
784        student_obj.invokeFactory('StudentApplication','application')
785        application = student_obj.application
786        self.portal_workflow.doActionFor(application,'open',dest_container=application)
787        da = {'Title': 'Application Data'}
788        student_obj.invokeFactory('StudentPersonal','personal')
789        da['jamb_reg_no'] = student.Entryregno
790        em = self.getEntryMode(student.Entryregno)
791        da['entry_mode'] = em
792        personal = student_obj.personal
793        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
794        dp = {'Title': 'Personal Data'}
795        student_obj.invokeFactory('StudentClearance','clearance')
796        clearance = student_obj.clearance
797        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
798        dc = {'Title': 'Clearance/Eligibility Record'}
799        dc['matric_no'] = matric_no
800        state = student.State
801        lga = student.LGA
802        if state and lga:
803            lga =  state + ' / ' + lga
804        else:
805            lga = "None"
806        da['jamb_lga'] = dc['lga'] = lga
807        da['app_email'] = dp['email'] = email
808        da['app_mobile'] = dp['phone'] = phone_nr
809        dp['firstname'] = student.Firstname
810        dp['middlename'] = student.Middlename
811        dp['lastname'] = student.Lastname
812        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
813        da['jamb_sex'] = student.Sex
814        dp['sex'] = student.Sex == 'F'
815        dp['perm_address'] = student.Permanent_Address
816        application.getContent().edit(mapping=da)
817        self.portal_workflow.doActionFor(application,'close',dest_container=application)
818        personal.getContent().edit(mapping=dp)
819        clearance.getContent().edit(mapping=dc)
820        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
821        #
822        # Study Course
823        #
824        student_obj.invokeFactory('StudentStudyCourse','study_course')
825        studycourse = student_obj.study_course
826        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
827        dsc = {}
828        dsc['study_course'] = certcode
829        dsc['current_level'] = level
830        dsc['current_verdict'] = verdict
831        dsc['current_mode'] = em
832        dsc['current_session'] = '05'
833        studycourse.getContent().edit(mapping=dsc)
834        #
835        # Level
836        #
837##        l = getattr(studycourse,level,None)
838##        if l is None:
839##            studycourse.invokeFactory('StudentStudyLevel', level)
840##            l = getattr(studycourse, level)
841##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
842##            l.getContent().edit(mapping={'Title': "Level %s" % level})
843###)
844
845    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
846    def admitOneStudent(self,brain,entry_session,pin_password):
847        "create Datastructure for an admitted Student"
848        #import pdb;pdb.set_trace()
849        if not hasattr(self,"_v_certificates"):
850            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
851        students_folder = self.portal_url.getPortalObject().campus.students
852        logger = logging.getLogger('WAeUPTool.admitOneStudent')
853        if brain.status != "admitted":
854            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
855            return
856        pin_parts = brain.pin.split('-')
857        if pin_parts and len(pin_parts) != 3:
858            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
859            return
860        if brain.course_admitted not in self._v_certificates:
861            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
862            return
863        if brain.sex not in (True,False):
864            logger.info('sex of %s not available' % (brain.reg_no))
865            return
866        student_id = self.generateStudentId('?')
867        students_folder.invokeFactory('Student', student_id)
868        student_object = getattr(students_folder,student_id)
869        if pin_password:
870            password = pin_parts[2]
871            self.makeStudentMember(student_id,password = password)
872        student_object.manage_setLocalRoles(student_id, ['Owner',])
873        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
874        #
875        # application
876        #
877        student_object.invokeFactory('StudentApplication','application')
878        application = student_object.application
879        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
880        da = {'Title': 'Application Data'}
881        da['jamb_reg_no'] = brain.reg_no
882
883        sex = 'M'
884        if brain.sex:
885            sex = 'F'
886        da['jamb_sex'] = sex
887        da['jamb_age'] = brain.jamb_age
888        #da['app_ac_pin'] = brain.pin
889        da['jamb_lga'] = brain.jamb_lga
890        da['jamb_state'] = brain.jamb_state
891        da['jamb_score'] = brain.aggregate
892        da['app_email'] = brain.email
893        da['app_mobile'] = brain.phone
894        if brain.entry_mode:
895            da['entry_mode'] = brain.entry_mode
896        elif brain.screening_type == 'pume':
897            da['entry_mode'] = 'ume_ft'
898        elif brain.screening_type == 'pde':
899            da['entry_mode'] = 'de_ft'
900        elif brain.screening_type == 'pce':
901            da['entry_mode'] = 'pce'
902        elif brain.screening_type == 'prence':
903            da['entry_mode'] = 'prence'
904        else:
905            da['entry_mode'] = ''
906        da['entry_session'] = entry_session
907        da['jamb_lastname'] = brain.lastname
908        da['jamb_middlename'] = brain.middlenames   # different field names!
909        da['jamb_firstname'] = brain.firstname
910        da['screening_application_date'] = brain.application_date
911        da['date_of_birth'] = brain.date_of_birth
912        da['jamb_first_cos'] = brain.course1
913        da['jamb_second_cos'] = brain.course2
914        da['course3'] = brain.course3
915        da['screening_type'] = brain.screening_type
916        da['screening_score'] = brain.screening_score
917        da['screening_date'] = brain.screening_date
918        da['hq_type'] = brain.hq_type
919        da['hq_grade'] = brain.hq_grade
920        da['aos'] = brain.aos
921
922        application.getContent().edit(mapping=da)
923        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
924        #
925        # personal
926        #
927        student_object.invokeFactory('StudentPersonal','personal')
928        personal = student_object.personal
929        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
930        dp = {'Title': 'Personal Data'}
931        dp['sex'] = brain.sex
932        dp['email'] = brain.email
933        dp['phone'] = brain.phone
934        dp['lastname'] = brain.lastname
935        dp['middlename'] = brain.middlenames   # different field names!
936        dp['firstname'] = brain.firstname
937        personal.getContent().edit(mapping=dp)
938        #
939        # clearance
940        #
941        student_object.invokeFactory('StudentClearance','clearance')
942        clearance = student_object.clearance
943        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
944        dc = {'Title': 'Clearance/Eligibility Record'}
945        dc['lga'] = brain.lga
946        dc['birthday'] = brain.date_of_birth
947        clearance.getContent().edit(mapping=dc)
948        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
949        #
950        # study Course
951        #
952        student_object.invokeFactory('StudentStudyCourse','study_course')
953        studycourse = student_object.study_course
954        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
955        dsc = {}
956        dsc['study_course'] = brain.course_admitted
957        dsc['current_verdict'] = ''
958        dsc['current_mode'] = da['entry_mode']
959        if da['entry_mode'].startswith('de'):
960            dsc['current_level'] = '200'
961        else:
962            dsc['current_level'] = '100'
963        dsc['current_session'] = entry_session
964        studycourse.getContent().edit(mapping=dsc)
965        #
966        # payments folder
967        student_object.invokeFactory('PaymentsFolder','payments')
968        payments = getattr(student_object,'payments')
969        dpay = {}
970        dpay['Title'] = 'Payments'
971        payments.getContent().edit(mapping=dpay)
972        self.portal_workflow.doActionFor(payments,'open')
973        #
974        # passport foto
975        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
976                                                              brain.screening_type,
977                                                              brain.reg_no)
978        images_dir = getImagesDir(student_id)
979        #images_dir = os.path.join("%s" % images_base,student_id)
980        letter_dir,student_dir = os.path.split(images_dir)
981        if not os.path.exists(letter_dir):
982            os.mkdir(letter_dir)
983        if not os.path.exists(images_dir):
984            os.mkdir(images_dir)
985        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
986        if os.path.exists(app_picture):
987            copy2(app_picture,image_name)
988        else:
989            logger.info('passport of %s/%s not found: %s' % (student_id,
990                                                             brain.reg_no,
991                                                             app_picture))
992
993        return student_id
994    ###)
995
996    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
997    def makeStudentLevel(self,student_id):
998        "create the StudyLevel for a returning Student"
999        #import pdb;pdb.set_trace()
1000        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1001        students_folder = self.portal_url.getPortalObject().campus.students
1002        res = self.students_catalog(id=student_id)
1003        if res:
1004            st = res[0]
1005        course = st.course
1006        matric_no = st.matric_no
1007        level = st.level
1008        res = self.results_import(matric_no = matric_no)
1009        if res:
1010            results = res
1011        logger.info('%s creating Level %s' % (student_id,level))
1012        #
1013        # Level
1014        #
1015        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1016        studycourse = getattr(student_obj,"study_course",None)
1017        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1018        l = getattr(studycourse,level,None)
1019        if l is None:
1020            studycourse.invokeFactory('StudentStudyLevel', level)
1021            l = getattr(studycourse, level)
1022            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1023            l.getContent().edit(mapping={'Title': "Level %s" % level})
1024        ###)
1025
1026    security.declarePublic('getHallInfo') ###(
1027    def getHallInfo(self,bed):
1028        """return Hall Info"""
1029        info = {}
1030        hall,block,room,letter = bed.split('_')
1031        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1032        if res and len(res) == 1:
1033            hall_brain = res[0]
1034            hall_doc = hall_brain.getObject().getContent()
1035        else:
1036            return info
1037        info['hall_title'] = hall_brain.Title
1038        info['maintenance_code'] = hall_doc.maintenance_code
1039        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1040        batch_doc = None
1041        for brain in res:
1042            if brain.id.startswith(info['maintenance_code']):
1043                batch_doc = brain.getObject().getContent()
1044                break
1045        if batch_doc is None:
1046            info['maintenance_fee'] = ''
1047        else:
1048            info['maintenance_fee'] = batch_doc.cost
1049        return info
1050    ###)
1051
1052    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1053    def removePictureFolder(self,student_id):
1054        """remove picture_folder by renaming it"""
1055        path = 'images'
1056        picture_path = os.path.join(i_home,path,student_id)
1057        if not os.path.exists(picture_path):
1058            return False
1059        os.rename(picture_path,picture_path + "_removed")
1060        return True
1061    ###)
1062
1063    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1064    def restorePictureFolder(self,student_id):
1065        """restore picture_folder by renaming it"""
1066        path = 'images'
1067        picture_path = os.path.join(i_home,path,student_id)
1068        if not os.path.exists(picture_path + "_removed"):
1069            return False
1070        os.rename(picture_path + "_removed",picture_path)
1071        return True
1072    ###)
1073
1074    security.declarePublic('picturesExist') ###(
1075    def picturesExist(self, ids,student_id=None):
1076        """check if pictures exist in the filesystem"""
1077        if student_id is None:
1078            student_id = self.getStudentId()
1079        if student_id is None:
1080            return False
1081        picture_path = getImagesDir(student_id)
1082        #picture_path = os.path.join(images_base,student_id)
1083        if not os.path.exists(picture_path):
1084            return False
1085        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1086        return set(ids).issubset(set(pictures))
1087    ###)
1088
1089    security.declarePublic('picturesList') ###(
1090    def picturesList(self):
1091        """check if pictures exist in the filesystem"""
1092        path = 'images'
1093        student_id = self.getStudentId()
1094        #picture_path = os.path.join(i_home,path,student_id)
1095        picture_path = getImagesDir(student_id)
1096        if not os.path.exists(picture_path):
1097            return []
1098        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1099    ###)
1100
1101    security.declarePublic('showFsPicture') ###(
1102    def showFsPicture(self,path):
1103        """return a picture from the filesystem"""
1104        #picture_path = os.path.join(i_home,path)
1105        picture_path = os.path.join(images_base,path)
1106        response = self.REQUEST.RESPONSE
1107        #import pdb;pdb.set_trace()
1108        registry = getToolByName(self, 'mimetypes_registry')
1109        mimetype = str(registry.lookupExtension(path.lower()) or
1110                    registry.lookupExtension('file.bin'))
1111        if os.path.exists(picture_path):
1112            response.setHeader('Content-type',mimetype)
1113            return open(picture_path).read()
1114        picture_path = os.path.join(i_home,'import',path)
1115        if os.path.exists(picture_path):
1116            return open(picture_path).read()
1117    ###)
1118
1119    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1120    def deleteAllCourses(self,department="All"):
1121        ''' delete the courses'''
1122        pm = self.portal_membership
1123        member = pm.getAuthenticatedMember()
1124
1125        if str(member) not in ("henrik","joachim"):
1126            return "not possible"
1127        if department == "All":
1128            res = self.portal_catalog({'meta_type': 'Department'})
1129        if len(res) < 1:
1130            return "No Departments found"
1131
1132        deleted = []
1133        for dep in res:
1134            cf = dep.getObject().courses
1135            if cf:
1136                cf.manage_delObjects(ids=cf.objectIds())
1137                deleted.append("deleted Courses in %s" % dep.getId)
1138        return "\r".join(deleted)
1139    ###)
1140
1141    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1142    def getLogfileLines(self,filename="event.log",numlines=20):
1143        """Get last NUMLINES lines of logfile FILENAME.
1144
1145        Return last lines' of a file in the instances logfile directory as
1146        a list. The number of returned lines equals `numlines' or less. If
1147        less than `numlines' lines are available, the whole file ist
1148        returned. If the file can not be opened or some other error
1149        occurs, empty list is returend.
1150        """
1151        result = []
1152        lines_hit = 0
1153
1154        # We only handle files in instances' log directory...
1155        logpath = os.path.join(i_home, "log")
1156        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1157        if not filename.startswith( logpath ):
1158            # Attempt to access file outside log-dir...
1159            return []
1160
1161        try:
1162            fd = file( filename, "rb" )
1163        except IOError:
1164            return []
1165        if not fd:
1166            return []
1167
1168        if os.linesep == None:
1169            linesep = '\n'
1170        else:
1171            linesep = os.linesep
1172
1173        # Try to find 'numlines' times a lineseparator, searching from end
1174        # and moving to the beginning of file...
1175        fd.seek( 0, 2) # Move to end of file...
1176        while lines_hit < numlines:
1177            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1178                                          # one step forward...
1179                lines_hit += 1
1180            try:
1181                fd.seek( -2, 1) # Go two bytes back from current pos...
1182            except IOError:
1183                # We cannot go back two bytes. Maybe the file is too small...
1184                break
1185        fd.seek(2,1)
1186
1187        # Read all lines from current position...
1188        result = fd.readlines()
1189        # Remove line endings...
1190        result = [x.strip() for x in result]
1191        fd.close()
1192        return result
1193    ###)
1194
1195    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1196    def getCallbacksFromLog(self,filename):
1197        """fix Online Payment Transactions from Z2.log entries"""
1198        import transaction
1199        import random
1200        from cgi import parse_qs
1201        from urlparse import urlparse
1202        #from pdb import set_trace
1203        wftool = self.portal_workflow
1204        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1205        students_folder = self.portal_url.getPortalObject().campus.students
1206        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1207        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1208        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1209        data = re.compile(s)
1210        start = True
1211        tr_count = 1
1212        total = 0
1213        #name = 'pume_results'
1214        #name = 'epaymentsuccessful_z2log2'
1215        name = filename
1216        no_import = []
1217        imported = []
1218        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1219        try:
1220            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1221        except:
1222            logger.error('Error reading %s' % name)
1223            return
1224        tas = []
1225        for line in transactions:
1226            dict = {}
1227            items = data.search(line)
1228            dict['idict'] = idict = items.groupdict()
1229            #print idict
1230            #from pdb import set_trace;set_trace()
1231            urlparsed = urlparse(idict['get'][4:])
1232            #print urlparsed
1233            path = urlparsed[2].split('/')
1234            dict['student_id'] = student_id = path[8]
1235            dict['payment_id'] = payment_id = path[10]
1236            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1237            tas.append(dict)
1238            tr_count += 1
1239        return tas
1240    ###)
1241
1242    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1243    def importOnlinePaymentTransactions(self):
1244        """load Online Payment Transactions from CSV values"""
1245        import transaction
1246        import random
1247        #from pdb import set_trace
1248        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1249        opt = self.online_payments_import
1250        students_folder = self.portal_url.getPortalObject().campus.students
1251        start = True
1252        tr_count = 1
1253        total = 0
1254        #name = 'pume_results'
1255        name = 'OnlineTransactions'
1256        no_import = []
1257        imported = []
1258        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1259        try:
1260            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1261        except:
1262            logger.error('Error reading %s.csv' % name)
1263            return
1264        for pay_transaction in transactions:
1265            if start:
1266                start = False
1267                logger.info('Start loading from %s.csv' % name)
1268                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1269                no_import.append('%s,"Error"' % s)
1270                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1271                format_error = format + ',"%(Error)s"'
1272            data = {}
1273
1274            # format of the first file sent by Tayo
1275            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1276            #data['student_id'] = student_id = pay_transaction['Payer ID']
1277            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1278            #data['response_code'] = response_code = pay_transaction['Resp Code']
1279            #data['amount'] = amount = pay_transaction['Amount']
1280
1281            # format of the second file sent by Tayo
1282            #data['datetime'] = date = 0
1283            #data['student_id'] = student_id = pay_transaction['Payer ID']
1284            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1285            #data['response_code'] = response_code = '00'
1286            #data['amount'] = amount = pay_transaction['Amount']
1287
1288            # format of the third file sent by Kehinde
1289            data['datetime'] = date = 0
1290            data['student_id'] = student_id = pay_transaction['customer_id']
1291            data['order_id'] = order_id = pay_transaction['merchant_reference']
1292            data['response_code'] = response_code = '00'
1293            data['amount'] = amount = pay_transaction['Amount']
1294
1295            dup = False
1296            if response_code == "12":
1297                continue
1298            try:
1299                opt.addRecord(**data)
1300            except ValueError:
1301                dup = True
1302            #from pdb import set_trace;set_trace()
1303            if dup:
1304                if response_code == "00":
1305                    try:
1306                        opt.modifyRecord(**data)
1307                    except:
1308                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1309                        continue
1310                else:
1311                    pay_transaction['Error'] = "Duplicate order_id"
1312                    no_import.append( format_error % pay_transaction)
1313                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1314                    continue
1315            tr_count += 1
1316            if tr_count > 1000:
1317                if len(no_import) > 0:
1318                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1319                             '\n'.join(no_import) + '\n')
1320                    no_import = []
1321                em = '%d transactions committed\n' % (tr_count)
1322                transaction.commit()
1323                regs = []
1324                logger.info(em)
1325                total += tr_count
1326                tr_count = 0
1327        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1328                                                '\n'.join(no_import))
1329        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1330    ###)
1331
1332    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
1333    def mass_create_faculty(self,mapping):
1334        "create a faculty"
1335        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
1336        academics_folder = self.portal_url.getPortalObject().campus.academics
1337        fid = mapping['code']
1338        if getattr(academics_folder,fid,None) is not None:
1339            return '', "Faculty with ID: %s exists" % fid
1340        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
1341        try:
1342            academics_folder.invokeFactory('Faculty', fid)
1343        except BadRequest,E:
1344            return '', "%s" % E
1345        f = getattr(academics_folder,fid,None)
1346        f.getContent().edit(mapping=mapping)
1347        return fid,''
1348    ###)
1349
1350    security.declareProtected(ModifyPortalContent,'mass_edit_faculty') ###(
1351    def mass_edit_faculty(self,mapping):
1352        "edit a faculty"
1353        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
1354        academics_folder = self.portal_url.getPortalObject().campus.academics
1355        fid = mapping['code']
1356        f = getattr(academics_folder,fid,None)
1357        if f is None:
1358            return '', "Faculty with ID: %s does not exist" % fid
1359        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
1360        f.getContent().edit(mapping=mapping)
1361        return fid,''
1362    ###)
1363
1364    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
1365    def mass_create_department(self,mapping):
1366        "create a department in the correct faculty"
1367        logger = logging.getLogger('WAeUPTool.mass_create_department')
1368        fid = mapping['faculty_code']
1369        if getattr(self,'_v_faculties',None) is None:
1370            res = self.portal_catalog(portal_type = "Faculty")
1371            self._v_faculties = {}
1372            for f in res:
1373                self._v_faculties[f.getId] = f.getObject()
1374        f = self._v_faculties.get(fid,None)
1375        if f is None:
1376            return '', "No Faculty with ID: %s" % fid
1377        else:
1378            did = mapping.get('code')
1379            d = getattr(f,did,None)
1380            if d is None or d.portal_type == "Faculty":
1381                logger.info('Creating Department %(code)s, %(title)s' % mapping)
1382                try:
1383                    f.invokeFactory('Department', did)
1384                except BadRequest,E:
1385                    return '', "%s" % E
1386                d = getattr(f,did)
1387                d.invokeFactory('CoursesFolder','courses')
1388                courses = getattr(d,'courses')
1389                dict = {'Title': 'Courses'}
1390                courses.getContent().edit(mapping=dict)
1391                d.invokeFactory('CertificatesFolder','certificates')
1392                certificates = getattr(d,'certificates')
1393                dict = {'Title': 'Certificates'}
1394                certificates.getContent().edit(mapping=dict)
1395            d.getContent().edit(mapping=mapping)
1396        return did,''
1397    ###)
1398
1399    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
1400    def mass_edit_department(self,mapping):
1401        "create a department in the correct faculty"
1402        logger = logging.getLogger('WAeUPTool.mass_create_department')
1403        academics_folder = self.portal_url.getPortalObject().campus.academics
1404        fid = mapping['faculty_code']
1405        did = mapping.get('code')
1406        try:
1407            d = getattr(getattr(academics_folder,fid),did,None)
1408        except KeyError:
1409            return '', "Department %s or Faculty %s wrong" % (did,fid)
1410        else:
1411            if d is None or d.portal_type == "Faculty":
1412                logger.info('Editing Department %(code)s, %(title)s' % mapping)
1413            d.getContent().edit(mapping=mapping)
1414        return did,''
1415    ###)
1416
1417    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
1418    def mass_create_course(self,mapping):
1419        #import pdb;pdb.set_trace()
1420        if getattr(self,'_v_course_list',None) is None:
1421            self._v_course_list = []
1422        if getattr(self,'_v_departments',None) is None:
1423            res = self.portal_catalog(portal_type = "Department")
1424            self._v_department_courses = {}
1425            for d in res:
1426                self._v_department_courses[d.getId] = getattr(d.getObject(),"courses",None)
1427        did = mapping['department_code']
1428        d = self._v_department_courses.get(did,None)
1429        if d is None:
1430            return '', "No Department with ID: %s" % did
1431        course_id = mapping.get('code')
1432        if course_id in self._v_course_list:
1433            return '', "Duplicate Course ID: %s" % did
1434        c = getattr(d,course_id,None)
1435        if c is not None:
1436            return '', "Duplicate Course ID: %s" % did
1437        try:
1438            d.invokeFactory('Course', course_id)
1439        except BadRequest,E:
1440            return '', "%s" % E
1441        self._v_course_list.append(course_id)
1442        c = getattr(d,course_id)
1443        c.getContent().edit(mapping=mapping)
1444        return course_id,''
1445    ###)
1446
1447    security.declareProtected(ModifyPortalContent,'mass_edit_course') ###(
1448    def mass_edit_course(self,mapping):
1449        #import pdb;pdb.set_trace()
1450        course_id = mapping.get('code')
1451        res = self.portal_catalog(id=course_id)
1452        if not res:
1453            return '', "No Course with ID: %s" % course_id
1454        c = res[0].getObject()
1455        c.getContent().edit(mapping=mapping)
1456        return course_id,''
1457    ###)
1458
1459    security.declareProtected(ModifyPortalContent,'mass_create_certificate') ###(
1460    def mass_create_certificate(self,mapping):
1461        if getattr(self,'_v_certificate_list',None) is None:
1462            self._v_certificate_list = []
1463        if getattr(self,'_v_department_certificates',None) is None:
1464            res = self.portal_catalog(portal_type = "Department")
1465            self._v_department_certificates = {}
1466            for d in res:
1467                self._v_department_certificates[d.getId] = getattr(d.getObject(),"certificates",None)
1468        did = mapping['department_code']
1469        d = self._v_department_certificates.get(did,None)
1470        if d is None:
1471            return '', "No Department with ID: %s" % did
1472        certificate_id = mapping.get('code')
1473        if certificate_id in self._v_certificate_list:
1474            return '', "Duplicate Certificate ID: %s" % did
1475        c = getattr(d,certificate_id,None)
1476        if c is not None:
1477            return '', "Duplicate Certificate ID: %s" % did
1478        try:
1479            d.invokeFactory('Certificate', certificate_id)
1480        except BadRequest,E:
1481            return '', "%s" % E
1482        self._v_certificate_list.append(certificate_id)
1483        c = getattr(d,certificate_id)
1484        c.getContent().edit(mapping=mapping)
1485        return certificate_id,''
1486    ###)
1487
1488    security.declareProtected(ModifyPortalContent,'mass_edit_certificate') ###(
1489    def mass_edit_certificate(self,mapping):
1490        #import pdb;pdb.set_trace()
1491        certificate_id = mapping.get('code')
1492        res = self.portal_catalog(id=certificate_id)
1493        if not res:
1494            return '', "No Certificate with ID: %s" % did
1495        c = res[0].getObject()
1496        c.getContent().edit(mapping=mapping)
1497        return certificate_id,''
1498    ###)
1499
1500    security.declareProtected(ModifyPortalContent,'mass_create_application') ###(
1501    def mass_create_application(self,mapping):
1502        #import pdb;pdb.set_trace()
1503        reg_no = mapping.get('reg_no')
1504        try:
1505            self.applicants_catalog.addRecord(**mapping)
1506        except ValueError:
1507            return '', "applicant record with reg_no %s already exists" % reg_no
1508        return reg_no,''
1509    ###)
1510
1511    security.declareProtected(ModifyPortalContent,'mass_edit_application') ###(
1512    def mass_edit_application(self,mapping):
1513        #import pdb;pdb.set_trace()
1514        reg_no = mapping.get('reg_no')
1515        status = mapping.get('status')
1516        res = self.applicants_catalog(reg_no = reg_no)
1517        if len(res):
1518            if res[0].status == 'created' and status != 'created':
1519                return '', "student object with id %s for %s already created, status cannot be changed" % (res[0].student_id, reg_no)
1520            elif status == 'created' and res[0].status != 'created':
1521                return '', "student object for %s has not yet been created, status cannot be set to 'created'" % (reg_no)
1522            else:
1523                self.applicants_catalog.modifyRecord(**mapping)
1524                return reg_no,''
1525        else:
1526            return '', "applicant record with reg_no %s does not exist" % reg_no
1527
1528    ###)
1529
1530    security.declareProtected(ModifyPortalContent,'mass_create_course_result') ###(
1531    def mass_create_course_result(self,mapping):
1532        #import pdb;pdb.set_trace()
1533        students_folder = self.portal_url.getPortalObject().campus.students
1534        if getattr(self,'_v_courses',None) is None:
1535            res = self.courses_catalog()
1536            self._v_courses = {}
1537            for brain in res:
1538                self._v_courses[brain.code] = brain
1539        if getattr(self,'_v_level_created',None) is None:
1540            self._v_level_created = []
1541        course_id = mapping.get('code')
1542        if course_id not in self._v_courses.keys():
1543            return '', "No course with ID: %s" % course_id
1544        id_key = ''
1545        for id_key in ('student_id','matric_no'):
1546            id_field = mapping.get(id_key,None)
1547            if id_field is not None:
1548                student_id = id_field
1549                break
1550        query = Eq(id_key,id_field)
1551        res = self.students_catalog.evalAdvancedQuery(query)
1552        if not res:
1553            return '', "no student with %(id_key)s %(id_field)s" % vars()
1554        student_rec = res[0]
1555        if id_field != "student_id":
1556            mapping['student_id'] = student_rec.id
1557        student_id = student_rec.id
1558        level_id = mapping['level_id']
1559        code = mapping['code']
1560        if student_id not in self._v_level_created:
1561            try:
1562                context = getattr(getattr(students_folder,
1563                                          "%(student_id)s" % vars()),
1564                                  'study_course')
1565            except:
1566                return '', "could not create level %(level_id)s for %(student_id)s" % vars()
1567            if level_id not in context.objectIds():
1568                context.invokeFactory('StudentStudyLevel',"%(level_id)s" % vars())
1569                level = getattr(context,"%(level_id)s" % vars())
1570                self.portal_workflow.doActionFor(level,'open')
1571                # the session string must not be copied into the level object
1572                current_verdict = getattr(student_rec,'current_verdict','')
1573                current_session = getattr(student_rec,'current_session','')
1574                if current_verdict and student_rec.current_level == level_id:
1575                    level.getContent().edit(mapping={'verdict': "%s" %
1576                                                     current_verdict,
1577                                                     'session': "%s" %
1578                                                     current_session,
1579                                                    })
1580                    self.portal_workflow.doActionFor(level,'close')
1581            self._v_level_created += student_id,
1582        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1583        for k in ('semester','credits',):
1584            mapping[k] = getattr(self._v_courses[course_id],k)
1585        try:
1586            self.course_results.addRecord(**mapping)
1587        except ValueError:
1588            return '', "course result already exists: %s" % key
1589        return key,''
1590    ###)
1591
1592    security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
1593    def mass_edit_course_result(self,mapping):
1594        #import pdb;pdb.set_trace()
1595        # if getattr(self,'_v_courses',None) is None:
1596        #     res = self.courses_catalog()
1597        #     self._v_courses = {}
1598        #     for brain in res:
1599        #         self._v_courses[brain.code] = brain
1600        # course_id = mapping.get('code')
1601        # if course_id not in self._v_courses.keys():
1602        #     return '', "No course with ID: %s" % did
1603        id_key = ''
1604        for id_key in ('student_id','matric_no'):
1605            id_field = mapping.get(id_key,None)
1606            if id_field is not None:
1607                student_id = id_field
1608                break
1609        query = Eq(id_key,id_field)
1610        res = self.students_catalog.evalAdvancedQuery(query)
1611        if not res:
1612            return '', "No student with %(id_field)s: %(id_key)s" % vars()
1613        if id_field != "student_id":
1614            mapping['student_id'] = res[0].id
1615        if not res:
1616            return '', "no student with %(id_key)s %(id_field)s" % vars()
1617        student_rec = res[0]
1618        student_id = student_rec.id
1619        level_id = mapping['level_id']
1620        code = mapping['code']
1621        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
1622        # for k in ('semester',):
1623        #     mapping[k] = getattr(self._v_courses[course_id],k)
1624        try:
1625            self.course_results.modifyRecord(**mapping)
1626        except KeyError:
1627            return '', "No course result to edit: %s" % key
1628        return key,''
1629    ###)
1630
1631    security.declareProtected(ModifyPortalContent,'mass_create_certificate_course') ###(
1632    def mass_create_certificate_course(self,mapping):
1633        if getattr(self,'_v_courses',None) is None:
1634            res = self.courses_catalog()
1635            self._v_courses= [course.code for course in res]
1636        if getattr(self,'_v_ceritficates',None) is None:
1637            res = self.portal_catalog(portal_type = "Certificate")
1638            self._v_certificates = {}
1639            for cert in res:
1640                self._v_certificates[cert.getId] = cert.getObject()
1641        certificate_course_id = mapping.get('code')
1642        if certificate_course_id not in self._v_courses:
1643            return '', "No Course with ID: %s" % certificate_course_id
1644        cert_id = mapping['certificate_code']
1645        cert = self._v_certificates.get(cert_id,None)
1646        if cert is None:
1647            return '', "No Certificate with ID: %s" % cert_id
1648        level_id = mapping.get('level')
1649        level = getattr(cert,level_id,None)
1650        if level is None:
1651            cert.invokeFactory('StudyLevel', level_id)
1652            level = getattr(cert,level_id,None)
1653        elif hasattr(level,certificate_course_id):
1654            return '', "Duplicate CertificateCourse ID: %s in %s/ %s" %\
1655            (certificate_course_id,cert_id,level_id)
1656        level.invokeFactory('CertificateCourse', certificate_course_id)
1657        c = getattr(level,certificate_course_id)
1658        c.getContent().edit(mapping=mapping)
1659        return certificate_course_id,''
1660    ###)
1661
1662    field2types_student = {   ###(
1663                      'StudentApplication':
1664                          {'id': 'application',
1665                           'title': 'Application Data',
1666                           'wf_transition_return': 'close',
1667                           'wf_transition_admit': 'remain',
1668                           'fields':
1669                             ('jamb_reg_no',
1670                              'entry_mode',
1671                              'entry_session',
1672                              'jamb_score',
1673                              'app_email',
1674                              'jamb_age',
1675                              'jamb_state',
1676                              'jamb_lga',
1677                              'jamb_sex',
1678                              )
1679                              },
1680                      #'StudentPume':
1681                      #    {'id': 'pume',
1682                      #     'title': 'Pume Data',
1683                      #     'wf_transition_return': 'close',
1684                      #     'wf_transition_admit': 'close',
1685                      #     'fields':
1686                      #       ('pume_score',
1687                      #        )
1688                      #        },
1689                      'StudentClearance':
1690                          {'id': 'clearance',
1691                           'title': 'Clearance/Eligibility Record',
1692                           'wf_transition_return': 'close',
1693                           'wf_transition_admit': 'remain',
1694                           'fields':
1695                             ('matric_no',
1696                              'nationality',
1697                              'lga',
1698                              'birthday',
1699                              )
1700                              },
1701                         'StudentPersonal':
1702                          {'id': 'personal',
1703                           'title': 'Personal Data',
1704                           'wf_transition_return': 'open',
1705                           'wf_transition_admit': 'remain',
1706                           'fields':
1707                             ('firstname',
1708                              'middlename',
1709                              'lastname',
1710                              'sex',
1711                              'email',
1712                              'phone',
1713                              'perm_address',
1714                              )
1715                              },
1716                         'StudentStudyCourse':
1717                          {'id': 'study_course',
1718                           'title': 'Study Course',
1719                           'wf_transition_return': 'open',
1720                           'wf_transition_admit': 'remain',
1721                           'fields':
1722                             ('study_course',
1723                              'current_level',
1724                              'current_session',
1725                              'current_mode',
1726                              'current_verdict',
1727                              'previous_verdict',
1728                              )
1729                              },
1730                         # 'StudentStudyLevel':
1731                         #  {'id': 'current_level',
1732                         #   'title': '',
1733                         #   'wf_transition_return': 'open',
1734                         #   'wf_transition_admit': 'remain',
1735                         #   'fields':
1736                         #     ('verdict',
1737                         #      'session',
1738                         #      )
1739                         #      },
1740                         'PaymentsFolder':
1741                          {'id': 'payments',
1742                           'title': 'Payments',
1743                           'wf_transition_return': 'open',
1744                           'wf_transition_admit': 'open',
1745                           'fields':
1746                             ()
1747                              },
1748                         }
1749    ###)
1750
1751    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1752    def mass_create_student(self,mapping):
1753        "create a students record due import"
1754        logger = logging.getLogger('WAeUPTool.mass_create_student')
1755        students_folder = self.portal_url.getPortalObject().campus.students
1756        jamb_reg_no = mapping.get('jamb_reg_no',None)
1757        if jamb_reg_no:
1758            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1759            if res:
1760                return '',"jamb_reg_no exists"
1761        matric_no = mapping.get('matric_no',None)
1762        if matric_no:
1763            res = self.students_catalog(matric_no = matric_no)
1764            if res:
1765                return '',"matric_no exists"
1766        sid = self.waeup_tool.generateStudentId('?')
1767        students_folder.invokeFactory('Student', sid)
1768        student_obj = getattr(students_folder,sid)
1769        f2t = self.field2types_student
1770        d = {}
1771        transition = mapping.get('reg_transition','admit')
1772        if transition not in ('admit','return'):
1773            return '',"no valid transition provided"
1774        for pt in f2t.keys():
1775            student_obj.invokeFactory(pt,f2t[pt]['id'])
1776            sub_obj = getattr(student_obj,f2t[pt]['id'])
1777            sub_doc = sub_obj.getContent()
1778            d['Title'] = f2t[pt]['title']
1779            for field in f2t[pt]['fields']:
1780                d[field] = mapping.get(field,'')
1781
1782            if pt == "StudentApplication":
1783                #d['jamb_sex']  = 'M'
1784                #if mapping.get('sex'):
1785                #    d['jamb_sex']  = 'F'
1786                d['jamb_firstname'] = mapping.get('firstname',None)
1787                d['jamb_middlename'] = mapping.get('middlename',None)
1788                d['jamb_lastname'] = mapping.get('lastname',None)
1789
1790            # if pt == "StudyCourse":
1791            #     for von,zu in (('entry_mode','current_mode'),
1792            #                    ('entry_session','current_session')):
1793            #         if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1794            #             d[zu] = mapping[von]
1795            sub_doc.edit(mapping = d)
1796
1797            #import pdb;pdb.set_trace()
1798            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1799            if new_state != "remain":
1800                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1801        self.portal_workflow.doActionFor(student_obj,transition)
1802        student_obj.manage_setLocalRoles(sid, ['Owner',])
1803        return sid,''
1804    ###)
1805
1806    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1807    def mass_edit_student(self,mapping):
1808        wftool = self.portal_workflow
1809        "edit a students record due import"
1810        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1811        students_folder = self.portal_url.getPortalObject().campus.students
1812        sid = mapping.get('id',None)
1813        jamb_reg_no = mapping.get('jamb_reg_no',None)
1814        matric_no = mapping.get('matric_no',None)
1815        editable_keys = mapping.keys()
1816        if sid:
1817            res = self.students_catalog(id = sid)
1818            if not res:
1819                return '',"no student with id %s" % sid
1820            if matric_no and res[0].matric_no and\
1821              matric_no != res[0].matric_no:
1822                logger.info("%s, old matric_no %s overwritten with %s" % (res[0].id,res[0].matric_no,matric_no))
1823            if jamb_reg_no and res[0].jamb_reg_no and\
1824              jamb_reg_no != res[0].jamb_reg_no:
1825                logger.info("%s, old reg_no %s overwritten with %s" % (res[0].id,res[0].jamb_reg_no,jamb_reg_no))
1826        elif jamb_reg_no:
1827            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1828            if not res:
1829                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1830            editable_keys.remove('jamb_reg_no')
1831        elif matric_no:
1832            res = self.students_catalog(matric_no = matric_no)
1833            if not res:
1834                return '',"no student with matric_no %s" % matric_no
1835            editable_keys.remove('matric_no')
1836
1837        ## included only to change wf state from admitted to returning
1838        #if res[0].review_state not in ('admitted','objection_raised'):
1839        #    return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1840        ## end inclusion
1841
1842        sid = res[0].id
1843        student_obj = getattr(students_folder,sid)
1844        f2t = self.field2types_student
1845        d = {}
1846        #import pdb;pdb.set_trace()
1847        any_change = False
1848        #special treatment for StudentStudyLevel
1849        d['verdict']  = mapping.get('current_verdict','')
1850        d['session']  = mapping.get('current_session','')
1851        current_level = mapping.get('current_level','')
1852        while d['session'] and d['verdict'] and current_level:
1853            sub_obj = getattr(student_obj,'study_course',None)
1854            if sub_obj is None:
1855                break
1856            level_obj = getattr(sub_obj,current_level,None)
1857            if  level_obj is None:
1858                break
1859            any_change = True
1860            level_obj.getContent().edit(mapping = d)
1861            try:
1862                wftool.doActionFor(level_obj,'close')
1863            except:
1864                pass
1865            break
1866        for pt in f2t.keys():
1867            #if pt == "StudentApplication":
1868            #    d['jamb_sex']  = 'M'
1869            #    if mapping.get('sex'):
1870            #        d['jamb_sex']  = 'F'
1871            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1872            if intersect and pt not in ('StudentStudyLevel',):
1873                object_id = f2t[pt]['id']
1874                sub_obj = getattr(student_obj,object_id,None)
1875                if sub_obj is None:
1876                    try:
1877                        student_obj.invokeFactory(pt,object_id)
1878                    except:
1879                        continue
1880                    sub_obj = getattr(student_obj,object_id)
1881                    if f2t[pt]['title'] != '':
1882                        d['Title'] = f2t[pt]['title']
1883                sub_doc = sub_obj.getContent()
1884                for field in intersect:
1885                    changed = False
1886                    if getattr(sub_doc,field,None) != mapping.get(field,''):
1887                        any_change = True
1888                        changed = True
1889                        d[field] = mapping.get(field,'')
1890                    if changed:
1891                        sub_doc.edit(mapping = d)
1892
1893
1894        ## included only to change wf state from admitted to returning
1895        #    if res[0].review_state in ('admitted','objection_raised'):
1896        #        new_state = f2t[pt]['wf_transition_return']
1897        #        sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1898        #        if sub_obj and new_state != "remain":
1899        #            try:
1900        #                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1901        #            except:
1902        #                #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1903        #                pass
1904        #if res[0].review_state in ('admitted','objection_raised'):
1905        #    wfaction = 'return'
1906        #    try:
1907        #        self.portal_workflow.doActionFor(student_obj,wfaction)
1908        #        logger.info('%s, wf state changed' % sid)
1909        #        any_change = True
1910        #    except:
1911        #        logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1912        #        pass
1913        ## end inclusion
1914
1915
1916        if any_change:
1917            return sid,''
1918        else:
1919            return sid,'not modified'
1920    ###)
1921
1922    security.declareProtected(ModifyPortalContent,"importData")###(
1923    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1924        """load data from CSV values"""
1925        import transaction
1926        import random
1927
1928        pm = self.portal_membership
1929        member = pm.getAuthenticatedMember()
1930
1931        logger = logging.getLogger('WAeUPTool.importData')
1932        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1933        students_folder = self.portal_url.getPortalObject().campus.students
1934        start = True
1935        tr_count = 0
1936        total_imported = 0
1937        total_not_imported = 0
1938        total = 0
1939        iname = "import_%s" % name
1940        if iname in ('import_application','import_course_result',):
1941            commit_after = 2000
1942        else:
1943            commit_after = 100
1944        stool = getToolByName(self, 'portal_schemas')
1945        ltool = getToolByName(self, 'portal_layouts')
1946        schema = stool._getOb(iname)
1947        if schema is None:
1948            em = 'No such schema %s' % iname
1949            logger.error('No such schema %s' % iname)
1950            return em
1951        layout = ltool._getOb(iname)
1952        if layout is None:
1953            em = 'No such layout %s' % iname
1954            logger.error(em)
1955            return em
1956        validators = {}
1957        for widget in layout.keys():
1958            validators[widget] = layout[widget].validate
1959        mode = "create"
1960        if edit:
1961            mode = "edit"
1962        importer_name = "mass_%(mode)s_%(name)s" % vars()
1963        importer = getattr(self, '%s' % importer_name,None)
1964        if importer is None:
1965            em = 'No importer function %s' % importer_name
1966            logger.error(em)
1967            return em
1968        not_imported = []
1969        imported = []
1970        try:
1971            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1972        except:
1973            em = 'Error reading %s.csv' % filename
1974            logger.error(em)
1975            return em
1976        for item in items:
1977            #import pdb;pdb.set_trace()
1978            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1979            if start:
1980                start = False
1981                adapters = [MappingStorageAdapter(schema, item)]
1982                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1983                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1984                import_keys = [k.strip() for k in attrs if not k.startswith('ignore')]
1985                diff2schema = set(import_keys).difference(set(schema.keys()))
1986                diff2layout = set(import_keys).difference(set(layout.keys()))
1987                if diff2schema:
1988                    em = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1989                    return em
1990                if mode == "create":
1991                    required_keys = [layout.getIdUnprefixed(id)
1992                                     for id,widget in layout.objectItems()
1993                                     if widget.is_required]
1994                    if not set(required_keys).issubset(set(import_keys)):
1995                        diff2import = set(required_keys).difference(set(import_keys))
1996                        em = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1997                        return em
1998                s = ','.join(['"%s"' % fn for fn in import_keys])
1999                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
2000                s = '"id",' + s
2001                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
2002                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
2003                format_error = format + ',"%(Error)s"'
2004                format = '"%(id)s",'+ format
2005
2006            dm = DataModel(item, adapters,context=self)
2007            ds = DataStructure(data=item,datamodel=dm)
2008            error_string = ""
2009            total += 1
2010            error_count = 0
2011            for k in import_keys:
2012                if not validators[k](ds,mode=mode):
2013                    if error_count:
2014                        error_string += ' ++ '
2015                    error_string += "%s: %s" % (k,
2016                                                  self.translation_service(ds.getError(k),
2017                                                                           ds.getErrorMapping(k)))
2018                    error_count += 1
2019            if not error_string:
2020                temp_item = item.copy()
2021                temp_item.update(dm)
2022                temp_item['id'],error = importer(temp_item)
2023                if error:
2024                    error_string += error
2025                else:
2026                    item = temp_item
2027            if error_string:
2028                item['Error'] = error_string
2029                not_imported.append(format_error % item)
2030                total_not_imported += 1
2031            else:
2032                em = format % item
2033                imported.append(em)
2034                tr_count += 1
2035                total_imported += 1
2036                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
2037
2038            if total and not total % commit_after:
2039                transaction.commit()
2040                if len(not_imported) > 0:
2041                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2042                             '\n'.join(not_imported) + '\n')
2043                    not_imported = []
2044                if len(imported) > 0:
2045                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2046                             '\n'.join(imported) + '\n')
2047                    imported = []
2048                em = '%d transactions committed\n' % (tr_count)
2049                regs = []
2050                logger.info(em)
2051                tr_count = 0
2052                #if total > 100:
2053                #    return
2054        if len(imported) > 0:
2055            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
2056                                                '\n'.join(imported))
2057        if len(not_imported) > 0:
2058            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
2059                                                '\n'.join(not_imported))
2060        em = "Finished import from %s.csv: %d imported, %d not imported (of total %d)" % (filename,total_imported,total_not_imported,total)
2061        logger.info(em)
2062        return em
2063    ###)
2064
2065    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
2066    def moveImagesToFS(self,student_id="O738726"):
2067        "move the images to the filesystem"
2068        images_dir = getImagesDir(student_id)
2069        #images_dir = os.path.join("%s" % images_base,student_id)
2070        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2071        stool = getToolByName(self, 'portal_schemas')
2072        schemas = ['student_application',
2073                   'student_clearance',
2074                   ]
2075        created = False
2076        for schema_id in schemas:
2077            schema = stool._getOb(schema_id)
2078            object = getattr(student_folder,schema_id[len('student_'):],None)
2079            if object is None:
2080                continue
2081            doc = object.getContent()
2082            for key in schema.keys():
2083                if schema[key].meta_type != "CPS Image Field":
2084                    continue
2085                #import pdb;pdb.set_trace()
2086                image = getattr(doc,key,None)
2087                if not image or not hasattr(image,"data"):
2088                    continue
2089                if not created:
2090                    if not os.path.exists(images_dir):
2091                        os.mkdir(images_dir)
2092                    created = True
2093                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
2094                open(filename,"wb").write(str(image.data))
2095    ###)
2096
2097    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
2098    def movePassportToFS(self,student_id="O738726"):
2099        "move the passports to the filesystem"
2100        images_dir = os.path.join("%s" % i_home,'passports')
2101        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
2102        stool = getToolByName(self, 'portal_schemas')
2103        schemas = ['student_application',
2104                   #'student_clearance',
2105                   ]
2106        created = False
2107        for schema_id in schemas:
2108            schema = stool._getOb(schema_id)
2109            object = getattr(student_folder,schema_id[len('student_'):],None)
2110            if object is None:
2111                continue
2112            doc = object.getContent()
2113            for key in schema.keys():
2114                if schema[key].meta_type != "CPS Image Field":
2115                    continue
2116                #import pdb;pdb.set_trace()
2117                image = getattr(doc,key)
2118                if not hasattr(image,"data"):
2119                    continue
2120                if not created:
2121                    if not os.path.exists(images_dir):
2122                        os.mkdir(images_dir)
2123                    created = True
2124                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
2125                open(filename,"wb").write(str(image.data))
2126    ###)
2127
2128InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.