source: WAeUP_SRP/base/WAeUPTool.py @ 3182

Last change on this file since 3182 was 3181, checked in by Henrik Bettermann, 17 years ago

logging mesaages reworked

Examples:

2008-02-18T21:24:40 INFO WAeUPTool.importStudents 100.000 % done in 91.89s, record added to students.pending, entry_mode: ug_pt, firstname: ABAYOME, matric_no: ART0310059, middlename: OMON, lastname: ANAVHE, reg_transition: return, sex: True, jamb_state: EDO, study_course: BISD, jamb_reg_no: BISD/008822, current_session: 05, current_mode: ug_pt, jamb_lga: ETSAKO-CENTRAL, ignore: 2005/2006, current_level: 100, current_verdict: B

2008-02-18T21:25:18 INFO WAeUPTool.importStudents finished importing in create mode from test.csv in 129.71 seconds, 99 records totally read, 31 added to students.imported, 68 added to students.pending, 0 already in students.pending
  • Property svn:keywords set to Id
File size: 79.2 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3181 2008-02-18 20:28:57Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60
61def getImagesDir(student_id):
62    return os.path.join("%s" % images_base,student_id[0],student_id)
63
64def getObject(object,name):
65    if object.hasObject(name):
66        return getattr(object,name)
67    return None
68
69class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
70    """WAeUP tool"""
71
72    id = 'waeup_tool'
73    meta_type = 'WAeUP Tool'
74    _actions = ()
75    security = ClassSecurityInfo()
76    security.declareObjectProtected(View)
77    manage_options = ( ActionProviderBase.manage_options
78                     + SimpleItem.manage_options
79                     )
80
81    security.declareProtected(View,'re_split') ###(
82    def re_split(self,split_string,string):
83        return re.split(split_string,string)
84    ###)
85
86    security.declareProtected(View,'difference') ###(
87    def difference(self,l1,l2):
88        return set(l1).difference(set(l2))
89    ###)
90
91    def rwrite(self,s): ###(
92        response = self.REQUEST.RESPONSE
93        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
94        response.write("%s<br />\r\n" % s)
95    ###)
96
97    def addtodict(self,d,key,item): ###(
98        d[key].append(item)
99        return d[key]
100    ###)
101
102    def sleep(self,secs): ###(
103        "sleep"
104        import time
105        time.sleep(secs)
106        return
107
108###)
109
110    security.declareProtected(View,'updateRoleMappingsFor') ###(
111    def updateRoleMappingsFor(self,wf_definition,ob):
112        "do so for public"
113        wf_def = getattr(self.portal_workflow,wf_definition)
114        wf_def.updateRoleMappingsFor(ob)
115    ###)
116
117    security.declareProtected(View,'getStatesLgas') ###(
118    def getStatesLgas(self):
119        """return lga info"""
120        voc = getattr(self.portal_vocabularies,'local_gov_areas')
121        states = []
122        lgas  = []
123        d = {}
124        wd = {}
125        for k,v in voc.items():
126            parts = v.split(' / ')
127            if len(parts) == 1:
128                state = parts[0].lower()
129                lga = ""
130            elif len(parts) == 2:
131                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133            else:
134                continue
135            if state not in states:
136                states.append(state)
137            if lga not in lgas:
138                lgas.append(lga)
139            words = re.split('[^a-zA-Z0-9/]',k)
140            words.sort()
141            wd[k] = words
142            d[k] = v
143        mapping = {}
144        mapping['word_dict'] = wd
145        mapping['lga_dict'] = d
146        mapping['states'] = states
147        mapping['lgas'] = lgas
148        return mapping
149    ###)
150
151    security.declareProtected(View,'findLga') ###(
152    def findLga(self,words,words_dict):
153        words = re.split('[^a-zA-Z0-9/]',words)
154        lga_words = []
155        for word in words:
156            if word:
157                lga_words += word.strip().lower(),
158        lga_words.sort()
159        state_lga = ''
160        while not state_lga:
161            for k,l in words_dict.items():
162                if lga_words == l:
163                    state_lga = k
164                    break
165            break
166        return state_lga
167    ###)
168    security.declareProtected(View,'getAccessInfo') ###(
169    def getAccessInfo(self,context):
170        "return a dict with access_info"
171        logger = logging.getLogger('WAeUPTool.getAccessInfo')
172        mtool = self.portal_membership
173        member = mtool.getAuthenticatedMember()
174        member_id = str(member)
175        info = {}
176        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178        is_staff = info['is_staff'] = not is_anonymous and not is_student
179        roles = member.getRolesInContext(context)
180        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181                                                                            "SectionManager" in roles or
182                                                                            "Manager" in roles)
183        is_allowed = info['is_allowed'] = not is_anonymous
184        requested_id = context.getStudentId()
185        student_id  = None
186        if is_allowed:
187            if not is_student and requested_id:
188                student_id  = requested_id
189            elif is_anonymous or member_id != requested_id:
190                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
191            else:
192                student_id = member_id
193        info['student_id'] = student_id
194        return info
195    ###)
196
197    security.declareProtected(ModifyPortalContent,'openLog') ###(
198    def openLog(self,name):
199        """open a log file"""
200        version = 1
201        path = "%s/log/%s_%d.log" % (i_home,name,version)
202        while os.path.exists(path):
203            version += 1
204            path = "%s/log/%s_%d.log" % (i_home,name,version)
205        log = open(path,"w")
206        return log
207    ###)
208
209    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
210    def bypassQueueCatalog(self,enable=True):
211        """bypass the QueueCatalog by setting all indexes to process imediate,
212        if enable is True (default) the old settings are restored
213        """
214
215    ###)
216
217    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
218    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
219        """measure Object access Time"""
220        import random
221        if hasattr(self,'portal_catalog_real'):
222            aq_portal = self.portal_catalog_real.evalAdvancedQuery
223        else:
224            aq_portal = self.portal_catalog.evalAdvancedQuery
225        nr_pts = int(nr_pts)
226        probe = int(probe)
227        intervall = probe/10
228        objects = ("application","clearance","personal")
229        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
230        #i = random.randrange(num_objects)
231        count = 0
232        found = 0
233        not_found = 0
234        t_found = 0
235        t_not_found = 0
236        time_found = time_not_found = 0.0
237        t_time_found = t_time_not_found = 0.0
238        accessed = []
239        t_min = 1000
240        t_max = 0
241        #import pdb;pdb.set_trace()
242        students = self.portal_catalog(portal_type="Student")
243        num_students = len(students)
244        if method == "d":
245            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
246            res = aq_portal(query)
247            brains = {}
248            for r in res:
249                sid = r.relative_path.split('/')[-2]
250                if brains.has_key(sid):
251                    brains[sid][r.portal_type] = r
252                else:
253                    brains[sid] = {r.portal_type : r}
254            brains_list = brains.keys()
255            num_objects = len(brains_list)
256        else:
257            num_objects = num_students
258        print "="*40
259        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
260                                                                        probe,
261                                                                        nr_pts,
262                                                                        num_objects)
263        print "nr found/not time found/not min/max"
264        elapse = time.time()
265        i_elapse = time.time()
266        c_elapse = time.clock()
267        for c in range(1,probe + 1):
268            i = random.randrange(num_objects)
269            if method in ('a','b','c'):
270                student_brain = students[i]
271            elif method == "d":
272                #import pdb;pdb.set_trace()
273                student_brain = brains[brains_list[i]]
274            if method == "c":
275                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
276                res = aq_portal(query)
277                this_portal_types = [r.portal_type for r in res]
278            for i in range(nr_pts):
279                oid = objects[i]
280                if method == "a":
281                    try:
282                        student_path = student_brain.getPath()
283                        path = "%s/%s" % (student_path,oid)
284                        doc = self.unrestrictedTraverse(path).getContent()
285                        found += 1
286                        i_time = time.time() - i_elapse
287                        time_found += i_time
288                    except:
289                        not_found += 1
290                        i_time = time.time() - i_elapse
291                        time_not_found += i_time
292                        pass
293                elif method == "b":
294                    try:
295                        student_object = student_brain.getObject()
296                        doc = getattr(student_object,oid).getContent()
297                        found += 1
298                        i_time = time.time() - i_elapse
299                        time_found += i_time
300                    except:
301                        i_time = time.time() - i_elapse
302                        time_not_found += i_time
303                        not_found += 1
304                        pass
305                elif method == "c":
306                    if portal_types[i] in this_portal_types:
307                        found += 1
308                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
309                        i_time = time.time() - i_elapse
310                        time_found += i_time
311                    else:
312                        not_found += 1
313                        i_time = time.time() - i_elapse
314                        time_not_found += i_time
315                elif method == "d":
316                    if student_brain.has_key(portal_types[i]):
317                        found += 1
318                        doc = student_brain[portal_types[i]].getObject().getContent()
319                        i_time = time.time() - i_elapse
320                        time_found += i_time
321                    else:
322                        not_found += 1
323                        i_time = time.time() - i_elapse
324                        time_not_found += i_time
325                i_elapse = time.time()
326            if c and (c % intervall == 0):
327                #i_time = time.time() - i_elapse
328                t_per = 0.0
329                if found:
330                    t_per = time_found/found
331                if t_per > t_max:
332                    t_max = t_per
333                if t_per > 0.0 and t_per < t_min:
334                    t_min = t_per
335                itf = 0.0
336                if found:
337                    itf = time_found/found
338                itnf = 0.0
339                if not_found :
340                    itnf = time_not_found / not_found
341                interval_time = time_found + time_not_found
342                s = "%(c)d: %(found)d/%(not_found)d " % vars()
343                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
344                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
345                print s
346                t_found += found
347                t_not_found += not_found
348                t_time_found += time_found
349                t_time_not_found += time_not_found
350                time_found = time_not_found = 0.0
351                found = not_found = 0
352        # t_found += found
353        # t_not_found += not_found
354        elapse = time.time() - elapse
355        itf = 0.0
356        if t_found:
357            itf = t_time_found/t_found
358        itnf = 0.0
359        if t_not_found:
360            itnf = t_time_not_found / t_not_found
361        #c_elapse = time.clock() - c_elapse
362        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
363        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
364        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
365        print "-"*40
366        print s
367        rel_found = float(t_found)/probe
368        rel_not_found = float(t_not_found)/probe
369        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
370        print estimated_total_time
371    ###)
372
373    security.declareProtected(ModifyPortalContent,'writeLog') ###(
374    def writeLog(self,logfile,s):
375        """write to the log file"""
376        logfile.write(s)
377
378###)
379
380    def generateStudentId(self,letter): ###(
381        import random
382        r = random
383        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
384        if letter == '?':
385            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
386        sid = "%c%d" % (letter,r.randint(99999,1000000))
387        students = self.portal_url.getPortalObject().campus.students
388##        while hasattr(students, sid):
389##            sid = "%c%d" % (letter,r.randint(99999,1000000))
390        while self.students_catalog(id = sid):
391            sid = "%c%d" % (letter,r.randint(99999,1000000))
392        return sid
393    ###)
394
395    def generatePassword(self,s=None): ###(
396        import random
397        r = random
398        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
399        if s is None:
400            s = 'abcdefghklmnpqrstuvwxy23456789'
401        pw = ''
402        while len(pw) < 6:
403            pw += r.choice(s)
404        return pw
405    ###)
406
407    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
408    def dumpSchoolfeePayments(self):
409        "dump paid schoolfees"
410        mtool = self.portal_membership
411        member = mtool.getAuthenticatedMember()
412        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
413        aq_student = self.students_catalog.evalAdvancedQuery
414        query = In('review_state',('schoolfee_paid',
415                                   'courses_registered',
416                                   'courses_validated',
417                                   ))
418        res = aq_student(query)
419        #import pdb;pdb.set_trace()
420        l = []
421        logger.info("start for %d" % len(res))
422        count = 1
423        log_after = 100
424        for student in res:
425            if not count % log_after:
426                logger.info("processed %d total %d" % (log_after,count))
427            count += 1
428            fee_dict =self.getSchoolFee(student)
429            fulltime = student.mode.endswith('_ft')
430            d = {}
431            d['student_id'] = student.id
432            d['name'] = student.name
433            d['amount'] = fee_dict.get(new_returning)
434            l += d,
435        csv_name = self.dumpListToCSV(l,'payments')
436        logger.info('%s dumped payments to %s' % (member,export_file))
437    ###)
438
439    security.declarePublic('dumpListToCSV') ###(
440    def dumpListToCSV(self,l,filename,fields=None):
441        """dump a list of dicts to a CSV file"""
442        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
443        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
444        if fields is None:
445            fields = l[0].keys()
446        headline = ','.join(fields)
447        out = open(export_file,"wb")
448        out.write(headline +'\n')
449        out.close()
450        out = open(export_file,"a")
451        csv_writer = csv.DictWriter(out,fields,)
452        csv_writer.writerows(l)
453        return export_file
454    ###)
455
456    security.declareProtected(ManagePortal, 'listMembers') ###(
457    def listMembers(self):
458        "list all members"
459        mtool = self.portal_membership
460        member = mtool.getAuthenticatedMember()
461        logger = logging.getLogger('WAeUPTool.listMembers')
462        if str(member) not in ('admin','joachim'):
463            logger.info('%s tried to list members' % (member))
464            return None
465        members = self.portal_directories.members
466        all = members.listEntryIdsAndTitles()
467        l = []
468        for user_id,name in all:
469            d = {}
470            d['user_id'] = user_id
471            d['name'] = name
472            d['pw'] = getattr(getattr(members,user_id),'password')
473            d['email'] = getattr(getattr(members,user_id),'email')
474            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
475            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
476            l += d,
477        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
478        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
479        logger.info('%s dumped memberlist to %s' % (member,export_file))
480        fields = l[0].keys()
481        headline = ','.join(fields)
482        out = open(export_file,"wb")
483        out.write(headline +'\n')
484        out.close()
485        out = open(export_file,"a")
486        csv_writer = csv.DictWriter(out,fields,)
487        csv_writer.writerows(l)
488    ###)
489
490    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
491    def removeDeletedDocIds(self, max=1000):
492        """
493        remove deleted docids from repository commit after max
494        """
495        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
496        repository = getToolByName(self, 'portal_repository')
497        pxtool = getToolByName(self, 'portal_proxies')
498        logger.info('start')
499        pxtool_infos = pxtool.getRevisionsUsed()
500        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
501
502        nb_revs = 0
503        docids_d = {} # all docids
504        unused_docids_d = {} # all docids that are unused
505        ids_unused_revs_docids = [] # ids for revs of unused docids
506        unused_ids = [] # ids for unused revs
507        total = 0
508        idlist = repository.objectIds()
509        to_delete = 0
510        found = False
511        for id in idlist:
512            docid, rev = repository._splitId(id)
513            if docid is None:
514                logger.info("invalid doc_id %s" % docid)
515                continue
516            nb_revs += 1
517            if not pxtool_infos.has_key(docid):
518                found = True
519                to_delete += 1
520                unused_ids.append(id)
521            elif not pxtool_infos[docid].has_key(rev):
522                found = True
523                to_delete += 1
524                unused_ids.append(id)
525            if found and not to_delete % max:
526                found = False
527                #import pdb;pdb.set_trace()
528                repository.manage_delObjects(unused_ids)
529                transaction.commit()
530                logger.info('removed %d total %d unused docids ' % (max,to_delete))
531        else:
532            if unused_ids:
533                repository.manage_delObjects(unused_ids)
534                transaction.commit()
535        logger.info('finished removing %d unused docids ' % (to_delete))
536
537
538###)
539
540    security.declareProtected(View,'getCredential') ###(
541    def getCredential(self,student_id):
542        student_entry = getattr(self.portal_directories.students,student_id,None)
543        if not self.isStaff():
544            mtool = self.portal_membership
545            member = mtool.getAuthenticatedMember()
546            logger = logging.getLogger('WAeUPTool.getCredential')
547            logger.info('%s tried to access password of %s' % (member,student_id))
548            return None
549        if student_entry is None:
550            return None
551        return getattr(student_entry,"password","not set")
552    ###)
553
554    security.declarePublic('checkPassword') ###(
555    def checkPassword(self,student_id,password):
556        student_entry = getattr(self.portal_directories.students,student_id,None)
557        if student_entry is None:
558            return False
559        return getattr(student_entry,"password","not set") == password
560    ###)
561
562    security.declarePublic('checkGenericPassword') ###(
563    def checkGenericPassword(self,member_id):
564        member_entry = getattr(self.portal_directories.members,member_id,None)
565        if member_entry is None:
566            return False
567        ltool = getToolByName(self, 'portal_layouts')
568        unsecure_words = ltool._getOb('members')['w__password'].check_words
569        password = getattr(member_entry,"password","not set")
570        is_unsecure = password in unsecure_words
571        if is_unsecure:
572            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
573            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
574        return is_unsecure
575    ###)
576
577    security.declareProtected(ModifyPortalContent,'editPassword') ###(
578    def editPassword(self,student_id,password):
579        "edit a student password"
580        student_entry = getattr(self.portal_directories.students,student_id,None)
581        if student_entry is None:
582            return
583        setattr(student_entry,'password',password)
584    ###)
585
586    security.declareProtected(ModifyPortalContent,'doCommit') ###(
587    def doCommit(self,logger=None):
588        "commit some transactions"
589        transaction.commit()
590    ###)
591
592    security.declarePublic('loadStudentFoto') ###(
593    def loadStudentFoto(self,student,filename,folder):
594        "return a student passport picture"
595        #import pdb;pdb.set_trace()
596        picture ="%s/import/%s/%s" % (i_home,folder,filename)
597        student_id = student.getId()
598        images_dir = getImagesDir(student_id)
599        if not os.path.exists(images_dir):
600            os.mkdir(images_dir)
601        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
602        for extension in ('.jpg','.JPG'):
603            fullname = "%(picture)s%(extension)s" % vars()
604            if os.path.exists(fullname):
605                copy2(fullname,image_name)
606                return "successfully copied passport picture"
607        return "passport picture not found: %s.jpg or .JPG" % picture
608    ###)
609
610    def old____loadStudentFoto(self,student,filename,folder): ###(
611        "return a student passport picture"
612        app = student.application
613        app_doc = app.getContent()
614        #clear = student.clearance
615        #clear_doc = clear.getContent()
616        #matric_no = clear_doc.matric_no.upper()
617        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
618        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
619        #import pdb;pdb.set_trace()
620        if os.path.exists(picture1):
621            file = open(picture1)
622        elif os.path.exists(picture2):
623            file = open(picture2)
624        else:
625            return "passport picture not found %s" % picture1
626        reopened = False
627        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
628            self.portal_workflow.doActionFor(app,'open')
629            reopened = True
630        outfile = file.read()
631        app_doc.manage_addFile('passport',
632                               file=outfile,
633                               title="%s.jpg" % filename)
634        if reopened:
635            self.portal_workflow.doActionFor(app,'close')
636        return "successfully loaded passport picture"
637    ###)
638
639    security.declareProtected(ModifyPortalContent,'createOne') ###(
640    def createOne(self,students_folder,student_brain,letter,commit=False):
641        sid = self.waeup_tool.generateStudentId(letter)
642        students_folder.invokeFactory('Student', sid)
643        student = getattr(students_folder,sid)
644        self.portal_workflow.doActionFor(student,'return')
645        student.manage_setLocalRoles(sid, ['Owner',])
646        matric_no = student_brain.matric_no
647        jamb_reg_no = student_brain.Entryregno
648        self.students_catalog.addRecord(id = sid,
649                                           matric_no = matric_no,
650                                           jamb_reg_no = jamb_reg_no,
651                                           sex = student_brain.Sex == "F",
652                                           name = "%s %s %s" % (student_brain.Firstname,
653                                                                student_brain.Middlename,
654                                                                student_brain.Lastname)
655                                        )
656        if commit:
657            transaction.commit()
658        return sid,jamb_reg_no
659    ###)
660
661    security.declareProtected(ModifyPortalContent,'addStudent') ###(
662    def addStudent(self,dict):
663        students_folder = self.portal_url.getPortalObject().campus.students
664        sid = self.waeup_tool.generateStudentId('?')
665        students_folder.invokeFactory('Student', sid)
666        student_obj = getattr(students_folder,sid)
667        f2t = self.field2types_student
668        #from pdb import set_trace; set_trace()
669        d = {}
670        #d['jamb_sex']  = 'M'
671        #if dict.get('sex'):
672        #    d['jamb_sex']  = 'F'
673
674        entry_session = dict.get('entry_session')
675        if entry_session == self.getSessionId()[0]:
676            wfaction = 'admit'
677            wft = 'wf_transition_admit'
678            password = None
679        else:
680            wfaction = 'return'
681            wft = 'wf_transition_return'
682            password = self.generatePassword()
683            self.makeStudentMember(sid,password)
684
685        for pt in f2t.keys():
686            student_obj.invokeFactory(pt,f2t[pt]['id'])
687            sub_obj = getattr(student_obj,f2t[pt]['id'])
688            sub_doc = sub_obj.getContent()
689            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
690            d['Title'] = f2t[pt]['title']
691            for field in f2t[pt]['fields']:
692                d[field] = dict.get(field,'')
693            sub_doc.edit(mapping = d)
694            new_state = f2t[pt][wft]
695            if new_state != "remain":
696                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
697        self.portal_workflow.doActionFor(student_obj,wfaction)
698        student_obj.manage_setLocalRoles(sid, ['Owner',])
699        return sid,password
700    ###)
701
702    security.declarePublic('getCertificateBrain') ###(
703    def getCertificateBrain(self,cert_id):
704        "do it"
705        res = ZCatalog.searchResults(self.portal_catalog_real,
706                                {'portal_type':"Certificate",
707                                      'id': cert_id})
708        if res:
709            return res[0]
710        return None
711    ###)
712
713    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
714    def get_csv_filenames(self):
715        "do it"
716        files = [file for file in os.listdir("%s/import/" % (i_home))
717                 if file.endswith('.csv') and (file.find('imported') == -1 and
718                                               file.find('pending') == -1)]
719        return files
720    ###)
721
722    security.declarePublic('findStudentByMatricelNo') ###(
723    def findStudentByMatricelNo(self,matric_no):
724        "do it"
725        res = ZCatalog.searchResults(self.portal_catalog_real,
726                                {'portal_type':"StudentClearance",
727                                 'SearchableText': matric_no})
728        if res:
729            return res[0]
730        return None
731    ###)
732
733    security.declarePublic('makeStudentMember') ###(
734    def makeStudentMember(self,sid,password='uNsEt'):
735        """make the student a member"""
736        membership = self.portal_membership
737        membership.addMember(sid,
738                             password ,
739                             roles=('Member',
740                                     'Student',
741                                     ),
742                             domains='',
743                             properties = {'memberareaCreationFlag': False,
744                                           'homeless': True},)
745        member = membership.getMemberById(sid)
746        self.portal_registration.afterAdd(member, sid, password, None)
747        #self.manage_setLocalRoles(sid, ['Owner',])
748    ###)
749
750    security.declareProtected(View,'makeStudentData') ###(
751    def makeStudentData(self,student_id,email=None,phone_nr=None):
752        "create Datastructure for a returning Student"
753        #import pdb;pdb.set_trace()
754        logger = logging.getLogger('WAeUPTool.makeStudentData')
755        students_folder = self.portal_url.getPortalObject().campus.students
756        #res = self.students_catalog(id=student_id)
757        #if res:
758        #    st = res[0]
759        #res = self.returning_import(matric_no = st.matric_no)
760        res = self.returning_import(id = student_id)
761        if res:
762            student = res[0]
763        else:
764            logger.info('Id %s not found in returning_import' % student_id)
765            return
766        logger.info('%s creates data structure' % student_id)
767        s_results = self.results_import(matric_no = student.matric_no)
768        if s_results:
769            lnr = self.getLevelFromResultsCosCode(s_results)
770            level = "%d00" % lnr
771            verdict,eligible = self.getVerdict(s_results[0].Verdict)
772            #if eligible:
773            #    level = "%d00" % (lnr + 1)
774        else:
775            logger.info('matric_no %s not found in results_import' % student.matric_no)
776            level = ''
777            verdict = ''
778        #student should not be allowed to perform this transition
779        #wftool = self.portal_workflow
780        #wftool.doActionFor(student,'return')
781        certcode_org = student.Coursemajorcode
782        certcode = makeCertificateCode(certcode_org)
783        certificate_brain = self.getCertificateBrain(certcode)
784        if not certificate_brain:
785            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
786            logger.info(em)
787        matric_no = student.matric_no
788        sid = student_id
789        student_obj = getattr(students_folder,sid)
790        if not getattr(student_obj,'application'):
791            student_obj.invokeFactory('StudentApplication','application')
792        application = student_obj.application
793        self.portal_workflow.doActionFor(application,'open',dest_container=application)
794        da = {'Title': 'Application Data'}
795        student_obj.invokeFactory('StudentPersonal','personal')
796        da['jamb_reg_no'] = student.Entryregno
797        em = self.getEntryMode(student.Entryregno)
798        da['entry_mode'] = em
799        personal = student_obj.personal
800        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
801        dp = {'Title': 'Personal Data'}
802        student_obj.invokeFactory('StudentClearance','clearance')
803        clearance = student_obj.clearance
804        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
805        dc = {'Title': 'Clearance/Eligibility Record'}
806        dc['matric_no'] = matric_no
807        state = student.State
808        lga = student.LGA
809        if state and lga:
810            lga =  state + ' / ' + lga
811        else:
812            lga = "None"
813        da['jamb_lga'] = dc['lga'] = lga
814        da['app_email'] = dp['email'] = email
815        da['app_mobile'] = dp['phone'] = phone_nr
816        dp['firstname'] = student.Firstname
817        dp['middlename'] = student.Middlename
818        dp['lastname'] = student.Lastname
819        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
820        da['jamb_sex'] = student.Sex
821        dp['sex'] = student.Sex == 'F'
822        dp['perm_address'] = student.Permanent_Address
823        application.getContent().edit(mapping=da)
824        self.portal_workflow.doActionFor(application,'close',dest_container=application)
825        personal.getContent().edit(mapping=dp)
826        clearance.getContent().edit(mapping=dc)
827        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
828        #
829        # Study Course
830        #
831        student_obj.invokeFactory('StudentStudyCourse','study_course')
832        studycourse = student_obj.study_course
833        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
834        dsc = {}
835        dsc['study_course'] = certcode
836        dsc['current_level'] = level
837        dsc['current_verdict'] = verdict
838        dsc['current_mode'] = em
839        dsc['current_session'] = '05'
840        studycourse.getContent().edit(mapping=dsc)
841        #
842        # Level
843        #
844##        l = getattr(studycourse,level,None)
845##        if l is None:
846##            studycourse.invokeFactory('StudentStudyLevel', level)
847##            l = getattr(studycourse, level)
848##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
849##            l.getContent().edit(mapping={'Title': "Level %s" % level})
850###)
851
852    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
853    def admitOneStudent(self,brain,entry_session,pin_password):
854        "create Datastructure for an admitted Student"
855        #import pdb;pdb.set_trace()
856        if not hasattr(self,"_v_certificates"):
857            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
858        students_folder = self.portal_url.getPortalObject().campus.students
859        logger = logging.getLogger('WAeUPTool.admitOneStudent')
860        if brain.status != "admitted":
861            logger.info('status of %s is %s' % (brain.reg_no,brain.status))
862            return
863        pin_parts = brain.pin.split('-')
864        if pin_parts and len(pin_parts) != 3:
865            logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no))
866            return
867        if brain.course_admitted not in self._v_certificates:
868            logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no))
869            return
870        if brain.sex not in (True,False):
871            logger.info('sex of %s not available' % (brain.reg_no))
872            return
873        student_id = self.generateStudentId('?')
874        students_folder.invokeFactory('Student', student_id)
875        student_object = getattr(students_folder,student_id)
876        if pin_password:
877            password = pin_parts[2]
878            self.makeStudentMember(student_id,password = password)
879        student_object.manage_setLocalRoles(student_id, ['Owner',])
880        #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no))
881        #
882        # application
883        #
884        student_object.invokeFactory('StudentApplication','application')
885        application = student_object.application
886        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
887        da = {'Title': 'Application Data'}
888        da['jamb_reg_no'] = brain.reg_no
889
890        sex = 'M'
891        if brain.sex:
892            sex = 'F'
893        da['jamb_sex'] = sex
894        da['jamb_age'] = brain.jamb_age
895        #da['app_ac_pin'] = brain.pin
896        da['jamb_lga'] = brain.jamb_lga
897        da['jamb_state'] = brain.jamb_state
898        da['jamb_score'] = brain.aggregate
899        da['app_email'] = brain.email
900        da['app_mobile'] = brain.phone
901        if brain.entry_mode:
902            da['entry_mode'] = brain.entry_mode
903        elif brain.screening_type == 'pume':
904            da['entry_mode'] = 'ume_ft'
905        elif brain.screening_type == 'pde':
906            da['entry_mode'] = 'de_ft'
907        elif brain.screening_type == 'pce':
908            da['entry_mode'] = 'pce'
909        elif brain.screening_type == 'prence':
910            da['entry_mode'] = 'prence'
911        else:
912            da['entry_mode'] = ''
913        da['entry_session'] = entry_session
914        da['jamb_lastname'] = brain.lastname
915        da['jamb_middlename'] = brain.middlenames   # different field names!
916        da['jamb_firstname'] = brain.firstname
917        da['screening_application_date'] = brain.application_date
918        da['date_of_birth'] = brain.date_of_birth
919        da['jamb_first_cos'] = brain.course1
920        da['jamb_second_cos'] = brain.course2
921        da['course3'] = brain.course3
922        da['screening_type'] = brain.screening_type
923        da['screening_score'] = brain.screening_score
924        da['screening_date'] = brain.screening_date
925        da['hq_type'] = brain.hq_type
926        da['hq_grade'] = brain.hq_grade
927        da['aos'] = brain.aos
928
929        application.getContent().edit(mapping=da)
930        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
931        #
932        # personal
933        #
934        student_object.invokeFactory('StudentPersonal','personal')
935        personal = student_object.personal
936        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
937        dp = {'Title': 'Personal Data'}
938        dp['sex'] = brain.sex
939        dp['email'] = brain.email
940        dp['phone'] = brain.phone
941        dp['lastname'] = brain.lastname
942        dp['middlename'] = brain.middlenames   # different field names!
943        dp['firstname'] = brain.firstname
944        personal.getContent().edit(mapping=dp)
945        #
946        # clearance
947        #
948        student_object.invokeFactory('StudentClearance','clearance')
949        clearance = student_object.clearance
950        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
951        dc = {'Title': 'Clearance/Eligibility Record'}
952        dc['lga'] = brain.lga
953        dc['birthday'] = brain.date_of_birth
954        clearance.getContent().edit(mapping=dc)
955        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
956        #
957        # study Course
958        #
959        student_object.invokeFactory('StudentStudyCourse','study_course')
960        studycourse = student_object.study_course
961        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
962        dsc = {}
963        dsc['study_course'] = brain.course_admitted
964        dsc['current_verdict'] = ''
965        dsc['current_mode'] = da['entry_mode']
966        if da['entry_mode'].startswith('de'):
967            dsc['current_level'] = '200'
968        elif da['entry_mode'].startswith('pre'):
969            dsc['current_level'] = '000'
970        else:
971            dsc['current_level'] = '100'
972        dsc['current_session'] = entry_session
973        studycourse.getContent().edit(mapping=dsc)
974        #
975        # payments folder
976        student_object.invokeFactory('PaymentsFolder','payments')
977        payments = getattr(student_object,'payments')
978        dpay = {}
979        dpay['Title'] = 'Payments'
980        payments.getContent().edit(mapping=dpay)
981        self.portal_workflow.doActionFor(payments,'open')
982        #
983        # passport foto
984        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
985                                                              brain.screening_type,
986                                                              brain.reg_no)
987        images_dir = getImagesDir(student_id)
988        #images_dir = os.path.join("%s" % images_base,student_id)
989        letter_dir,student_dir = os.path.split(images_dir)
990        if not os.path.exists(letter_dir):
991            os.mkdir(letter_dir)
992        if not os.path.exists(images_dir):
993            os.mkdir(images_dir)
994        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
995        if os.path.exists(app_picture):
996            copy2(app_picture,image_name)
997        else:
998            logger.info('passport of %s/%s not found: %s' % (student_id,
999                                                             brain.reg_no,
1000                                                             app_picture))
1001
1002        return student_id
1003    ###)
1004
1005    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1006    def makeStudentLevel(self,student_id):
1007        "create the StudyLevel for a returning Student"
1008        #import pdb;pdb.set_trace()
1009        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1010        students_folder = self.portal_url.getPortalObject().campus.students
1011        res = self.students_catalog(id=student_id)
1012        if res:
1013            st = res[0]
1014        course = st.course
1015        matric_no = st.matric_no
1016        level = st.level
1017        res = self.results_import(matric_no = matric_no)
1018        if res:
1019            results = res
1020        logger.info('%s creating Level %s' % (student_id,level))
1021        #
1022        # Level
1023        #
1024        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1025        studycourse = getattr(student_obj,"study_course",None)
1026        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1027        l = getattr(studycourse,level,None)
1028        if l is None:
1029            studycourse.invokeFactory('StudentStudyLevel', level)
1030            l = getattr(studycourse, level)
1031            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1032            l.getContent().edit(mapping={'Title': "Level %s" % level})
1033        ###)
1034
1035    security.declarePublic('getHallInfo') ###(
1036    def getHallInfo(self,bed):
1037        """return Hall Info"""
1038        info = {}
1039        hall,block,room,letter = bed.split('_')
1040        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1041        if res and len(res) == 1:
1042            hall_brain = res[0]
1043            hall_doc = hall_brain.getObject().getContent()
1044        else:
1045            return info
1046        info['hall_title'] = hall_brain.Title
1047        info['maintenance_code'] = hall_doc.maintenance_code
1048        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1049        batch_doc = None
1050        for brain in res:
1051            if brain.id.startswith(info['maintenance_code']):
1052                batch_doc = brain.getObject().getContent()
1053                break
1054        if batch_doc is None:
1055            info['maintenance_fee'] = ''
1056        else:
1057            info['maintenance_fee'] = batch_doc.cost
1058        return info
1059    ###)
1060
1061    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1062    def removePictureFolder(self,student_id):
1063        """remove picture_folder by renaming it"""
1064        path = 'images'
1065        picture_path = os.path.join(i_home,path,student_id)
1066        if not os.path.exists(picture_path):
1067            return False
1068        os.rename(picture_path,picture_path + "_removed")
1069        return True
1070    ###)
1071
1072    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1073    def restorePictureFolder(self,student_id):
1074        """restore picture_folder by renaming it"""
1075        path = 'images'
1076        picture_path = os.path.join(i_home,path,student_id)
1077        if not os.path.exists(picture_path + "_removed"):
1078            return False
1079        os.rename(picture_path + "_removed",picture_path)
1080        return True
1081    ###)
1082
1083    security.declarePublic('picturesExist') ###(
1084    def picturesExist(self, ids,student_id=None):
1085        """check if pictures exist in the filesystem"""
1086        if student_id is None:
1087            student_id = self.getStudentId()
1088        if student_id is None:
1089            return False
1090        picture_path = getImagesDir(student_id)
1091        #picture_path = os.path.join(images_base,student_id)
1092        if not os.path.exists(picture_path):
1093            return False
1094        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1095        return set(ids).issubset(set(pictures))
1096    ###)
1097
1098    security.declarePublic('picturesList') ###(
1099    def picturesList(self):
1100        """check if pictures exist in the filesystem"""
1101        path = 'images'
1102        student_id = self.getStudentId()
1103        #picture_path = os.path.join(i_home,path,student_id)
1104        picture_path = getImagesDir(student_id)
1105        if not os.path.exists(picture_path):
1106            return []
1107        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1108    ###)
1109
1110    security.declarePublic('showFsPicture') ###(
1111    def showFsPicture(self,path):
1112        """return a picture from the filesystem"""
1113        #picture_path = os.path.join(i_home,path)
1114        picture_path = os.path.join(images_base,path)
1115        response = self.REQUEST.RESPONSE
1116        #import pdb;pdb.set_trace()
1117        registry = getToolByName(self, 'mimetypes_registry')
1118        mimetype = str(registry.lookupExtension(path.lower()) or
1119                    registry.lookupExtension('file.bin'))
1120        if os.path.exists(picture_path):
1121            response.setHeader('Content-type',mimetype)
1122            return open(picture_path).read()
1123        picture_path = os.path.join(i_home,'import',path)
1124        if os.path.exists(picture_path):
1125            return open(picture_path).read()
1126    ###)
1127
1128    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1129    def deleteAllCourses(self,department="All"):
1130        ''' delete the courses'''
1131        pm = self.portal_membership
1132        member = pm.getAuthenticatedMember()
1133
1134        if str(member) not in ("henrik","joachim"):
1135            return "not possible"
1136        if department == "All":
1137            res = self.portal_catalog({'meta_type': 'Department'})
1138        if len(res) < 1:
1139            return "No Departments found"
1140
1141        deleted = []
1142        for dep in res:
1143            cf = dep.getObject().courses
1144            if cf:
1145                cf.manage_delObjects(ids=cf.objectIds())
1146                deleted.append("deleted Courses in %s" % dep.getId)
1147        return "\r".join(deleted)
1148    ###)
1149
1150    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1151    def getLogfileLines(self,filename="event.log",numlines=20):
1152        """Get last NUMLINES lines of logfile FILENAME.
1153
1154        Return last lines' of a file in the instances logfile directory as
1155        a list. The number of returned lines equals `numlines' or less. If
1156        less than `numlines' lines are available, the whole file ist
1157        returned. If the file can not be opened or some other error
1158        occurs, empty list is returend.
1159        """
1160        result = []
1161        lines_hit = 0
1162
1163        # We only handle files in instances' log directory...
1164        logpath = os.path.join(i_home, "log")
1165        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1166        if not filename.startswith( logpath ):
1167            # Attempt to access file outside log-dir...
1168            return []
1169
1170        try:
1171            fd = file( filename, "rb" )
1172        except IOError:
1173            return []
1174        if not fd:
1175            return []
1176
1177        if os.linesep == None:
1178            linesep = '\n'
1179        else:
1180            linesep = os.linesep
1181
1182        # Try to find 'numlines' times a lineseparator, searching from end
1183        # and moving to the beginning of file...
1184        fd.seek( 0, 2) # Move to end of file...
1185        while lines_hit < numlines:
1186            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1187                                          # one step forward...
1188                lines_hit += 1
1189            try:
1190                fd.seek( -2, 1) # Go two bytes back from current pos...
1191            except IOError:
1192                # We cannot go back two bytes. Maybe the file is too small...
1193                break
1194        fd.seek(2,1)
1195
1196        # Read all lines from current position...
1197        result = fd.readlines()
1198        # Remove line endings...
1199        result = [x.strip() for x in result]
1200        fd.close()
1201        return result
1202    ###)
1203
1204    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1205    def getCallbacksFromLog(self,filename):
1206        """fix Online Payment Transactions from Z2.log entries"""
1207        import transaction
1208        import random
1209        from cgi import parse_qs
1210        from urlparse import urlparse
1211        #from pdb import set_trace
1212        wftool = self.portal_workflow
1213        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1214        students_folder = self.portal_url.getPortalObject().campus.students
1215        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1216        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1217        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1218        data = re.compile(s)
1219        start = True
1220        tr_count = 1
1221        total = 0
1222        #name = 'pume_results'
1223        #name = 'epaymentsuccessful_z2log2'
1224        name = filename
1225        no_import = []
1226        imported = []
1227        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1228        try:
1229            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1230        except:
1231            logger.error('Error reading %s' % name)
1232            return
1233        tas = []
1234        for line in transactions:
1235            dict = {}
1236            items = data.search(line)
1237            dict['idict'] = idict = items.groupdict()
1238            #print idict
1239            #from pdb import set_trace;set_trace()
1240            urlparsed = urlparse(idict['get'][4:])
1241            #print urlparsed
1242            path = urlparsed[2].split('/')
1243            dict['student_id'] = student_id = path[8]
1244            dict['payment_id'] = payment_id = path[10]
1245            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1246            tas.append(dict)
1247            tr_count += 1
1248        return tas
1249    ###)
1250
1251    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1252    def importOnlinePaymentTransactions(self):
1253        """load Online Payment Transactions from CSV values"""
1254        import transaction
1255        import random
1256        #from pdb import set_trace
1257        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1258        opt = self.online_payments_import
1259        students_folder = self.portal_url.getPortalObject().campus.students
1260        start = True
1261        tr_count = 1
1262        total = 0
1263        #name = 'pume_results'
1264        name = 'OnlineTransactions'
1265        no_import = []
1266        imported = []
1267        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1268        try:
1269            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1270        except:
1271            logger.error('Error reading %s.csv' % name)
1272            return
1273        for pay_transaction in transactions:
1274            if start:
1275                start = False
1276                logger.info('Start loading from %s.csv' % name)
1277                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1278                no_import.append('%s,"Error"' % s)
1279                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1280                format_error = format + ',"%(Error)s"'
1281            data = {}
1282
1283            # format of the first file sent by Tayo
1284            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1285            #data['student_id'] = student_id = pay_transaction['Payer ID']
1286            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1287            #data['response_code'] = response_code = pay_transaction['Resp Code']
1288            #data['amount'] = amount = pay_transaction['Amount']
1289
1290            # format of the second file sent by Tayo
1291            #data['datetime'] = date = 0
1292            #data['student_id'] = student_id = pay_transaction['Payer ID']
1293            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1294            #data['response_code'] = response_code = '00'
1295            #data['amount'] = amount = pay_transaction['Amount']
1296
1297            # format of the third file sent by Kehinde
1298            data['datetime'] = date = 0
1299            data['student_id'] = student_id = pay_transaction['customer_id']
1300            data['order_id'] = order_id = pay_transaction['merchant_reference']
1301            data['response_code'] = response_code = '00'
1302            data['amount'] = amount = pay_transaction['Amount']
1303
1304            dup = False
1305            if response_code == "12":
1306                continue
1307            try:
1308                opt.addRecord(**data)
1309            except ValueError:
1310                dup = True
1311            #from pdb import set_trace;set_trace()
1312            if dup:
1313                if response_code == "00":
1314                    try:
1315                        opt.modifyRecord(**data)
1316                    except:
1317                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1318                        continue
1319                else:
1320                    pay_transaction['Error'] = "Duplicate order_id"
1321                    no_import.append( format_error % pay_transaction)
1322                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1323                    continue
1324            tr_count += 1
1325            if tr_count > 1000:
1326                if len(no_import) > 0:
1327                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1328                             '\n'.join(no_import) + '\n')
1329                    no_import = []
1330                em = '%d transactions committed\n' % (tr_count)
1331                transaction.commit()
1332                regs = []
1333                logger.info(em)
1334                total += tr_count
1335                tr_count = 0
1336        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1337                                                '\n'.join(no_import))
1338        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1339    ###)
1340
1341    security.declareProtected(ModifyPortalContent,"importData")###(
1342    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1343        """load data from CSV values"""
1344        import transaction
1345        import random
1346        students_folder = self.portal_url.getPortalObject().campus.students
1347        pending_only = False
1348        pend_str = '--'
1349        elapse = time.time()
1350        #
1351        # preparations
1352        #
1353        if filename == pend_str:
1354            pending_only = True
1355        mode = "create"
1356        if edit:
1357            if filename.endswith('_toDelete'):
1358                mode = "delete"
1359            else:
1360                mode = "edit"
1361        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1362        importer = eval("%sImport" % importer_name)(mode,self)
1363        logger = importer.logger
1364        if importer.init_errors:
1365            logger.info(self.init_errors)
1366            return self.init_errors
1367        member = importer.member
1368        current = importer.current
1369        import_date = importer.import_date
1370        #
1371        # not_imported
1372        #
1373        info = importer.info
1374        data_keys = importer.data_keys
1375        csv_keys = importer.csv_keys
1376        #csv_keys.extend(info.keys())
1377        headline_mapping = dict((k,k) for k in csv_keys)
1378        #
1379        # pending
1380        #
1381        pending_path = importer.pending_path
1382        pending_tmp = importer.pending_tmp
1383        pending_backup = importer.pending_backup
1384        pending_fn = importer.pending_fn
1385        imported_path = importer.imported_path
1386        imported_fn = importer.imported_fn
1387        commit_after = importer.commit_after
1388        pending = []
1389        pending_digests = []
1390        #total_added_to_pending = 0
1391        if not pending_only:
1392            pending,pending_digests = importer.makeIdLists()
1393            pending_at_start = len(pending)
1394        datafile = open(pending_tmp,"w")
1395        pending_csv_writer = csv.DictWriter(datafile,
1396                                                    csv_keys,
1397                                                    extrasaction='ignore')
1398        pending_csv_writer.writerow(headline_mapping)
1399        datafile.close()
1400        #
1401        # imported
1402        #
1403        if not os.path.exists(imported_path):
1404            datafile = open(imported_path,"w")
1405            imported_csv_writer = csv.DictWriter(datafile,
1406                                                 csv_keys,
1407                                                 extrasaction='ignore')
1408            imported_csv_writer.writerow(headline_mapping)
1409            datafile.close()
1410        start = True
1411        tr_count = 0
1412        total = 0
1413        total_added_to_imported = 0
1414        total_pending = 0
1415        if pending_only:
1416            import_source_path = pending_path
1417        else:
1418            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1419        if not os.path.exists(import_source_path):
1420            em = 'No  %(import_source_fn)s' % vars()
1421            return em
1422        import_source_fn = os.path.split(import_source_path)[1]
1423        info['imported_from'] = import_source_fn
1424        attrs = csv.reader(open(import_source_path,"rb")).next()
1425        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
1426                                                        or k.strip() in info.keys())]
1427        diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1428        diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1429        if diff2schema and diff2schema != set(['id',]):
1430            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1431            return msg
1432        if mode in importer.required_modes:
1433            required_keys = [importer.layout.getIdUnprefixed(id)
1434                                for id,widget in importer.layout.objectItems()
1435                                if widget.is_required]
1436            if not set(required_keys).issubset(set(import_keys)):
1437                diff2import = set(required_keys).difference(set(import_keys))
1438                msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1439                return msg
1440        #
1441        # start importing
1442        #
1443        try:
1444            reader = csv.DictReader(open(import_source_path,"rb"))
1445        except:
1446            msg = 'Error reading %s.csv' % filename
1447            logger.error(msg)
1448            return msg
1449        items = [item for item in reader]
1450        total_to_import = len(items)
1451        tti_float = float(total_to_import)
1452        if pending_only:
1453            pending_at_start = total_to_import
1454        count = 0
1455        imported = []
1456        old_commit_count = 0
1457        with_error_count = imported_count = 0
1458        already_in = 0
1459        for record in items:
1460            item = {}
1461            for k,v in record.items():
1462                if v:
1463                    item[k.strip()] = v.strip()
1464            count += 1
1465            #item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1466            if start:
1467                start = False
1468                adapters = [MappingStorageAdapter(importer.schema, item)]
1469                logger.info('%(member)s starts import from %(import_source_fn)s in %(mode)s mode' % vars())
1470                #logger.info('%s starts import from %s in %s mode with schema and layout %s' % (member,filename,mode,iname))
1471            dm = DataModel(item, adapters,context=self)
1472            ds = DataStructure(data=item,datamodel=dm)
1473            error_string = ""
1474            total += 1
1475            error_count = 0
1476            for k in import_keys:
1477                if k not in item.keys() or k not in importer.validators.keys():
1478                    continue
1479                if not importer.validators[k](ds,mode=mode):
1480                    if error_count:
1481                        error_string += ' ++ '
1482                    error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1483                                                                           ds.getErrorMapping(k)))
1484                    error_count += 1
1485            if error_string:
1486                error = error_string
1487                id = ''
1488                mapping = item
1489            else:
1490                temp_item = item.copy()
1491                temp_item.update(dm)
1492                results = importer.import_method(temp_item)
1493                id = results[0]
1494                error = results[1]
1495                mapping = results[2]
1496            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1497            info['error'] = error
1498            mapping.update(info)
1499            log_list = []
1500            if error:
1501                digest = makeDigest(mapping,data_keys)
1502                with_error_count += 1
1503                #percent_finished = (with_error_count + imported_count)/tti_float*100
1504                if digest not in pending_digests:
1505                    pending_digests += digest,
1506                    pending.append(mapping)
1507                    if not pending_only:
1508                        log_list += "record added to %(pending_fn)s, %(data_string)s" % vars(),
1509                        #logger.info("%(percent_finished)6.3f %% done added to %(pending_fn)s %(data_string)s" % vars())
1510                else:
1511                    already_in += 1
1512                    #logger.info("already in %(pending_fn)s %(data_string)s" % vars())
1513                    pass
1514            else:
1515                imported_count += 1
1516                imported += mapping,
1517                log_list += "record imported and added to %(imported_fn)s, %(data_string)s" % vars(),
1518                #percent_finished = (with_error_count + imported_count)/tti_float*100
1519                #logger.info("%(percent_finished)6.3f %% done imported and added to  %(imported_fn)s %(data_string)s" % vars())
1520            if log_list:
1521                time_till_now = time.time() - elapse
1522                percent_finished = (with_error_count + imported_count)/tti_float*100
1523                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1524                logger.info(' '.join(log_list))
1525            finished = count > total_to_import - 1
1526            must_commit = False
1527            if len(imported) != old_commit_count:
1528                if not len(imported) % commit_after:
1529                    must_commit = True
1530                    old_commit_count = len(imported)
1531
1532            if must_commit or finished:
1533                if len(imported):
1534                    transaction.commit()
1535                    datafile = open(imported_path,"a")
1536                    writer = csv.DictWriter(datafile,
1537                                            csv_keys,
1538                                            extrasaction='ignore')
1539                    writer.writerows(imported)
1540                    datafile.close()
1541                    total_added_to_imported += len(imported)
1542                    imported = []
1543                if len(pending) > 0:
1544                    datafile = open(pending_tmp,"a")
1545                    writer = csv.DictWriter(datafile,
1546                                            csv_keys,
1547                                            extrasaction='ignore')
1548                    writer.writerows(pending)
1549                    datafile.close()
1550                    total_pending += len(pending)
1551                    #total_added_to_pending += len(pending)
1552                    pending = []
1553                if not finished:
1554                    msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
1555                    logger.info(msg)
1556        elapse = time.time() - elapse
1557        copy2(pending_tmp,pending_path)
1558        msg = "finished importing in %(mode)s mode from %(import_source_fn)s in %(elapse)6.2f seconds, " % vars()
1559        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1560        if pending_only:
1561            removed_pending = pending_at_start - total_pending
1562            msg += "%(removed_pending)d removed from %(pending_fn)s, " % vars()
1563        else:
1564            added_pending = total_pending - pending_at_start
1565            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1566        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1567        logger.info(msg)
1568        os.remove(pending_tmp)
1569        return msg
1570    ###)
1571
1572    def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###(
1573        """load data from CSV values"""
1574        import transaction
1575        import random
1576        member = self.portal_membership.getAuthenticatedMember()
1577        logger = logging.getLogger('WAeUPTool.importData')
1578        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1579        import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S")
1580        students_folder = self.portal_url.getPortalObject().campus.students
1581        start = True
1582        tr_count = 0
1583        total_imported = 0
1584        total_not_imported = 0
1585        total = 0
1586        pending_only = False
1587        pend_str = '_pending'
1588        if name.endswith(pend_str):
1589            pending_only = True
1590            name = name[:-len(pend_str)]
1591        iname = "import_%s" % name
1592        if name in ('application','course_result',):
1593            commit_after = 2000
1594        else:
1595            commit_after = 100
1596        stool = getToolByName(self, 'portal_schemas')
1597        ltool = getToolByName(self, 'portal_layouts')
1598        schema = stool._getOb(iname)
1599        if schema is None:
1600            em = 'No such schema %s' % iname
1601            logger.error('No such schema %s' % iname)
1602            return em
1603        layout = ltool._getOb(iname)
1604        if layout is None:
1605            em = 'No such layout %s' % iname
1606            logger.error(em)
1607            return em
1608        validators = {}
1609        for widget in layout.keys():
1610            validators[widget] = layout[widget].validate
1611        mode = "create"
1612        if edit:
1613            if filename.endswith('_toDelete'):
1614                mode = "delete"
1615            else:
1616                mode = "edit"
1617        importer_name = "mass_%(mode)s_%(name)s" % vars()
1618        importer = getattr(self, '%s' % importer_name,None)
1619        if importer is None:
1620            em = 'No importer function %s' % importer_name
1621            logger.error(em)
1622            return em
1623        pending_fn = "%s/import/%ss_pending.csv" % (i_home,name)
1624        pending_imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
1625        if pending_only:
1626            import_source_fn = pending_fn
1627            imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current)
1628            not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current)
1629            if not os.path.exists(pending_fn):
1630                em = 'No  %s' % os.path.split(pending_fn)
1631                return em
1632        else:
1633            import_source_fn = "%s/import/%s.csv" % (i_home,filename)
1634            imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
1635            not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
1636        if not os.path.exists(import_source_fn):
1637            em = 'No  %s' % os.path.split(import_soure_fn)
1638            return em
1639        attrs = csv.reader(open(import_source_fn,"rb")).next()
1640        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
1641                                                        or k.strip() == 'Error')]
1642        diff2schema = set(import_keys).difference(set(schema.keys()))
1643        diff2layout = set(import_keys).difference(set(layout.keys()))
1644        if diff2schema and diff2schema != set(['id',]):
1645            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1646            return msg
1647        if mode in ("create","edit"):
1648            required_keys = [layout.getIdUnprefixed(id)
1649                                for id,widget in layout.objectItems()
1650                                if widget.is_required]
1651            if not set(required_keys).issubset(set(import_keys)):
1652                diff2import = set(required_keys).difference(set(import_keys))
1653                msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
1654                return msg
1655        #
1656        # not_imported
1657        #
1658        not_imported_keys = import_keys + ["Error",]
1659        not_imported_file = open(not_imported_fn,"w")
1660        not_imported_csv_writer = csv.DictWriter(not_imported_file,
1661                                                    not_imported_keys,
1662                                                    extrasaction='ignore')
1663        mapping = dict((k,k) for k in not_imported_keys)
1664        not_imported_csv_writer.writerow(mapping)
1665        not_imported = []
1666        #
1667        # imported
1668        #
1669        imported_keys = import_keys[:]
1670        if 'id' not in imported_keys:
1671            imported_keys.insert(0,"id")
1672        imported_file = open(imported_fn,"w")
1673        imported_csv_writer = csv.DictWriter(imported_file,
1674                                             imported_keys,
1675                                             extrasaction='ignore')
1676        mapping = dict((k,k) for k in imported_keys)
1677        imported_csv_writer.writerow(mapping)
1678        imported = []
1679        #
1680        # pending
1681        #
1682        pending_records = {}
1683        imports_pending = 0
1684        total_pending_imported = 0
1685        total_pending = 0
1686        format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1687        format_error = format + ',"%(Error)s"'
1688        #format = '"%(id)s",'+ format
1689        imported = []
1690        if name in ('verdict','course_result',):
1691            #pending_keys = imported_keys[:]
1692            sname = "%s_pending" % iname
1693            pending_schema = stool._getOb(sname)
1694            if schema is None:
1695                em = 'No such pending_schema %s' % sname
1696                logger.error('No such schema %s' % sname)
1697                return em
1698            pending_keys = pending_schema.keys()
1699            pending_keys += "Error",
1700            if 'id' not in pending_keys:
1701                pending_keys.insert(0,'id')
1702            pending_records = {}
1703            if os.path.exists(pending_fn):
1704                pending_imports_file = open(pending_fn)
1705                pending_imports = csv.DictReader(pending_imports_file,
1706                                                 fieldnames=pending_keys)
1707                if pending_imports:
1708                    pending_records = dict((row['id'],row) for row in pending_imports
1709                                          if row['id'] != 'id')
1710                pending_imports_file.close()
1711            new_imports_pending = 0
1712            pending_imported = []
1713            if pending_only:
1714                pending_imported_ids = []
1715                pending_imported = []
1716                for record in pending_records.values():
1717                    item = {}
1718                    for k,v in record.items():
1719                        if v:
1720                            item[k] = v
1721                    results = importer(item)
1722                    id = results[0]
1723                    error = results[1]
1724                    is_pending = len(results) == 3
1725                    if is_pending:
1726                        continue
1727                    del item['Error']
1728                    msg = ";".join(["%s : %s" % (k,v) for k,v in item.items()])
1729                    logger.info("imported from %(pending_fn)s %(msg)s" % vars())
1730                    pending_imported.append(item)
1731                    pending_imported_ids += id,
1732                    total_pending_imported = len(pending_imported_ids)
1733                    logger.info("imported %d pending imports" % len(pending_imported_ids))
1734                for id in pending_imported_ids:
1735                    if id:
1736                        del pending_records[id]
1737                    else:
1738                        logger.info("tried to delete record with empty id")
1739            pending_imports_file = open(pending_fn,'w')
1740            pending_csv_writer = csv.DictWriter(pending_imports_file,
1741                                                pending_keys,
1742                                                extrasaction='ignore')
1743            mapping = dict((k,k) for k in pending_keys)
1744            pending_csv_writer.writerow(mapping)
1745            if len(pending_imported) > 0:
1746                pending_imported_file = open(pending_imported_fn,'w')
1747                pending_imported_csv_writer = csv.DictWriter(pending_imported_file,
1748                                                             pending_keys,
1749                                                             extrasaction='ignore')
1750                pending_imported_csv_writer.writerow(dict((k,k) for k in imported_keys))
1751                pending_imported_csv_writer.writerows(pending_imported)
1752        #
1753        if pending_only:
1754            items = []
1755        else:
1756            try:
1757                items = csv.DictReader(open(import_source_fn,"rb"))
1758            except:
1759                msg = 'Error reading %s.csv' % filename
1760                logger.error(msg)
1761                return msg
1762            not_imported = []
1763        pending_keys = pending_records.keys()[:]
1764        for item in items:
1765            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
1766            if start:
1767                start = False
1768                adapters = [MappingStorageAdapter(schema, item)]
1769                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
1770            dm = DataModel(item, adapters,context=self)
1771            ds = DataStructure(data=item,datamodel=dm)
1772            error_string = ""
1773            total += 1
1774            error_count = 0
1775            for k in import_keys:
1776                if not validators[k](ds,mode=mode):
1777                    if error_count:
1778                        error_string += ' ++ '
1779                    error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1780                                                                           ds.getErrorMapping(k)))
1781                    error_count += 1
1782            if error_string:
1783                item['Error'] = error_string
1784                not_imported.append(item)
1785                total_not_imported += 1
1786                continue
1787            temp_item = item.copy()
1788            temp_item.update(dm)
1789            #id,error = importer(temp_item)
1790            results = importer(temp_item)
1791            id = results[0]
1792            error = results[1]
1793            is_pending = len(results) == 3
1794            if is_pending:
1795                temp_item = results[2]
1796                temp_item['Error'] = error
1797                msg = format_error % temp_item
1798                #if id not in pending_records.keys():
1799                if id not in pending_keys:
1800                    temp_item['id'] = id
1801                    pending_records[id] = temp_item
1802                    logger.info("%(id)s added to pending %(msg)s" % vars())
1803                    pending_keys += id,
1804                    new_imports_pending += 1
1805                else:
1806                    logger.info("%(id)s already in pending %(msg)s" % vars())
1807                continue
1808            elif error:
1809                item['Error'] = error
1810                not_imported.append(item)
1811                total_not_imported += 1
1812                continue
1813            item = temp_item
1814            item['id'] = id
1815            imported.append(item)
1816            tr_count += 1
1817            total_imported += 1
1818            msg = format % item
1819            logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
1820            if total and not total % commit_after:
1821                transaction.commit()
1822                if len(imported) > 0:
1823                    imported_csv_writer.writerows(imported)
1824                    imported = []
1825                if len(not_imported) > 0:
1826                    not_imported_csv_writer.writerows(not_imported)
1827                    not_imported = []
1828                if len(pending_records) > 0:
1829                    pending_csv_writer.writerows(pending_records.values())
1830                    total_pending += len(pending_records)
1831                    pending_records = {}
1832                msg = '%d transactions committed\n' % (tr_count)
1833                regs = []
1834                logger.info(msg)
1835                tr_count = 0
1836        if len(imported) > 0:
1837            imported_csv_writer.writerows(imported)
1838        if len(not_imported) > 0:
1839            not_imported_csv_writer.writerows(not_imported)
1840        if len(pending_records) > 0:
1841            pending_csv_writer.writerows(pending_records.values())
1842            total_pending += len(pending_records)
1843        import_source_fn = os.path.split(import_source_fn)[1]
1844        pending_fn = os.path.split(pending_fn)[1]
1845        msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
1846        msg += "%(total_imported)d imported, %(total_not_imported)d not imported, " % vars()
1847        if total_pending:
1848            if pending_only:
1849                msg += "%(new_imports_pending)d pending added, %(total_pending_imported)d pending imported " % vars()
1850            else:
1851                msg += "%(new_imports_pending)d pending added " % vars()
1852            msg += "(of total %(total)d), " % vars()
1853            msg += "%(total_pending)d total pending in %(pending_fn)s " % vars()
1854        logger.info(msg)
1855        return msg
1856    ###)
1857
1858    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1859    def moveImagesToFS(self,student_id="O738726"):
1860        "move the images to the filesystem"
1861        images_dir = getImagesDir(student_id)
1862        #images_dir = os.path.join("%s" % images_base,student_id)
1863        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1864        stool = getToolByName(self, 'portal_schemas')
1865        schemas = ['student_application',
1866                   'student_clearance',
1867                   ]
1868        created = False
1869        for schema_id in schemas:
1870            schema = stool._getOb(schema_id)
1871            object = getattr(student_folder,schema_id[len('student_'):],None)
1872            if object is None:
1873                continue
1874            doc = object.getContent()
1875            for key in schema.keys():
1876                if schema[key].meta_type != "CPS Image Field":
1877                    continue
1878                #import pdb;pdb.set_trace()
1879                image = getattr(doc,key,None)
1880                if not image or not hasattr(image,"data"):
1881                    continue
1882                if not created:
1883                    if not os.path.exists(images_dir):
1884                        os.mkdir(images_dir)
1885                    created = True
1886                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1887                open(filename,"wb").write(str(image.data))
1888    ###)
1889
1890    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1891    def movePassportToFS(self,student_id="O738726"):
1892        "move the passports to the filesystem"
1893        images_dir = os.path.join("%s" % i_home,'passports')
1894        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1895        stool = getToolByName(self, 'portal_schemas')
1896        schemas = ['student_application',
1897                   #'student_clearance',
1898                   ]
1899        created = False
1900        for schema_id in schemas:
1901            schema = stool._getOb(schema_id)
1902            object = getattr(student_folder,schema_id[len('student_'):],None)
1903            if object is None:
1904                continue
1905            doc = object.getContent()
1906            for key in schema.keys():
1907                if schema[key].meta_type != "CPS Image Field":
1908                    continue
1909                #import pdb;pdb.set_trace()
1910                image = getattr(doc,key)
1911                if not hasattr(image,"data"):
1912                    continue
1913                if not created:
1914                    if not os.path.exists(images_dir):
1915                        os.mkdir(images_dir)
1916                    created = True
1917                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1918                open(filename,"wb").write(str(image.data))
1919    ###)
1920
1921InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.