source: WAeUP_SRP/base/WAeUPTool.py @ 3349

Last change on this file since 3349 was 3349, checked in by joachim, 17 years ago

include timing functions,
to enable set with_timing to True

  • Property svn:keywords set to Id
File size: 72.6 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3349 2008-03-18 17:54:22Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108
109###)
110
111    security.declareProtected(View,'updateRoleMappingsFor') ###(
112    def updateRoleMappingsFor(self,wf_definition,ob):
113        "do so for public"
114        wf_def = getattr(self.portal_workflow,wf_definition)
115        wf_def.updateRoleMappingsFor(ob)
116    ###)
117
118    security.declareProtected(View,'getStatesLgas') ###(
119    def getStatesLgas(self):
120        """return lga info"""
121        voc = getattr(self.portal_vocabularies,'local_gov_areas')
122        states = []
123        lgas  = []
124        d = {}
125        wd = {}
126        for k,v in voc.items():
127            parts = v.split(' / ')
128            if len(parts) == 1:
129                state = parts[0].lower()
130                lga = ""
131            elif len(parts) == 2:
132                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134            else:
135                continue
136            if state not in states:
137                states.append(state)
138            if lga not in lgas:
139                lgas.append(lga)
140            words = re.split('[^a-zA-Z0-9/]',k)
141            words.sort()
142            wd[k] = words
143            d[k] = v
144        mapping = {}
145        mapping['word_dict'] = wd
146        mapping['lga_dict'] = d
147        mapping['states'] = states
148        mapping['lgas'] = lgas
149        return mapping
150    ###)
151
152    security.declareProtected(View,'findLga') ###(
153    def findLga(self,words,words_dict):
154        words = re.split('[^a-zA-Z0-9/]',words)
155        lga_words = []
156        for word in words:
157            if word:
158                lga_words += word.strip().lower(),
159        lga_words.sort()
160        state_lga = ''
161        while not state_lga:
162            for k,l in words_dict.items():
163                if lga_words == l:
164                    state_lga = k
165                    break
166            break
167        return state_lga
168    ###)
169    security.declareProtected(View,'getAccessInfo') ###(
170    def getAccessInfo(self,context):
171        "return a dict with access_info"
172        logger = logging.getLogger('WAeUPTool.getAccessInfo')
173        mtool = self.portal_membership
174        member = mtool.getAuthenticatedMember()
175        member_id = str(member)
176        info = {}
177        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179        is_staff = info['is_staff'] = not is_anonymous and not is_student
180        roles = member.getRolesInContext(context)
181        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182                                                                            "SectionManager" in roles or
183                                                                            "Manager" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif is_anonymous or member_id != requested_id:
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378
379###)
380
381    def generateStudentId(self,letter): ###(
382        import random
383        r = random
384        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385        if letter == '?':
386            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387        sid = "%c%d" % (letter,r.randint(99999,1000000))
388        students = self.portal_url.getPortalObject().campus.students
389##        while hasattr(students, sid):
390##            sid = "%c%d" % (letter,r.randint(99999,1000000))
391        while self.students_catalog(id = sid):
392            sid = "%c%d" % (letter,r.randint(99999,1000000))
393        return sid
394    ###)
395
396    def generatePassword(self,s=None): ###(
397        import random
398        r = random
399        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400        if s is None:
401            s = 'abcdefghklmnpqrstuvwxy23456789'
402        pw = ''
403        while len(pw) < 6:
404            pw += r.choice(s)
405        return pw
406    ###)
407
408    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409    def dumpSchoolfeePayments(self):
410        "dump paid schoolfees"
411        mtool = self.portal_membership
412        member = mtool.getAuthenticatedMember()
413        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414        aq_student = self.students_catalog.evalAdvancedQuery
415        query = In('review_state',('schoolfee_paid',
416                                   'courses_registered',
417                                   'courses_validated',
418                                   ))
419        res = aq_student(query)
420        #import pdb;pdb.set_trace()
421        l = []
422        logger.info("start for %d" % len(res))
423        count = 1
424        log_after = 100
425        for student in res:
426            if not count % log_after:
427                logger.info("processed %d total %d" % (log_after,count))
428            count += 1
429            fee_dict =self.getSchoolFee(student)
430            fulltime = student.mode.endswith('_ft')
431            d = {}
432            d['student_id'] = student.id
433            d['name'] = student.name
434            d['amount'] = fee_dict.get(new_returning)
435            l += d,
436        csv_name = self.dumpListToCSV(l,'payments')
437        logger.info('%s dumped payments to %s' % (member,export_file))
438    ###)
439
440    security.declarePublic('dumpListToCSV') ###(
441    def dumpListToCSV(self,l,filename,fields=None):
442        """dump a list of dicts to a CSV file"""
443        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445        if fields is None:
446            fields = l[0].keys()
447        headline = ','.join(fields)
448        out = open(export_file,"wb")
449        out.write(headline +'\n')
450        out.close()
451        out = open(export_file,"a")
452        csv_writer = csv.DictWriter(out,fields,)
453        csv_writer.writerows(l)
454        return export_file
455    ###)
456
457    security.declareProtected(ManagePortal, 'listMembers') ###(
458    def listMembers(self):
459        "list all members"
460        mtool = self.portal_membership
461        member = mtool.getAuthenticatedMember()
462        logger = logging.getLogger('WAeUPTool.listMembers')
463        if str(member) not in ('admin','joachim'):
464            logger.info('%s tried to list members' % (member))
465            return None
466        members = self.portal_directories.members
467        all = members.listEntryIdsAndTitles()
468        l = []
469        for user_id,name in all:
470            d = {}
471            d['user_id'] = user_id
472            d['name'] = name
473            d['pw'] = getattr(getattr(members,user_id),'password')
474            d['email'] = getattr(getattr(members,user_id),'email')
475            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477            l += d,
478        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480        logger.info('%s dumped member list to %s' % (member,export_file))
481        fields = l[0].keys()
482        headline = ','.join(fields)
483        out = open(export_file,"wb")
484        out.write(headline +'\n')
485        out.close()
486        out = open(export_file,"a")
487        csv_writer = csv.DictWriter(out,fields,)
488        csv_writer.writerows(l)
489    ###)
490
491    security.declareProtected(ManagePortal, 'listStudents') ###(
492    def listStudents(self):
493        "list all students"
494        mtool = self.portal_membership
495        member = mtool.getAuthenticatedMember()
496        logger = logging.getLogger('WAeUPTool.listStudents')
497        if str(member) not in ('admin','joachim'):
498            logger.info('%s tried to list students' % (member))
499            return None
500        students = self.portal_directories.students
501        all = students.listEntryIdsAndTitles()
502        l = []
503        for user_id,name in all:
504            d = {}
505            d['user_id'] = user_id
506            d['name'] = name
507            d['pw'] = getattr(getattr(students,user_id),'password')
508            d['email'] = getattr(getattr(students,user_id),'email')
509            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511            l += d,
512        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514        logger.info('%s dumped student list to %s' % (member,export_file))
515        fields = l[0].keys()
516        headline = ','.join(fields)
517        out = open(export_file,"wb")
518        out.write(headline +'\n')
519        out.close()
520        out = open(export_file,"a")
521        csv_writer = csv.DictWriter(out,fields,)
522        csv_writer.writerows(l)
523    ###)
524
525    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526    def removeDeletedDocIds(self, max=1000):
527        """
528        remove deleted docids from repository commit after max
529        """
530        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531        repository = getToolByName(self, 'portal_repository')
532        pxtool = getToolByName(self, 'portal_proxies')
533        logger.info('start')
534        pxtool_infos = pxtool.getRevisionsUsed()
535        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
536
537        nb_revs = 0
538        docids_d = {} # all docids
539        unused_docids_d = {} # all docids that are unused
540        ids_unused_revs_docids = [] # ids for revs of unused docids
541        unused_ids = [] # ids for unused revs
542        total = 0
543        idlist = repository.objectIds()
544        to_delete = 0
545        found = False
546        for id in idlist:
547            docid, rev = repository._splitId(id)
548            if docid is None:
549                logger.info("invalid doc_id %s" % docid)
550                continue
551            nb_revs += 1
552            if not pxtool_infos.has_key(docid):
553                found = True
554                to_delete += 1
555                unused_ids.append(id)
556            elif not pxtool_infos[docid].has_key(rev):
557                found = True
558                to_delete += 1
559                unused_ids.append(id)
560            if found and not to_delete % max:
561                found = False
562                #import pdb;pdb.set_trace()
563                repository.manage_delObjects(unused_ids)
564                transaction.commit()
565                logger.info('removed %d total %d unused docids ' % (max,to_delete))
566        else:
567            if unused_ids:
568                repository.manage_delObjects(unused_ids)
569                transaction.commit()
570        logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573###)
574
575    security.declareProtected(View,'getCredential') ###(
576    def getCredential(self,student_id):
577        student_entry = getattr(self.portal_directories.students,student_id,None)
578        if not self.isStaff():
579            mtool = self.portal_membership
580            member = mtool.getAuthenticatedMember()
581            logger = logging.getLogger('WAeUPTool.getCredential')
582            logger.info('%s tried to access password of %s' % (member,student_id))
583            return None
584        if student_entry is None:
585            return None
586        return getattr(student_entry,"password","not set")
587    ###)
588
589    security.declarePublic('checkPassword') ###(
590    def checkPassword(self,student_id,password):
591        student_entry = getattr(self.portal_directories.students,student_id,None)
592        if student_entry is None:
593            return False
594        return getattr(student_entry,"password","not set") == password
595    ###)
596
597    security.declarePublic('checkGenericPassword') ###(
598    def checkGenericPassword(self,member_id):
599        member_entry = getattr(self.portal_directories.members,member_id,None)
600        if member_entry is None:
601            return False
602        ltool = getToolByName(self, 'portal_layouts')
603        unsecure_words = ltool._getOb('members')['w__password'].check_words
604        password = getattr(member_entry,"password","not set")
605        is_unsecure = password in unsecure_words
606        if is_unsecure:
607            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609        return is_unsecure
610    ###)
611
612    security.declareProtected(ModifyPortalContent,'editPassword') ###(
613    def editPassword(self,student_id,password):
614        "edit a student password"
615        student_entry = getattr(self.portal_directories.students,student_id,None)
616        if student_entry is None:
617            return
618        setattr(student_entry,'password',password)
619    ###)
620
621    security.declareProtected(ModifyPortalContent,'doCommit') ###(
622    def doCommit(self,logger=None):
623        "commit some transactions"
624        transaction.commit()
625    ###)
626
627    security.declarePublic('loadStudentFoto') ###(
628    def loadStudentFoto(self,student,filename,folder):
629        "return a student passport picture"
630        #import pdb;pdb.set_trace()
631        picture ="%s/import/%s/%s" % (i_home,folder,filename)
632        student_id = student.getId()
633        images_dir = getImagesDir(student_id)
634        if not os.path.exists(images_dir):
635            os.mkdir(images_dir)
636        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637        for extension in ('.jpg','.JPG'):
638            fullname = "%(picture)s%(extension)s" % vars()
639            if os.path.exists(fullname):
640                copy2(fullname,image_name)
641                return "successfully copied passport picture"
642        return "passport picture not found: %s.jpg or .JPG" % picture
643    ###)
644
645    def old____loadStudentFoto(self,student,filename,folder): ###(
646        "return a student passport picture"
647        app = student.application
648        app_doc = app.getContent()
649        #clear = student.clearance
650        #clear_doc = clear.getContent()
651        #matric_no = clear_doc.matric_no.upper()
652        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654        #import pdb;pdb.set_trace()
655        if os.path.exists(picture1):
656            file = open(picture1)
657        elif os.path.exists(picture2):
658            file = open(picture2)
659        else:
660            return "passport picture not found %s" % picture1
661        reopened = False
662        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663            self.portal_workflow.doActionFor(app,'open')
664            reopened = True
665        outfile = file.read()
666        app_doc.manage_addFile('passport',
667                               file=outfile,
668                               title="%s.jpg" % filename)
669        if reopened:
670            self.portal_workflow.doActionFor(app,'close')
671        return "successfully loaded passport picture"
672    ###)
673
674    security.declareProtected(ModifyPortalContent,'createOne') ###(
675    def createOne(self,students_folder,student_brain,letter,commit=False):
676        sid = self.waeup_tool.generateStudentId(letter)
677        students_folder.invokeFactory('Student', sid)
678        student = getattr(students_folder,sid)
679        self.portal_workflow.doActionFor(student,'return')
680        student.manage_setLocalRoles(sid, ['Owner',])
681        matric_no = student_brain.matric_no
682        jamb_reg_no = student_brain.Entryregno
683        self.students_catalog.addRecord(id = sid,
684                                           matric_no = matric_no,
685                                           jamb_reg_no = jamb_reg_no,
686                                           sex = student_brain.Sex == "F",
687                                           name = "%s %s %s" % (student_brain.Firstname,
688                                                                student_brain.Middlename,
689                                                                student_brain.Lastname)
690                                        )
691        if commit:
692            transaction.commit()
693        return sid,jamb_reg_no
694    ###)
695
696    security.declareProtected(ModifyPortalContent,'addStudent') ###(
697    def addStudent(self,dict):
698        students_folder = self.portal_url.getPortalObject().campus.students
699        sid = self.waeup_tool.generateStudentId('?')
700        students_folder.invokeFactory('Student', sid)
701        student_obj = getattr(students_folder,sid)
702        f2t = self.field2types_student
703        #from pdb import set_trace; set_trace()
704        d = {}
705        #d['jamb_sex']  = 'M'
706        #if dict.get('sex'):
707        #    d['jamb_sex']  = 'F'
708
709        entry_session = dict.get('entry_session')
710        if entry_session == self.getSessionId()[0]:
711            wfaction = 'admit'
712            wft = 'wf_transition_admit'
713            password = None
714        else:
715            wfaction = 'return'
716            wft = 'wf_transition_return'
717            password = self.generatePassword()
718            self.makeStudentMember(sid,password)
719
720        for pt in f2t.keys():
721            student_obj.invokeFactory(pt,f2t[pt]['id'])
722            sub_obj = getattr(student_obj,f2t[pt]['id'])
723            sub_doc = sub_obj.getContent()
724            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725            d['Title'] = f2t[pt]['title']
726            for field in f2t[pt]['fields']:
727                d[field] = dict.get(field,'')
728            sub_doc.edit(mapping = d)
729            new_state = f2t[pt][wft]
730            if new_state != "remain":
731                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732        self.portal_workflow.doActionFor(student_obj,wfaction)
733        student_obj.manage_setLocalRoles(sid, ['Owner',])
734        return sid,password
735    ###)
736
737    security.declarePublic('getCertificateBrain') ###(
738    def getCertificateBrain(self,cert_id):
739        "do it"
740        res = ZCatalog.searchResults(self.portal_catalog_real,
741                                {'portal_type':"Certificate",
742                                      'id': cert_id})
743        if res:
744            return res[0]
745        return None
746    ###)
747
748    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749    def get_csv_filenames(self):
750        "do it"
751        files = [file for file in os.listdir("%s/import/" % (i_home))
752                 if file.endswith('.csv') and (file.find('imported') == -1 and
753                                               file.find('pending') == -1)]
754        return files
755    ###)
756
757    security.declarePublic('findStudentByMatricelNo') ###(
758    def findStudentByMatricelNo(self,matric_no):
759        "do it"
760        res = ZCatalog.searchResults(self.portal_catalog_real,
761                                {'portal_type':"StudentClearance",
762                                 'SearchableText': matric_no})
763        if res:
764            return res[0]
765        return None
766    ###)
767
768    security.declarePublic('makeStudentMember') ###(
769    def makeStudentMember(self,sid,password='uNsEt'):
770        """make the student a member"""
771        membership = self.portal_membership
772        membership.addMember(sid,
773                             password ,
774                             roles=('Member',
775                                     'Student',
776                                     ),
777                             domains='',
778                             properties = {'memberareaCreationFlag': False,
779                                           'homeless': True},)
780        member = membership.getMemberById(sid)
781        self.portal_registration.afterAdd(member, sid, password, None)
782        #self.manage_setLocalRoles(sid, ['Owner',])
783    ###)
784
785    security.declareProtected(View,'makeStudentData') ###(
786    def makeStudentData(self,student_id,email=None,phone_nr=None):
787        "create Datastructure for a returning Student"
788        #import pdb;pdb.set_trace()
789        logger = logging.getLogger('WAeUPTool.makeStudentData')
790        students_folder = self.portal_url.getPortalObject().campus.students
791        #res = self.students_catalog(id=student_id)
792        #if res:
793        #    st = res[0]
794        #res = self.returning_import(matric_no = st.matric_no)
795        res = self.returning_import(id = student_id)
796        if res:
797            student = res[0]
798        else:
799            logger.info('Id %s not found in returning_import' % student_id)
800            return
801        logger.info('%s creates data structure' % student_id)
802        s_results = self.results_import(matric_no = student.matric_no)
803        if s_results:
804            lnr = self.getLevelFromResultsCosCode(s_results)
805            level = "%d00" % lnr
806            verdict,eligible = self.getVerdict(s_results[0].Verdict)
807            #if eligible:
808            #    level = "%d00" % (lnr + 1)
809        else:
810            logger.info('matric_no %s not found in results_import' % student.matric_no)
811            level = ''
812            verdict = ''
813        #student should not be allowed to perform this transition
814        #wftool = self.portal_workflow
815        #wftool.doActionFor(student,'return')
816        certcode_org = student.Coursemajorcode
817        certcode = makeCertificateCode(certcode_org)
818        certificate_brain = self.getCertificateBrain(certcode)
819        if not certificate_brain:
820            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821            logger.info(em)
822        matric_no = student.matric_no
823        sid = student_id
824        student_obj = getattr(students_folder,sid)
825        if not getattr(student_obj,'application'):
826            student_obj.invokeFactory('StudentApplication','application')
827        application = student_obj.application
828        self.portal_workflow.doActionFor(application,'open',dest_container=application)
829        da = {'Title': 'Application Data'}
830        student_obj.invokeFactory('StudentPersonal','personal')
831        da['jamb_reg_no'] = student.Entryregno
832        em = self.getEntryMode(student.Entryregno)
833        da['entry_mode'] = em
834        personal = student_obj.personal
835        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836        dp = {'Title': 'Personal Data'}
837        student_obj.invokeFactory('StudentClearance','clearance')
838        clearance = student_obj.clearance
839        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840        dc = {'Title': 'Clearance/Eligibility Record'}
841        dc['matric_no'] = matric_no
842        state = student.State
843        lga = student.LGA
844        if state and lga:
845            lga =  state + ' / ' + lga
846        else:
847            lga = "None"
848        da['jamb_lga'] = dc['lga'] = lga
849        da['app_email'] = dp['email'] = email
850        da['app_mobile'] = dp['phone'] = phone_nr
851        dp['firstname'] = student.Firstname
852        dp['middlename'] = student.Middlename
853        dp['lastname'] = student.Lastname
854        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855        da['jamb_sex'] = student.Sex
856        dp['sex'] = student.Sex == 'F'
857        dp['perm_address'] = student.Permanent_Address
858        application.getContent().edit(mapping=da)
859        self.portal_workflow.doActionFor(application,'close',dest_container=application)
860        personal.getContent().edit(mapping=dp)
861        clearance.getContent().edit(mapping=dc)
862        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863        #
864        # Study Course
865        #
866        student_obj.invokeFactory('StudentStudyCourse','study_course')
867        studycourse = student_obj.study_course
868        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869        dsc = {}
870        dsc['study_course'] = certcode
871        dsc['current_level'] = level
872        dsc['current_verdict'] = verdict
873        dsc['current_mode'] = em
874        dsc['current_session'] = '05'
875        studycourse.getContent().edit(mapping=dsc)
876        #
877        # Level
878        #
879##        l = getattr(studycourse,level,None)
880##        if l is None:
881##            studycourse.invokeFactory('StudentStudyLevel', level)
882##            l = getattr(studycourse, level)
883##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
884##            l.getContent().edit(mapping={'Title': "Level %s" % level})
885###)
886
887    def init_timing(self): ###(
888        if self.with_timing:
889            if not hasattr(self,'_v_step_times'):
890                self._v_timer_count = 0
891                self._v_total = 0
892                self._v_step_times = {}
893                current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
894                self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
895            self.timer_step = 0
896            self.total_time = 0
897            self.elapse = time.time()
898            self.i_elapse = time.time()
899            self.c_elapse = time.clock()
900
901###)
902
903    def do_timing(self): ###(
904        if self.with_timing:
905            try:
906                raise 'dummy'
907            except:
908                frame = sys.exc_traceback.tb_frame.f_back
909                locals = frame.f_locals
910                globals = frame.f_globals
911                functionname = frame.f_code.co_name
912                filename = os.path.basename(frame.f_code.co_filename)
913                lineno = frame.f_lineno
914                mod_line = "%(functionname)s:%(lineno)s" % vars()
915            i_time = time.time() - self.i_elapse
916            td = {}
917            if self._v_step_times.has_key(mod_line):
918                a_time = self._v_step_times[mod_line]['a_time'] + i_time
919                td['a_time'] = a_time
920            else:
921                td['a_time'] = i_time
922            td['i_time'] = i_time
923            self._v_step_times[mod_line] = td
924            self.i_time = i_time
925            self.total_time += i_time
926            self.timer_step +=1
927            self.i_elapse = time.time()
928
929###)
930
931    security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
932    def print_timing(self):
933        if self.with_timing:
934            l = []
935            timer_count = self._v_timer_count + 1
936            mod_lines = self._v_step_times.keys()
937            mod_lines.sort(cmp,reverse=0)
938            for mod_line in mod_lines:
939                td = self._v_step_times[mod_line]
940                i_time = td['i_time']
941                a_time = td['a_time']/(self._v_timer_count + 1)
942                l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
943            total_time = self.total_time
944            total_avarage = self._v_total / timer_count
945            l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
946            print "\r\n".join(l)
947            out = open(self._v_timer_file,'a')
948            out.write("\r\n".join(l))
949            out.close()
950
951###)
952
953    security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
954    def get_timing_data(self):
955        if self.with_timing:
956            timer_count = self._v_timer_count + 1
957            results = {}
958            for k,d in self._v_step_times.items():
959                dd = {}
960                dd['a_time'] = d['a_time']/timer_count
961                dd['i_time'] = d['i_time']
962                dd['count'] = timer_count
963                results[k] = dd
964            dd = {}
965            dd['a_time'] = self._v_total / timer_count
966            dd['i_time'] = self.total_time
967            dd['count'] = timer_count
968            results["total"] = dd
969            return results
970    ###)
971
972    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
973    def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
974        "create Datastructure for an admitted Student"
975        #import pdb;pdb.set_trace()
976        logger = logging.getLogger('WAeUPTool.admitOneStudent')
977        self.with_timing = with_timing
978        if not hasattr(self,"_v_certificates"):
979            self._v_certificates = self.getCertificatesDict()
980        students_folder = self.portal_url.getPortalObject().campus.students
981        if brain.screening_type in ('cest','sandwich',):
982            reg_no = "%s%s/07" % (brain.course1,brain.serial)
983        else:
984            reg_no = brain.reg_no
985        res = self.students_catalog(jamb_reg_no = reg_no)
986        if res:
987            logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
988            return
989        if brain.status != "admitted":
990            logger.info('status of %s is %s' % (reg_no,brain.status))
991            return
992        pin_parts = brain.pin.split('-')
993        if pin_parts and len(pin_parts) != 3:
994            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
995            return
996        if brain.course_admitted not in self._v_certificates:
997            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
998            return
999        if brain.sex not in (True,False):
1000            logger.info('sex of %s not available' % (reg_no))
1001            return
1002        self.init_timing()
1003        student_id = self.generateStudentId('?')
1004        students_folder.invokeFactory('Student', student_id)
1005        student_object = getattr(students_folder,student_id)
1006        self.do_timing()
1007        if pin_password:
1008            password = pin_parts[2]
1009            self.makeStudentMember(student_id,password = password)
1010        student_object.manage_setLocalRoles(student_id, ['Owner',])
1011        self.do_timing()
1012        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
1013        #
1014        # application
1015        #
1016        student_object.invokeFactory('StudentApplication','application')
1017        application = student_object.application
1018        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
1019        self.do_timing()
1020        da = {'Title': 'Application Data'}
1021        da['jamb_reg_no'] = reg_no
1022
1023        sex = 'M'
1024        if brain.sex:
1025            sex = 'F'
1026        da['jamb_sex'] = sex
1027        da['jamb_age'] = brain.jamb_age
1028        da['app_reg_pin'] = brain.pin
1029        da['jamb_lga'] = brain.jamb_lga
1030        da['jamb_state'] = brain.jamb_state
1031        da['jamb_score'] = brain.aggregate
1032        da['app_email'] = brain.email
1033        da['app_mobile'] = brain.phone
1034
1035        da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1036
1037        da['entry_session'] = entry_session
1038        da['jamb_lastname'] = brain.lastname
1039        da['jamb_middlename'] = brain.middlenames   # different field names!
1040        da['jamb_firstname'] = brain.firstname
1041        da['screening_application_date'] = brain.application_date
1042        da['date_of_birth'] = brain.date_of_birth
1043        da['jamb_first_cos'] = brain.course1
1044        da['jamb_second_cos'] = brain.course2
1045        da['course3'] = brain.course3
1046        da['screening_type'] = brain.screening_type
1047        da['screening_score'] = brain.screening_score
1048        da['screening_date'] = brain.screening_date
1049        da['hq_type'] = brain.hq_type
1050        da['hq_grade'] = brain.hq_grade
1051        da['aos'] = brain.aos
1052
1053        application.getContent().edit(mapping=da)
1054        self.do_timing()
1055        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
1056        #
1057        # personal
1058        #
1059        student_object.invokeFactory('StudentPersonal','personal')
1060        personal = student_object.personal
1061        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1062        self.do_timing()
1063        dp = {'Title': 'Personal Data'}
1064        dp['sex'] = brain.sex
1065        dp['email'] = brain.email
1066        dp['phone'] = brain.phone
1067        dp['lastname'] = brain.lastname
1068        dp['middlename'] = brain.middlenames   # different field names!
1069        dp['firstname'] = brain.firstname
1070        personal.getContent().edit(mapping=dp)
1071        self.do_timing()
1072        #
1073        # clearance
1074        #
1075        student_object.invokeFactory('StudentClearance','clearance')
1076        clearance = student_object.clearance
1077        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1078        dc = {'Title': 'Clearance/Eligibility Record'}
1079        dc['lga'] = brain.lga
1080        dc['birthday'] = brain.date_of_birth
1081        clearance.getContent().edit(mapping=dc)
1082        self.do_timing()
1083        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1084        #
1085        # study Course
1086        #
1087        student_object.invokeFactory('StudentStudyCourse','study_course')
1088        studycourse = student_object.study_course
1089        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1090        self.do_timing()
1091        dsc = {}
1092        dsc['study_course'] = brain.course_admitted
1093        dsc['current_verdict'] = ''
1094        dsc['current_mode'] = da['entry_mode']
1095        if da['entry_mode'].startswith('de'):
1096            dsc['current_level'] = '200'
1097        elif da['entry_mode'].startswith('pre'):
1098            dsc['current_level'] = '000'
1099        else:
1100            dsc['current_level'] = '100'
1101        dsc['current_session'] = entry_session
1102        studycourse.getContent().edit(mapping=dsc)
1103        self.do_timing()
1104        #
1105        # payments folder
1106        student_object.invokeFactory('PaymentsFolder','payments')
1107        payments = getattr(student_object,'payments')
1108        self.do_timing()
1109        dpay = {}
1110        dpay['Title'] = 'Payments'
1111        payments.getContent().edit(mapping=dpay)
1112        self.portal_workflow.doActionFor(payments,'open')
1113        self.do_timing()
1114        #
1115        # passport foto
1116        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1117                                                              brain.screening_type,
1118                                                              brain.reg_no)
1119        images_dir = getImagesDir(student_id)
1120        #images_dir = os.path.join("%s" % images_base,student_id)
1121        letter_dir,student_dir = os.path.split(images_dir)
1122        if not os.path.exists(letter_dir):
1123            os.mkdir(letter_dir)
1124        if not os.path.exists(images_dir):
1125            os.mkdir(images_dir)
1126        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1127        if os.path.exists(app_picture):
1128            copy2(app_picture,image_name)
1129        else:
1130            logger.info('passport of %s/%s not found: %s' % (student_id,
1131                                                             brain.reg_no,
1132                                                             app_picture))
1133
1134        self.do_timing()
1135        self.print_timing()
1136        if with_timing:
1137            self.timer_step = 0
1138            self._v_timer_count += 1
1139            self._v_total += self.total_time
1140        return student_id
1141    ###)
1142
1143    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1144    def makeStudentLevel(self,student_id):
1145        "create the StudyLevel for a returning Student"
1146        #import pdb;pdb.set_trace()
1147        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1148        students_folder = self.portal_url.getPortalObject().campus.students
1149        res = self.students_catalog(id=student_id)
1150        if res:
1151            st = res[0]
1152        course = st.course
1153        matric_no = st.matric_no
1154        level = st.level
1155        res = self.results_import(matric_no = matric_no)
1156        if res:
1157            results = res
1158        logger.info('%s creating Level %s' % (student_id,level))
1159        #
1160        # Level
1161        #
1162        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1163        studycourse = getattr(student_obj,"study_course",None)
1164        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1165        l = getattr(studycourse,level,None)
1166        if l is None:
1167            studycourse.invokeFactory('StudentStudyLevel', level)
1168            l = getattr(studycourse, level)
1169            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1170            l.getContent().edit(mapping={'Title': "Level %s" % level})
1171        ###)
1172
1173    security.declarePublic('getHallInfo') ###(
1174    def getHallInfo(self,bed):
1175        """return Hall Info"""
1176        info = {}
1177        hall,block,room,letter = bed.split('_')
1178        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1179        if res and len(res) == 1:
1180            hall_brain = res[0]
1181            hall_doc = hall_brain.getObject().getContent()
1182        else:
1183            return info
1184        info['hall_title'] = hall_brain.Title
1185        info['maintenance_code'] = hall_doc.maintenance_code
1186        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1187        batch_doc = None
1188        for brain in res:
1189            if brain.id.startswith(info['maintenance_code']):
1190                batch_doc = brain.getObject().getContent()
1191                break
1192        if batch_doc is None:
1193            info['maintenance_fee'] = ''
1194        else:
1195            info['maintenance_fee'] = batch_doc.cost
1196        return info
1197    ###)
1198
1199    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1200    def removePictureFolder(self,student_id):
1201        """remove picture_folder by renaming it"""
1202        path = 'images'
1203        picture_path = os.path.join(i_home,path,student_id)
1204        if not os.path.exists(picture_path):
1205            return False
1206        os.rename(picture_path,picture_path + "_removed")
1207        return True
1208    ###)
1209
1210    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1211    def restorePictureFolder(self,student_id):
1212        """restore picture_folder by renaming it"""
1213        path = 'images'
1214        picture_path = os.path.join(i_home,path,student_id)
1215        if not os.path.exists(picture_path + "_removed"):
1216            return False
1217        os.rename(picture_path + "_removed",picture_path)
1218        return True
1219    ###)
1220
1221    security.declarePublic('picturesExist') ###(
1222    def picturesExist(self, ids,student_id=None):
1223        """check if pictures exist in the filesystem"""
1224        if student_id is None:
1225            student_id = self.getStudentId()
1226        if student_id is None:
1227            return False
1228        picture_path = getImagesDir(student_id)
1229        #picture_path = os.path.join(images_base,student_id)
1230        if not os.path.exists(picture_path):
1231            return False
1232        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1233        return set(ids).issubset(set(pictures))
1234    ###)
1235
1236    security.declarePublic('picturesList') ###(
1237    def picturesList(self):
1238        """check if pictures exist in the filesystem"""
1239        path = 'images'
1240        student_id = self.getStudentId()
1241        #picture_path = os.path.join(i_home,path,student_id)
1242        picture_path = getImagesDir(student_id)
1243        if not os.path.exists(picture_path):
1244            return []
1245        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1246    ###)
1247
1248    security.declarePublic('showFsPicture') ###(
1249    def showFsPicture(self,path):
1250        """return a picture from the filesystem"""
1251        #picture_path = os.path.join(i_home,path)
1252        picture_path = os.path.join(images_base,path)
1253        response = self.REQUEST.RESPONSE
1254        #import pdb;pdb.set_trace()
1255        registry = getToolByName(self, 'mimetypes_registry')
1256        mimetype = str(registry.lookupExtension(path.lower()) or
1257                    registry.lookupExtension('file.bin'))
1258        if os.path.exists(picture_path):
1259            response.setHeader('Content-type',mimetype)
1260            return open(picture_path).read()
1261        picture_path = os.path.join(i_home,'import',path)
1262        if os.path.exists(picture_path):
1263            return open(picture_path).read()
1264    ###)
1265
1266    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1267    def deleteAllCourses(self,department="All"):
1268        ''' delete the courses'''
1269        pm = self.portal_membership
1270        member = pm.getAuthenticatedMember()
1271
1272        if str(member) not in ("henrik","joachim"):
1273            return "not possible"
1274        if department == "All":
1275            res = self.portal_catalog({'meta_type': 'Department'})
1276        if len(res) < 1:
1277            return "No Departments found"
1278
1279        deleted = []
1280        for dep in res:
1281            cf = dep.getObject().courses
1282            if cf:
1283                cf.manage_delObjects(ids=cf.objectIds())
1284                deleted.append("deleted Courses in %s" % dep.getId)
1285        return "\r".join(deleted)
1286    ###)
1287
1288    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1289    def getLogfileLines(self,filename="event.log",numlines=20):
1290        """Get last NUMLINES lines of logfile FILENAME.
1291
1292        Return last lines' of a file in the instances logfile directory as
1293        a list. The number of returned lines equals `numlines' or less. If
1294        less than `numlines' lines are available, the whole file ist
1295        returned. If the file can not be opened or some other error
1296        occurs, empty list is returend.
1297        """
1298        result = []
1299        lines_hit = 0
1300
1301        # We only handle files in instances' log directory...
1302        logpath = os.path.join(i_home, "log")
1303        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1304        if not filename.startswith( logpath ):
1305            # Attempt to access file outside log-dir...
1306            return []
1307
1308        try:
1309            fd = file( filename, "rb" )
1310        except IOError:
1311            return []
1312        if not fd:
1313            return []
1314
1315        if os.linesep == None:
1316            linesep = '\n'
1317        else:
1318            linesep = os.linesep
1319
1320        # Try to find 'numlines' times a lineseparator, searching from end
1321        # and moving to the beginning of file...
1322        fd.seek( 0, 2) # Move to end of file...
1323        while lines_hit < numlines:
1324            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1325                                          # one step forward...
1326                lines_hit += 1
1327            try:
1328                fd.seek( -2, 1) # Go two bytes back from current pos...
1329            except IOError:
1330                # We cannot go back two bytes. Maybe the file is too small...
1331                break
1332        fd.seek(2,1)
1333
1334        # Read all lines from current position...
1335        result = fd.readlines()
1336        # Remove line endings...
1337        result = [x.strip() for x in result]
1338        fd.close()
1339        return result
1340    ###)
1341
1342    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1343    def getCallbacksFromLog(self,filename):
1344        """fix Online Payment Transactions from Z2.log entries"""
1345        import transaction
1346        import random
1347        from cgi import parse_qs
1348        from urlparse import urlparse
1349        #from pdb import set_trace
1350        wftool = self.portal_workflow
1351        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1352        students_folder = self.portal_url.getPortalObject().campus.students
1353        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1354        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1355        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1356        data = re.compile(s)
1357        start = True
1358        tr_count = 1
1359        total = 0
1360        #name = 'pume_results'
1361        #name = 'epaymentsuccessful_z2log2'
1362        name = filename
1363        no_import = []
1364        imported = []
1365        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1366        try:
1367            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1368        except:
1369            logger.error('Error reading %s' % name)
1370            return
1371        tas = []
1372        for line in transactions:
1373            dict = {}
1374            items = data.search(line)
1375            dict['idict'] = idict = items.groupdict()
1376            #print idict
1377            #from pdb import set_trace;set_trace()
1378            urlparsed = urlparse(idict['get'][4:])
1379            #print urlparsed
1380            path = urlparsed[2].split('/')
1381            dict['student_id'] = student_id = path[8]
1382            dict['payment_id'] = payment_id = path[10]
1383            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1384            tas.append(dict)
1385            tr_count += 1
1386        return tas
1387    ###)
1388
1389    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1390    def importOnlinePaymentTransactions(self):
1391        """load Online Payment Transactions from CSV values"""
1392        import transaction
1393        import random
1394        #from pdb import set_trace
1395        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1396        opt = self.online_payments_import
1397        students_folder = self.portal_url.getPortalObject().campus.students
1398        start = True
1399        tr_count = 1
1400        total = 0
1401        #name = 'pume_results'
1402        name = 'OnlineTransactions'
1403        no_import = []
1404        imported = []
1405        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1406        try:
1407            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1408        except:
1409            logger.error('Error reading %s.csv' % name)
1410            return
1411        for pay_transaction in transactions:
1412            if start:
1413                start = False
1414                logger.info('Start loading from %s.csv' % name)
1415                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1416                no_import.append('%s,"Error"' % s)
1417                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1418                format_error = format + ',"%(Error)s"'
1419            data = {}
1420
1421            # format of the first file sent by Tayo
1422            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1423            #data['student_id'] = student_id = pay_transaction['Payer ID']
1424            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1425            #data['response_code'] = response_code = pay_transaction['Resp Code']
1426            #data['amount'] = amount = pay_transaction['Amount']
1427
1428            # format of the second file sent by Tayo
1429            #data['datetime'] = date = 0
1430            #data['student_id'] = student_id = pay_transaction['Payer ID']
1431            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1432            #data['response_code'] = response_code = '00'
1433            #data['amount'] = amount = pay_transaction['Amount']
1434
1435            # format of the third file sent by Kehinde
1436            data['datetime'] = date = 0
1437            data['student_id'] = student_id = pay_transaction['customer_id']
1438            data['order_id'] = order_id = pay_transaction['merchant_reference']
1439            data['response_code'] = response_code = '00'
1440            data['amount'] = amount = pay_transaction['Amount']
1441
1442            dup = False
1443            if response_code == "12":
1444                continue
1445            try:
1446                opt.addRecord(**data)
1447            except ValueError:
1448                dup = True
1449            #from pdb import set_trace;set_trace()
1450            if dup:
1451                if response_code == "00":
1452                    try:
1453                        opt.modifyRecord(**data)
1454                    except:
1455                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1456                        continue
1457                else:
1458                    pay_transaction['Error'] = "Duplicate order_id"
1459                    no_import.append( format_error % pay_transaction)
1460                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1461                    continue
1462            tr_count += 1
1463            if tr_count > 1000:
1464                if len(no_import) > 0:
1465                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1466                             '\n'.join(no_import) + '\n')
1467                    no_import = []
1468                em = '%d transactions committed\n' % (tr_count)
1469                transaction.commit()
1470                regs = []
1471                logger.info(em)
1472                total += tr_count
1473                tr_count = 0
1474        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1475                                                '\n'.join(no_import))
1476        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1477    ###)
1478
1479    security.declareProtected(ModifyPortalContent,"importData")###(
1480    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1481        """load data from CSV values"""
1482        import transaction
1483        import random
1484        students_folder = self.portal_url.getPortalObject().campus.students
1485        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1486        pending_only = False
1487        pend_str = '--'
1488        elapse = time.time()
1489        #
1490        # preparations
1491        #
1492        if filename == pend_str:
1493            pending_only = True
1494        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1495        importer = eval("%sImport" % importer_name)(self)
1496        logger = importer.logger
1497        if importer.init_errors:
1498            logger.info(importer.init_errors)
1499            return importer.init_errors
1500        member = importer.member
1501        #current = importer.current
1502        import_date = importer.import_date
1503        #
1504        # not_imported
1505        #
1506        info = importer.info
1507        data_keys = importer.data_keys
1508        csv_keys = importer.csv_keys
1509        #csv_keys.extend(info.keys())
1510        headline_mapping = dict((k,k) for k in csv_keys)
1511        #
1512        # pending
1513        #
1514        pending_path = importer.pending_path
1515        pending_tmp = importer.pending_tmp
1516        pending_backup = importer.pending_backup
1517        pending_fn = importer.pending_fn
1518        imported_path = importer.imported_path
1519        imported_fn = importer.imported_fn
1520        commit_after = importer.commit_after
1521        pending = []
1522        pending_digests = []
1523        #total_added_to_pending = 0
1524        if not pending_only:
1525            pending,pending_digests = importer.makeIdLists()
1526            pending_at_start = len(pending)
1527        datafile = open(pending_tmp,"w")
1528        pending_csv_writer = csv.DictWriter(datafile,
1529                                                    csv_keys,
1530                                                    extrasaction='ignore')
1531        pending_csv_writer.writerow(headline_mapping)
1532        datafile.close()
1533        #
1534        # imported
1535        #
1536        if not os.path.exists(imported_path):
1537            datafile = open(imported_path,"w")
1538            imported_csv_writer = csv.DictWriter(datafile,
1539                                                 csv_keys,
1540                                                 extrasaction='ignore')
1541            imported_csv_writer.writerow(headline_mapping)
1542            datafile.close()
1543        start = True
1544        tr_count = 0
1545        total = 0
1546        total_added_to_imported = 0
1547        total_pending = 0
1548        import_source_done = ""
1549        if pending_only:
1550            import_source_path = pending_path
1551        else:
1552            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1553            import_source_done = "%s/import/%s.done" % (i_home,filename)
1554        if not os.path.exists(import_source_path):
1555            fn = os.path.split(import_source_path)[1]
1556            em = 'no import file %(fn)s' % vars()
1557            return em
1558        import_source_fn = os.path.split(import_source_path)[1]
1559        if not pending_only:
1560            info['imported_from'] = import_source_fn
1561        headline = csv.reader(open(import_source_path,"rb")).next()
1562        if "import_mode" not in headline:
1563            msg = 'import_mode must be in heading'
1564            return msg
1565        invalid_keys = importer.checkHeadline(headline)
1566        if invalid_keys:
1567            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1568
1569        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1570                                                         or k.strip() in info.keys())]
1571        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1572        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1573        # if diff2schema and diff2schema != set(['id',]):
1574        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1575        #     return msg
1576        #
1577        # start importing
1578        #
1579        try:
1580            reader = csv.DictReader(open(import_source_path,"rb"))
1581        except:
1582            msg = 'Error reading %s.csv' % filename
1583            logger.error(msg)
1584            return msg
1585        items = [item for item in reader]
1586        total_to_import = len(items)
1587        tti_float = float(total_to_import)
1588        if pending_only:
1589            pending_at_start = total_to_import
1590        count = 0
1591        imported = []
1592        old_commit_count = -1
1593        error_count = imported_count = 0
1594        already_in = 0
1595        for record in items:
1596            item = {}
1597            empty_value_keys = []
1598            for k,v in record.items():
1599                if k is None:
1600                    continue
1601                if v:
1602                    if v == EMPTY:
1603                        empty_value_keys += k,
1604                        v = ''
1605                    item[k.strip()] = v.strip()
1606            count += 1
1607            if start:
1608                start = False
1609                adapters = [MappingStorageAdapter(importer.schema, item)]
1610                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1611            dm = DataModel(item, adapters,context=self)
1612            ds = DataStructure(data=item,datamodel=dm)
1613            error_string = ""
1614            total += 1
1615            import_mode = item.get('import_mode','')
1616            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1617            if import_method is None:
1618                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1619            elif (import_mode in importer.required_modes and
1620                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1621                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1622                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1623            else:
1624                for k in import_keys:
1625                    if k not in item.keys() or k not in importer.validators.keys():
1626                        continue
1627                    if not importer.validators[k](ds,mode=import_mode):
1628                        error_string += ' ++ '
1629                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1630                                                                           ds.getErrorMapping(k)))
1631            if error_string:
1632                error = error_string
1633                id = ''
1634                mapping = item
1635            else:
1636                temp_item = item.copy()
1637                temp_item.update(dm)
1638                results = import_method(temp_item)
1639                id = results[0]
1640                error = results[1]
1641                mapping = results[2]
1642            #
1643            # restore some values
1644            #
1645            for k in empty_value_keys:
1646                mapping[k] = EMPTY
1647            if mapping.has_key('sex'):
1648                #import pdb;pdb.set_trace()
1649                if mapping['sex'] == True:
1650                    mapping['sex'] = 'F'
1651                elif mapping['sex'] == False:
1652                    mapping['sex'] = 'M'
1653            if pending_only:
1654                info['imported_from'] = item['imported_from']
1655            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1656            info['error'] = error
1657            info['import_record_no'] = count + 1
1658            mapping.update(info)
1659            log_list = []
1660            if error:
1661                error_count += 1
1662                digest = makeDigest(mapping,data_keys)
1663                if digest not in pending_digests:
1664                    pending_digests += digest,
1665                    pending.append(mapping)
1666                    if not pending_only:
1667                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1668                else:
1669                    already_in += 1
1670                    pass
1671            else:
1672                imported_count += 1
1673                imported += mapping,
1674                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1675            if log_list:
1676                time_till_now = time.time() - elapse
1677                percent_finished = (error_count + imported_count)/tti_float*100
1678                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1679                logger.info(' '.join(log_list))
1680            finished = count > total_to_import - 1
1681            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1682            if must_commit:
1683                old_commit_count = imported_count
1684
1685            if must_commit or finished:
1686                if len(imported):
1687                    transaction.commit()
1688                    datafile = open(imported_path,"a")
1689                    writer = csv.DictWriter(datafile,
1690                                            csv_keys,
1691                                            extrasaction='ignore')
1692                    writer.writerows(imported)
1693                    datafile.close()
1694                    total_added_to_imported += len(imported)
1695                    imported = []
1696                if len(pending) > 0:
1697                    datafile = open(pending_tmp,"a")
1698                    writer = csv.DictWriter(datafile,
1699                                            csv_keys,
1700                                            extrasaction='ignore')
1701                    writer.writerows(pending)
1702                    datafile.close()
1703                    total_pending += len(pending)
1704                    #total_added_to_pending += len(pending)
1705                    pending = []
1706                if not finished:
1707                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1708                    logger.info(msg)
1709        elapse = time.time() - elapse
1710        if os.path.exists(pending_path):
1711            copy2(pending_path,pending_backup)
1712        copy2(pending_tmp,pending_path)
1713        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1714        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1715        if pending_only:
1716            removed_pending = pending_at_start - total_pending
1717            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1718        else:
1719            added_pending = total_pending - pending_at_start
1720            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1721        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1722        logger.info(msg)
1723        if import_source_done:
1724            copy(import_source_path,import_source_done)
1725            os.remove(import_source_path)
1726            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1727            if upload is not None:
1728                upload_doc = upload.getContent()
1729                mapping = {}
1730                #mapping['import_date'] = DateTime.DateTime()
1731                mapping['import_date'] = import_date
1732                mapping['imported_by'] = importer.imported_by
1733                mapping['import_message'] = msg
1734                upload_doc.edit(mapping = mapping)
1735        os.remove(pending_tmp)
1736        return msg
1737    ###)
1738
1739    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1740    def moveImagesToFS(self,student_id="O738726"):
1741        "move the images to the filesystem"
1742        images_dir = getImagesDir(student_id)
1743        #images_dir = os.path.join("%s" % images_base,student_id)
1744        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1745        stool = getToolByName(self, 'portal_schemas')
1746        schemas = ['student_application',
1747                   'student_clearance',
1748                   ]
1749        created = False
1750        for schema_id in schemas:
1751            schema = stool._getOb(schema_id)
1752            object = getattr(student_folder,schema_id[len('student_'):],None)
1753            if object is None:
1754                continue
1755            doc = object.getContent()
1756            for key in schema.keys():
1757                if schema[key].meta_type != "CPS Image Field":
1758                    continue
1759                #import pdb;pdb.set_trace()
1760                image = getattr(doc,key,None)
1761                if not image or not hasattr(image,"data"):
1762                    continue
1763                if not created:
1764                    if not os.path.exists(images_dir):
1765                        os.mkdir(images_dir)
1766                    created = True
1767                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1768                open(filename,"wb").write(str(image.data))
1769    ###)
1770
1771    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1772    def movePassportToFS(self,student_id="O738726"):
1773        "move the passports to the filesystem"
1774        images_dir = os.path.join("%s" % i_home,'passports')
1775        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1776        stool = getToolByName(self, 'portal_schemas')
1777        schemas = ['student_application',
1778                   #'student_clearance',
1779                   ]
1780        created = False
1781        for schema_id in schemas:
1782            schema = stool._getOb(schema_id)
1783            object = getattr(student_folder,schema_id[len('student_'):],None)
1784            if object is None:
1785                continue
1786            doc = object.getContent()
1787            for key in schema.keys():
1788                if schema[key].meta_type != "CPS Image Field":
1789                    continue
1790                #import pdb;pdb.set_trace()
1791                image = getattr(doc,key)
1792                if not hasattr(image,"data"):
1793                    continue
1794                if not created:
1795                    if not os.path.exists(images_dir):
1796                        os.mkdir(images_dir)
1797                    created = True
1798                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1799                open(filename,"wb").write(str(image.data))
1800    ###)
1801
1802InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.