source: WAeUP_SRP/trunk/WAeUPTool.py @ 3863

Last change on this file since 3863 was 3858, checked in by Henrik Bettermann, 16 years ago

implement password notification module (send mail not yet tested)
  • Property svn:keywords set to Id
File size: 75.4 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3858 2009-01-17 07:29:24Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108    ###)
109
110    security.declareProtected(View,'updateRoleMappingsFor') ###(
111    def updateRoleMappingsFor(self,wf_definition,ob):
112        "do so for public"
113        wf_def = getattr(self.portal_workflow,wf_definition)
114        wf_def.updateRoleMappingsFor(ob)
115    ###)
116
117    security.declareProtected(View,'getStatesLgas') ###(
118    def getStatesLgas(self):
119        """return lga info"""
120        voc = getattr(self.portal_vocabularies,'local_gov_areas')
121        states = []
122        lgas  = []
123        d = {}
124        wd = {}
125        for k,v in voc.items():
126            parts = v.split(' / ')
127            if len(parts) == 1:
128                state = parts[0].lower()
129                lga = ""
130            elif len(parts) == 2:
131                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133            else:
134                continue
135            if state not in states:
136                states.append(state)
137            if lga not in lgas:
138                lgas.append(lga)
139            words = re.split('[^a-zA-Z0-9/]',k)
140            words.sort()
141            wd[k] = words
142            d[k] = v
143        mapping = {}
144        mapping['word_dict'] = wd
145        mapping['lga_dict'] = d
146        mapping['states'] = states
147        mapping['lgas'] = lgas
148        return mapping
149    ###)
150
151    security.declareProtected(View,'findLga') ###(
152    def findLga(self,words,words_dict):
153        words = re.split('[^a-zA-Z0-9/]',words)
154        lga_words = []
155        for word in words:
156            if word:
157                lga_words += word.strip().lower(),
158        lga_words.sort()
159        state_lga = ''
160        while not state_lga:
161            for k,l in words_dict.items():
162                if lga_words == l:
163                    state_lga = k
164                    break
165            break
166        return state_lga
167    ###)
168    security.declareProtected(View,'getAccessInfo') ###(
169    def getAccessInfo(self,context):
170        "return a dict with access_info"
171        logger = logging.getLogger('WAeUPTool.getAccessInfo')
172        mtool = self.portal_membership
173        member = mtool.getAuthenticatedMember()
174        member_id = str(member)
175        info = {}
176        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178        is_staff = info['is_staff'] = not is_anonymous and not is_student
179        roles = member.getRolesInContext(context)
180        info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181                                                        "SectionManager" in roles or
182                                                        "Manager" in roles)
183        info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184        #is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if not is_anonymous and requested_id:
188            if (is_student and member_id == requested_id) or is_staff:
189                student_id = requested_id
190            else:   
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192        info['student_id'] = student_id
193        return info
194    ###)
195
196    security.declareProtected(ModifyPortalContent,'openLog') ###(
197    def openLog(self,name):
198        """open a log file"""
199        version = 1
200        path = "%s/log/%s_%d.log" % (i_home,name,version)
201        while os.path.exists(path):
202            version += 1
203            path = "%s/log/%s_%d.log" % (i_home,name,version)
204        log = open(path,"w")
205        return log
206    ###)
207
208    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
209    def bypassQueueCatalog(self,enable=True):
210        """bypass the QueueCatalog by setting all indexes to process imediate,
211        if enable is True (default) the old settings are restored
212        """
213
214    ###)
215
216    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
217    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
218        """measure Object access Time"""
219        import random
220        if hasattr(self,'portal_catalog_real'):
221            aq_portal = self.portal_catalog_real.evalAdvancedQuery
222        else:
223            aq_portal = self.portal_catalog.evalAdvancedQuery
224        nr_pts = int(nr_pts)
225        probe = int(probe)
226        intervall = probe/10
227        objects = ("application","clearance","personal")
228        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
229        #i = random.randrange(num_objects)
230        count = 0
231        found = 0
232        not_found = 0
233        t_found = 0
234        t_not_found = 0
235        time_found = time_not_found = 0.0
236        t_time_found = t_time_not_found = 0.0
237        accessed = []
238        t_min = 1000
239        t_max = 0
240        #import pdb;pdb.set_trace()
241        students = self.portal_catalog(portal_type="Student")
242        num_students = len(students)
243        if method == "d":
244            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
245            res = aq_portal(query)
246            brains = {}
247            for r in res:
248                sid = r.relative_path.split('/')[-2]
249                if brains.has_key(sid):
250                    brains[sid][r.portal_type] = r
251                else:
252                    brains[sid] = {r.portal_type : r}
253            brains_list = brains.keys()
254            num_objects = len(brains_list)
255        else:
256            num_objects = num_students
257        print "="*40
258        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
259                                                                        probe,
260                                                                        nr_pts,
261                                                                        num_objects)
262        print "nr found/not time found/not min/max"
263        elapse = time.time()
264        i_elapse = time.time()
265        c_elapse = time.clock()
266        for c in range(1,probe + 1):
267            i = random.randrange(num_objects)
268            if method in ('a','b','c'):
269                student_brain = students[i]
270            elif method == "d":
271                #import pdb;pdb.set_trace()
272                student_brain = brains[brains_list[i]]
273            if method == "c":
274                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
275                res = aq_portal(query)
276                this_portal_types = [r.portal_type for r in res]
277            for i in range(nr_pts):
278                oid = objects[i]
279                if method == "a":
280                    try:
281                        student_path = student_brain.getPath()
282                        path = "%s/%s" % (student_path,oid)
283                        doc = self.unrestrictedTraverse(path).getContent()
284                        found += 1
285                        i_time = time.time() - i_elapse
286                        time_found += i_time
287                    except:
288                        not_found += 1
289                        i_time = time.time() - i_elapse
290                        time_not_found += i_time
291                        pass
292                elif method == "b":
293                    try:
294                        student_object = student_brain.getObject()
295                        doc = getattr(student_object,oid).getContent()
296                        found += 1
297                        i_time = time.time() - i_elapse
298                        time_found += i_time
299                    except:
300                        i_time = time.time() - i_elapse
301                        time_not_found += i_time
302                        not_found += 1
303                        pass
304                elif method == "c":
305                    if portal_types[i] in this_portal_types:
306                        found += 1
307                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
308                        i_time = time.time() - i_elapse
309                        time_found += i_time
310                    else:
311                        not_found += 1
312                        i_time = time.time() - i_elapse
313                        time_not_found += i_time
314                elif method == "d":
315                    if student_brain.has_key(portal_types[i]):
316                        found += 1
317                        doc = student_brain[portal_types[i]].getObject().getContent()
318                        i_time = time.time() - i_elapse
319                        time_found += i_time
320                    else:
321                        not_found += 1
322                        i_time = time.time() - i_elapse
323                        time_not_found += i_time
324                i_elapse = time.time()
325            if c and (c % intervall == 0):
326                #i_time = time.time() - i_elapse
327                t_per = 0.0
328                if found:
329                    t_per = time_found/found
330                if t_per > t_max:
331                    t_max = t_per
332                if t_per > 0.0 and t_per < t_min:
333                    t_min = t_per
334                itf = 0.0
335                if found:
336                    itf = time_found/found
337                itnf = 0.0
338                if not_found :
339                    itnf = time_not_found / not_found
340                interval_time = time_found + time_not_found
341                s = "%(c)d: %(found)d/%(not_found)d " % vars()
342                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
343                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
344                print s
345                t_found += found
346                t_not_found += not_found
347                t_time_found += time_found
348                t_time_not_found += time_not_found
349                time_found = time_not_found = 0.0
350                found = not_found = 0
351        # t_found += found
352        # t_not_found += not_found
353        elapse = time.time() - elapse
354        itf = 0.0
355        if t_found:
356            itf = t_time_found/t_found
357        itnf = 0.0
358        if t_not_found:
359            itnf = t_time_not_found / t_not_found
360        #c_elapse = time.clock() - c_elapse
361        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
362        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
363        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
364        print "-"*40
365        print s
366        rel_found = float(t_found)/probe
367        rel_not_found = float(t_not_found)/probe
368        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
369        print estimated_total_time
370    ###)
371
372    security.declareProtected(ModifyPortalContent,'writeLog') ###(
373    def writeLog(self,logfile,s):
374        """write to the log file"""
375        logfile.write(s)
376    ###)
377
378    def generateStudentId(self,letter): ###(
379        import random
380        logger = logging.getLogger('WAeUPTool.generateStudentId')
381        r = random
382        if letter == '?':
383            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
384        sid = "%c%d" % (letter,r.randint(99999,1000000))
385        students = self.portal_url.getPortalObject().campus.students
386        while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid):
387            if self.waeup_tool.picturePathExists(sid):
388                logger.info('picture path %s exists' % sid)
389            sid = "%c%d" % (letter,r.randint(99999,1000000))
390        return sid
391    ###)
392
393    def generatePassword(self,s=None): ###(
394        import random
395        r = random
396        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
397        if s is None:
398            s = 'abcdefghklmnpqrstuvwxy23456789'
399        pw = ''
400        while len(pw) < 6:
401            pw += r.choice(s)
402        return pw
403    ###)
404
405    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
406    def dumpSchoolfeePayments(self):
407        "dump paid schoolfees"
408        mtool = self.portal_membership
409        member = mtool.getAuthenticatedMember()
410        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
411        aq_student = self.students_catalog.evalAdvancedQuery
412        query = In('review_state',('schoolfee_paid',
413                                   'courses_registered',
414                                   'courses_validated',
415                                   ))
416        res = aq_student(query)
417        #import pdb;pdb.set_trace()
418        l = []
419        logger.info("start for %d" % len(res))
420        count = 1
421        log_after = 100
422        for student in res:
423            if not count % log_after:
424                logger.info("processed %d total %d" % (log_after,count))
425            count += 1
426            fee_dict =self.getSchoolFee(student)
427            fulltime = student.mode.endswith('_ft')
428            d = {}
429            d['student_id'] = student.id
430            d['name'] = student.name
431            d['amount'] = fee_dict.get(new_returning)
432            l += d,
433        csv_name = self.dumpListToCSV(l,'payments')
434        logger.info('%s dumped payments to %s' % (member,export_file))
435    ###)
436
437    security.declarePublic('dumpListToCSV') ###(
438    def dumpListToCSV(self,l,filename,fields=None):
439        """dump a list of dicts to a CSV file"""
440        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
441        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
442        if fields is None:
443            fields = l[0].keys()
444        headline = ','.join(fields)
445        out = open(export_file,"wb")
446        out.write(headline +'\n')
447        out.close()
448        out = open(export_file,"a")
449        csv_writer = csv.DictWriter(out,fields,)
450        csv_writer.writerows(l)
451        return export_file
452    ###)
453
454    security.declareProtected(ManagePortal, 'listMembers') ###(
455    def listMembers(self):
456        "list all members"
457        mtool = self.portal_membership
458        member = mtool.getAuthenticatedMember()
459        logger = logging.getLogger('WAeUPTool.listMembers')
460        if str(member) not in ('admin','joachim'):
461            logger.info('%s tried to list members' % (member))
462            return None
463        members = self.portal_directories.members
464        all = members.listEntryIdsAndTitles()
465        l = []
466        for user_id,name in all:
467            d = {}
468            d['user_id'] = user_id
469            d['name'] = name
470            d['pw'] = getattr(getattr(members,user_id),'password')
471            d['email'] = getattr(getattr(members,user_id),'email')
472            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
473            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
474            l += d,
475        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
476        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
477        logger.info('%s dumped member list to %s' % (member,export_file))
478        fields = l[0].keys()
479        headline = ','.join(fields)
480        out = open(export_file,"wb")
481        out.write(headline +'\n')
482        out.close()
483        out = open(export_file,"a")
484        csv_writer = csv.DictWriter(out,fields,)
485        csv_writer.writerows(l)
486    ###)
487
488    security.declareProtected(ManagePortal, 'listStudents') ###(
489    def listStudents(self):
490        "list all students"
491        mtool = self.portal_membership
492        member = mtool.getAuthenticatedMember()
493        logger = logging.getLogger('WAeUPTool.listStudents')
494        if str(member) not in ('admin','joachim'):
495            logger.info('%s tried to list students' % (member))
496            return None
497        students = self.portal_directories.students
498        all = students.listEntryIdsAndTitles()
499        l = []
500        for user_id,name in all:
501            d = {}
502            d['user_id'] = user_id
503            d['name'] = name
504            d['pw'] = getattr(getattr(students,user_id),'password')
505            d['email'] = getattr(getattr(students,user_id),'email')
506            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
507            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
508            l += d,
509        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
510        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
511        logger.info('%s dumped student list to %s' % (member,export_file))
512        fields = l[0].keys()
513        headline = ','.join(fields)
514        out = open(export_file,"wb")
515        out.write(headline +'\n')
516        out.close()
517        out = open(export_file,"a")
518        csv_writer = csv.DictWriter(out,fields,)
519        csv_writer.writerows(l)
520    ###)
521
522    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
523    def removeDeletedDocIds(self, max=1000):
524        """
525        remove deleted docids from repository commit after max
526        """
527        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
528        repository = getToolByName(self, 'portal_repository')
529        pxtool = getToolByName(self, 'portal_proxies')
530        logger.info('start')
531        pxtool_infos = pxtool.getRevisionsUsed()
532        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
533
534        nb_revs = 0
535        docids_d = {} # all docids
536        unused_docids_d = {} # all docids that are unused
537        ids_unused_revs_docids = [] # ids for revs of unused docids
538        unused_ids = [] # ids for unused revs
539        total = 0
540        idlist = repository.objectIds()
541        to_delete = 0
542        found = False
543        for id in idlist:
544            docid, rev = repository._splitId(id)
545            if docid is None:
546                logger.info("invalid doc_id %s" % docid)
547                continue
548            nb_revs += 1
549            if not pxtool_infos.has_key(docid):
550                found = True
551                to_delete += 1
552                unused_ids.append(id)
553            elif not pxtool_infos[docid].has_key(rev):
554                found = True
555                to_delete += 1
556                unused_ids.append(id)
557            if found and not to_delete % max:
558                found = False
559                #import pdb;pdb.set_trace()
560                repository.manage_delObjects(unused_ids)
561                transaction.commit()
562                logger.info('removed %d total %d unused docids ' % (max,to_delete))
563        else:
564            if unused_ids:
565                repository.manage_delObjects(unused_ids)
566                transaction.commit()
567        logger.info('finished removing %d unused docids ' % (to_delete))
568    ###)
569
570    security.declarePublic('getCredential') ###(
571    def getCredential(self,student_id):
572        student_entry = getattr(self.portal_directories.students,student_id,None)
573        #import pdb;pdb.set_trace()
574        #if not self.isStaff():
575        #    mtool = self.portal_membership
576        #    member = mtool.getAuthenticatedMember()
577        #    logger = logging.getLogger('WAeUPTool.getCredential')
578        #    logger.info('%s tried to access password of %s' % (member,student_id))
579        #    return None
580        if student_entry is None:
581            return None
582        return getattr(student_entry,"password","not set")
583    ###)
584
585    security.declarePublic('checkPassword') ###(
586    def checkPassword(self,student_id,password):
587        student_entry = getattr(self.portal_directories.students,student_id,None)
588        if student_entry is None:
589            return False
590        return getattr(student_entry,"password","not set") == password
591    ###)
592
593    security.declarePublic('checkGenericPassword') ###(
594    def checkGenericPassword(self,member_id):
595        member_entry = getattr(self.portal_directories.members,member_id,None)
596        if member_entry is None:
597            return False
598        ltool = getToolByName(self, 'portal_layouts')
599        unsecure_words = ltool._getOb('members')['w__password'].check_words
600        password = getattr(member_entry,"password","not set")
601        is_unsecure = password in unsecure_words
602        if is_unsecure:
603            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
604            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
605        return is_unsecure
606    ###)
607
608    security.declareProtected(ModifyPortalContent,'editPassword') ###(
609    def editPassword(self,student_id,password):
610        "edit a student password"
611        student_entry = getattr(self.portal_directories.students,student_id,None)
612        if student_entry is None:
613            return
614        setattr(student_entry,'password',password)
615    ###)
616
617    security.declareProtected(ModifyPortalContent,'doCommit') ###(
618    def doCommit(self,logger=None):
619        "commit some transactions"
620        transaction.commit()
621    ###)
622
623    security.declarePublic('loadStudentFoto') ###(
624    def loadStudentFoto(self,student,filename,folder):
625        "return a student passport picture"
626        #import pdb;pdb.set_trace()
627        picture ="%s/import/%s/%s" % (i_home,folder,filename)
628        student_id = student.getId()
629        images_dir = getImagesDir(student_id)
630        if not os.path.exists(images_dir):
631            os.mkdir(images_dir)
632        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
633        for extension in ('.jpg','.JPG'):
634            fullname = "%(picture)s%(extension)s" % vars()
635            if os.path.exists(fullname):
636                copy2(fullname,image_name)
637                return "successfully copied passport picture"
638        return "passport picture not found: %s.jpg or .JPG" % picture
639    ###)
640
641
642    security.declareProtected(ModifyPortalContent,'createOne') ###(
643    def createOne(self,students_folder,student_brain,letter,commit=False):
644        sid = self.waeup_tool.generateStudentId(letter)
645        students_folder.invokeFactory('Student', sid)
646        student = getattr(students_folder,sid)
647        self.portal_workflow.doActionFor(student,'return')
648        student.manage_setLocalRoles(sid, ['Owner',])
649        matric_no = student_brain.matric_no
650        jamb_reg_no = student_brain.Entryregno
651        self.students_catalog.addRecord(id = sid,
652                                           matric_no = matric_no,
653                                           jamb_reg_no = jamb_reg_no,
654                                           sex = student_brain.Sex == "F",
655                                           name = "%s %s %s" % (student_brain.Firstname,
656                                                                student_brain.Middlename,
657                                                                student_brain.Lastname)
658                                        )
659        if commit:
660            transaction.commit()
661        return sid,jamb_reg_no
662    ###)
663
664    # Functions without docstring cannot be called from outside (via the URL),
665    # thus we can declare addStudent public.
666    security.declarePublic('addStudent') ###(
667    def addStudent(self,dict):
668        students_folder = self.portal_url.getPortalObject().campus.students
669        sid = self.waeup_tool.generateStudentId('?')
670        students_folder.invokeFactory('Student', sid)
671        student_obj = getattr(students_folder,sid)
672        f2t = StudentImport.field2types_student
673        #from pdb import set_trace; set_trace()
674        d = {}
675        #d['jamb_sex']  = 'M'
676        #if dict.get('sex'):
677        #    d['jamb_sex']  = 'F'
678
679        entry_session = dict.get('entry_session')
680        if entry_session == self.getSessionId()[0]:
681            wfaction = 'admit'
682            wft = 'wf_transition_admit'
683            password = None
684        else:
685            wfaction = 'return'
686            wft = 'wf_transition_return'
687            password = self.generatePassword()
688            self.makeStudentMember(sid,password)
689
690        for pt in f2t.keys():
691            student_obj.invokeFactory(pt,f2t[pt]['id'])
692            sub_obj = getattr(student_obj,f2t[pt]['id'])
693            sub_doc = sub_obj.getContent()
694            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
695            #d['Title'] = f2t[pt]['title']
696            for field in f2t[pt]['fields']:
697                d[field] = dict.get(field,'')
698            sub_doc.edit(mapping = d)
699            new_state = f2t[pt][wft]
700            if new_state != "remain":
701                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
702        self.portal_workflow.doActionFor(student_obj,wfaction)
703        student_obj.manage_setLocalRoles(sid, ['Owner',])
704        return sid,password
705    ###)
706
707    security.declarePublic('getCertificateBrain') ###(
708    def getCertificateBrain(self,cert_id):
709        "do it"
710        res = ZCatalog.searchResults(self.portal_catalog_real,
711                                {'portal_type':"Certificate",
712                                      'id': cert_id})
713        if res:
714            return res[0]
715        return None
716    ###)
717
718    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
719    def get_csv_filenames(self):
720        "do it"
721        files = [file for file in os.listdir("%s/import/" % (i_home))
722                 if file.endswith('.csv') and (file.find('imported') == -1 and
723                                               file.find('pending') == -1)]
724        return files
725    ###)
726
727    security.declarePublic('findStudentByMatricelNo') ###(
728    def findStudentByMatricelNo(self,matric_no):
729        "do it"
730        res = ZCatalog.searchResults(self.portal_catalog_real,
731                                {'portal_type':"StudentClearance",
732                                 'SearchableText': matric_no})
733        if res:
734            return res[0]
735        return None
736    ###)
737
738
739    security.declarePublic('getOfficerName') ###(
740    def getOfficerName(self,mid):
741        """get the real name of a member"""
742        membership = self.portal_membership
743        member_record = membership.getMemberById(mid)
744        return member_record.getProperty('fullname',None)
745    ###)
746
747
748    security.declarePublic('makeStudentMember') ###(
749    def makeStudentMember(self,sid,password='uNsEt'):
750        """make the student a member"""
751        membership = self.portal_membership
752        membership.addMember(sid,
753                             password ,
754                             roles=('Member',
755                                     'Student',
756                                     ),
757                             domains='',
758                             properties = {'memberareaCreationFlag': False,
759                                           'homeless': True},)
760        member = membership.getMemberById(sid)
761        self.portal_registration.afterAdd(member, sid, password, None)
762        #self.manage_setLocalRoles(sid, ['Owner',])
763    ###)
764
765    security.declareProtected(View,'makeStudentData') ###(
766    def makeStudentData(self,student_id,email=None,phone_nr=None):
767        "create Datastructure for a returning Student"
768        #import pdb;pdb.set_trace()
769        logger = logging.getLogger('WAeUPTool.makeStudentData')
770        students_folder = self.portal_url.getPortalObject().campus.students
771        #res = self.students_catalog(id=student_id)
772        #if res:
773        #    st = res[0]
774        #res = self.returning_import(matric_no = st.matric_no)
775        res = self.returning_import(id = student_id)
776        if res:
777            student = res[0]
778        else:
779            logger.info('Id %s not found in returning_import' % student_id)
780            return
781        logger.info('%s creates data structure' % student_id)
782        s_results = self.results_import(matric_no = student.matric_no)
783        if s_results:
784            lnr = self.getLevelFromResultsCosCode(s_results)
785            level = "%d00" % lnr
786            verdict,eligible = self.getVerdict(s_results[0].Verdict)
787            #if eligible:
788            #    level = "%d00" % (lnr + 1)
789        else:
790            logger.info('matric_no %s not found in results_import' % student.matric_no)
791            level = ''
792            verdict = ''
793        #student should not be allowed to perform this transition
794        #wftool = self.portal_workflow
795        #wftool.doActionFor(student,'return')
796        certcode_org = student.Coursemajorcode
797        certcode = makeCertificateCode(certcode_org)
798        certificate_brain = self.getCertificateBrain(certcode)
799        if not certificate_brain:
800            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
801            logger.info(em)
802        matric_no = student.matric_no
803        sid = student_id
804        student_obj = getattr(students_folder,sid)
805        if not getattr(student_obj,'application'):
806            student_obj.invokeFactory('StudentApplication','application')
807        application = student_obj.application
808        self.portal_workflow.doActionFor(application,'open',dest_container=application)
809        da = {'Title': 'Application Data'}
810        student_obj.invokeFactory('StudentPersonal','personal')
811        da['jamb_reg_no'] = student.Entryregno
812        em = self.getEntryMode(student.Entryregno)
813        da['entry_mode'] = em
814        personal = student_obj.personal
815        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
816        dp = {'Title': 'Personal Data'}
817        student_obj.invokeFactory('StudentClearance','clearance')
818        clearance = student_obj.clearance
819        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
820        dc = {'Title': 'Clearance/Eligibility Record'}
821        dc['matric_no'] = matric_no
822        state = student.State
823        lga = student.LGA
824        if state and lga:
825            lga =  state + ' / ' + lga
826        else:
827            lga = "None"
828        da['jamb_lga'] = dc['lga'] = lga
829        da['app_email'] = dp['email'] = email
830        da['app_mobile'] = dp['phone'] = phone_nr
831        dp['firstname'] = student.Firstname
832        dp['middlename'] = student.Middlename
833        dp['lastname'] = student.Lastname
834        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
835        da['jamb_sex'] = student.Sex
836        dp['sex'] = student.Sex == 'F'
837        dp['perm_address'] = student.Permanent_Address
838        application.getContent().edit(mapping=da)
839        self.portal_workflow.doActionFor(application,'close',dest_container=application)
840        personal.getContent().edit(mapping=dp)
841        clearance.getContent().edit(mapping=dc)
842        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
843        #
844        # Study Course
845        #
846        student_obj.invokeFactory('StudentStudyCourse','study_course')
847        studycourse = student_obj.study_course
848        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
849        dsc = {}
850        dsc['study_course'] = certcode
851        dsc['current_level'] = level
852        dsc['current_verdict'] = verdict
853        dsc['current_mode'] = em   #no longer used
854        dsc['current_session'] = '05'
855        studycourse.getContent().edit(mapping=dsc)
856        #
857        # Level
858        #
859        # l = getattr(studycourse,level,None)
860        # if l is None:
861        #     studycourse.invokeFactory('StudentStudyLevel', level)
862        #     l = getattr(studycourse, level)
863        #     self.portal_workflow.doActionFor(l,'open',dest_container=l)
864        #     l.getContent().edit(mapping={'Title': "Level %s" % level})
865        ###)
866
867    def init_timing(self): ###(
868        if self.with_timing:
869            if not hasattr(self,'_v_step_times'):
870                self._v_timer_count = 0
871                self._v_total = 0
872                self._v_step_times = {}
873                current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
874                self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
875            self.timer_step = 0
876            self.total_time = 0
877            self.elapse = time.time()
878            self.i_elapse = time.time()
879            self.c_elapse = time.clock()
880    ###)
881
882    def do_timing(self): ###(
883        if self.with_timing:
884            try:
885                raise 'dummy'
886            except:
887                frame = sys.exc_traceback.tb_frame.f_back
888                locals = frame.f_locals
889                globals = frame.f_globals
890                functionname = frame.f_code.co_name
891                filename = os.path.basename(frame.f_code.co_filename)
892                lineno = frame.f_lineno
893                mod_line = "%(functionname)s:%(lineno)s" % vars()
894            i_time = time.time() - self.i_elapse
895            td = {}
896            if self._v_step_times.has_key(mod_line):
897                a_time = self._v_step_times[mod_line]['a_time'] + i_time
898                td['a_time'] = a_time
899            else:
900                td['a_time'] = i_time
901            td['i_time'] = i_time
902            self._v_step_times[mod_line] = td
903            self.i_time = i_time
904            self.total_time += i_time
905            self.timer_step +=1
906            self.i_elapse = time.time()
907    ###)
908
909    security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
910    def print_timing(self):
911        if self.with_timing:
912            l = []
913            timer_count = self._v_timer_count + 1
914            mod_lines = self._v_step_times.keys()
915            mod_lines.sort(cmp,reverse=0)
916            for mod_line in mod_lines:
917                td = self._v_step_times[mod_line]
918                i_time = td['i_time']
919                a_time = td['a_time']/(self._v_timer_count + 1)
920                l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
921            total_time = self.total_time
922            total_avarage = self._v_total / timer_count
923            l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
924            print "\r\n".join(l)
925            out = open(self._v_timer_file,'a')
926            out.write("\r\n".join(l))
927            out.close()
928    ###)
929
930    security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
931    def get_timing_data(self):
932        if self.with_timing:
933            timer_count = self._v_timer_count + 1
934            results = {}
935            for k,d in self._v_step_times.items():
936                dd = {}
937                dd['a_time'] = d['a_time']/timer_count
938                dd['i_time'] = d['i_time']
939                dd['count'] = timer_count
940                results[k] = dd
941            dd = {}
942            dd['a_time'] = self._v_total / timer_count
943            dd['i_time'] = self.total_time
944            dd['count'] = timer_count
945            results["total"] = dd
946            return results
947    ###)
948
949    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
950    def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
951        "create Datastructure for an admitted Student"
952        #import pdb;pdb.set_trace()
953        logger = logging.getLogger('WAeUPTool.admitOneStudent')
954        self.with_timing = with_timing
955
956        if brain.screening_type in ('cest','sandwich',):
957            reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
958        else:
959            reg_no = brain.reg_no
960
961        #ignore argument entry_session
962        if not brain.entry_session:
963            logger.info('no entry_session for %s provided' % (reg_no))
964            return
965
966        if not hasattr(self,"_v_certificates"):
967            self._v_certificates = self.getCertificatesDict()
968        students_folder = self.portal_url.getPortalObject().campus.students
969
970        res = self.students_catalog(jamb_reg_no = reg_no)
971        if res:
972            logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
973            return
974        if brain.status != "admitted":
975            logger.info('status of %s is %s' % (reg_no,brain.status))
976            return
977        pin_parts = brain.pin.split('-')
978        if pin_parts and len(pin_parts) != 3:
979            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
980            return
981        if not brain.course_admitted:
982            logger.info('no course_admitted provided for %s' % (reg_no))
983            return           
984        if brain.course_admitted not in self._v_certificates:
985            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
986            return
987        if brain.sex not in (True,False):
988            logger.info('sex of %s not available' % (reg_no))
989            return
990        self.init_timing()
991        student_id = self.generateStudentId('?')
992        students_folder.invokeFactory('Student', student_id)
993        student_object = getattr(students_folder,student_id)
994        self.do_timing()
995        if pin_password:
996            password = pin_parts[2]
997            self.makeStudentMember(student_id,password = password)
998        student_object.manage_setLocalRoles(student_id, ['Owner',])
999        self.do_timing()
1000        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
1001        #
1002        # application
1003        #
1004        student_object.invokeFactory('StudentApplication','application')
1005        application = student_object.application
1006        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
1007        #self.do_timing()
1008        da = {'Title': 'Application Data'}
1009        da['jamb_reg_no'] = reg_no
1010
1011        sex = 'M'
1012        if brain.sex:
1013            sex = 'F'
1014        da['jamb_sex'] = sex
1015        da['jamb_age'] = brain.jamb_age
1016        da['app_reg_pin'] = brain.pin
1017        da['jamb_lga'] = brain.jamb_lga
1018        da['jamb_state'] = brain.jamb_state
1019        da['jamb_score'] = brain.aggregate
1020        da['app_email'] = brain.email
1021        da['app_mobile'] = brain.phone
1022
1023        # entry_mode cannot be retrieved from the certtificate
1024        # because ug_ft has two different entry modes
1025                       
1026        if brain.entry_mode:                      # does not happen because import_application
1027            da['entry_mode'] = brain.entry_mode   # schema has no field entry_mode
1028        elif brain.screening_type == 'pume': 
1029            da['entry_mode'] = 'ume_ft' 
1030        elif brain.screening_type == 'pde': 
1031            da['entry_mode'] = 'de_ft' 
1032        else:
1033            da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1034       
1035        #elif brain.screening_type == 'pce': 
1036        #   da['entry_mode'] = 'pce' 
1037        #elif brain.screening_type == 'prence': 
1038        #   da['entry_mode'] = 'prence' 
1039        #else: 
1040        #   da['entry_mode'] = '' 
1041     
1042
1043        #da['entry_session'] = entry_session
1044        da['entry_session'] = brain.entry_session
1045        da['jamb_lastname'] = brain.lastname
1046        da['jamb_middlename'] = brain.middlenames   # different field names!
1047        da['jamb_firstname'] = brain.firstname
1048        da['screening_application_date'] = brain.application_date
1049        da['date_of_birth'] = brain.date_of_birth
1050        da['jamb_first_cos'] = brain.course1
1051        da['jamb_second_cos'] = brain.course2
1052        da['course3'] = brain.course3
1053        da['screening_type'] = brain.screening_type
1054        da['screening_score'] = brain.screening_score
1055        da['screening_date'] = brain.screening_date
1056        da['hq_type'] = brain.hq_type
1057        da['hq_grade'] = brain.hq_grade
1058        da['aos'] = brain.aos
1059
1060        application.getContent().edit(mapping=da)
1061        self.do_timing()
1062        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
1063        #
1064        # personal
1065        #
1066        student_object.invokeFactory('StudentPersonal','personal')
1067        personal = student_object.personal
1068        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1069        #self.do_timing()
1070        dp = {'Title': 'Personal Data'}
1071        dp['sex'] = brain.sex
1072        dp['email'] = brain.email
1073        dp['phone'] = brain.phone
1074        dp['lastname'] = brain.lastname
1075        dp['middlename'] = brain.middlenames   # different field names!
1076        dp['firstname'] = brain.firstname
1077        personal.getContent().edit(mapping=dp)
1078        self.do_timing()
1079        #
1080        # clearance
1081        #
1082        student_object.invokeFactory('StudentClearance','clearance')
1083        clearance = student_object.clearance
1084        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1085        dc = {'Title': 'Clearance/Eligibility Record'}
1086        dc['lga'] = brain.lga
1087        dc['birthday'] = brain.date_of_birth
1088        clearance.getContent().edit(mapping=dc)
1089        self.do_timing()
1090        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1091        #
1092        # study Course
1093        #
1094        student_object.invokeFactory('StudentStudyCourse','study_course')
1095        studycourse = student_object.study_course
1096        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1097        #self.do_timing()
1098        dsc = {}
1099        dsc['study_course'] = brain.course_admitted
1100        dsc['current_verdict'] = ''
1101        dsc['current_mode'] = da['entry_mode'] # no longer used
1102        if da['entry_mode'].startswith('de'):
1103            dsc['current_level'] = '200'
1104        elif da['entry_mode'].startswith('pre'):
1105            dsc['current_level'] = '000'
1106        else:
1107            dsc['current_level'] = '100'
1108        dsc['current_session'] = brain.entry_session
1109        studycourse.getContent().edit(mapping=dsc)
1110        self.do_timing()
1111        #
1112        # payments folder
1113        student_object.invokeFactory('PaymentsFolder','payments')
1114        payments = getattr(student_object,'payments')
1115        #self.do_timing()
1116        dpay = {}
1117        dpay['Title'] = 'Payments'
1118        payments.getContent().edit(mapping=dpay)
1119        self.portal_workflow.doActionFor(payments,'open')
1120        self.do_timing()
1121        #
1122        # passport foto
1123        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1124                                                              brain.screening_type,
1125                                                              brain.reg_no)
1126        images_dir = getImagesDir(student_id)
1127        #images_dir = os.path.join("%s" % images_base,student_id)
1128        letter_dir,student_dir = os.path.split(images_dir)
1129        if not os.path.exists(letter_dir):
1130            os.mkdir(letter_dir)
1131        if not os.path.exists(images_dir):
1132            os.mkdir(images_dir)
1133        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1134        if os.path.exists(app_picture):
1135            copy2(app_picture,image_name)
1136        else:
1137            logger.info('passport of %s/%s not found: %s' % (student_id,
1138                                                             brain.reg_no,
1139                                                             app_picture))
1140
1141        self.do_timing()
1142        self.print_timing()
1143        if with_timing:
1144            self.timer_step = 0
1145            self._v_timer_count += 1
1146            self._v_total += self.total_time
1147        return student_id
1148    ###)
1149
1150    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1151    def makeStudentLevel(self,student_id):
1152        "create the StudyLevel for a returning Student"
1153        #import pdb;pdb.set_trace()
1154        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1155        students_folder = self.portal_url.getPortalObject().campus.students
1156        res = self.students_catalog(id=student_id)
1157        if res:
1158            st = res[0]
1159        course = st.course
1160        matric_no = st.matric_no
1161        level = st.level
1162        res = self.results_import(matric_no = matric_no)
1163        if res:
1164            results = res
1165        logger.info('%s creating Level %s' % (student_id,level))
1166        #
1167        # Level
1168        #
1169        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1170        studycourse = getattr(student_obj,"study_course",None)
1171        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1172        l = getattr(studycourse,level,None)
1173        if l is None:
1174            studycourse.invokeFactory('StudentStudyLevel', level)
1175            l = getattr(studycourse, level)
1176            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1177            l.getContent().edit(mapping={'Title': "Level %s" % level})
1178        ###)
1179
1180    security.declarePublic('getHallInfo') ###(
1181    def getHallInfo(self,bed):
1182        """return Hall Info"""
1183        info = {}
1184        bedsplit = bed.split('_')
1185        if len(bedsplit) == 4:
1186            hall,block,room,letter = bed.split('_')
1187        else:
1188            info['maintenance_code'] = 'None'
1189            return info
1190        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1191        if res and len(res) == 1:
1192            hall_brain = res[0]
1193            hall_doc = hall_brain.getObject().getContent()
1194        else:
1195            return info
1196        info['hall_title'] = hall_brain.Title
1197        info['maintenance_code'] = hall_doc.maintenance_code
1198        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1199        batch_doc = None
1200        for brain in res:
1201            if brain.id.startswith(info['maintenance_code']):
1202                batch_doc = brain.getObject().getContent()
1203                break
1204        if batch_doc is None:
1205            info['maintenance_fee'] = ''
1206        else:
1207            info['maintenance_fee'] = batch_doc.cost
1208        return info
1209    ###)
1210
1211    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1212    def removePictureFolder(self,student_id):
1213        """remove picture_folder by renaming it"""
1214        picture_path = getImagesDir(student_id)
1215        dest_path = os.path.join("%s" % images_base,'removed',student_id)
1216        dest_path = dest_path + "_removed"
1217        if os.path.exists(dest_path) or not os.path.exists(picture_path):
1218            return False
1219        os.rename(picture_path,dest_path)
1220        return True
1221    ###)
1222
1223    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1224    def restorePictureFolder(self,student_id):
1225        """restore picture_folder by renaming it"""
1226        picture_path = getImagesDir(student_id)
1227        orig_path = os.path.join("%s" % images_base,'removed',student_id)
1228        orig_path = orig_path + "_removed"
1229        if os.path.exists(picture_path) or not os.path.exists(orig_path):
1230            return False       
1231        os.rename(picture_path + "_removed",picture_path)
1232        return True
1233    ###)
1234
1235    security.declarePublic('picturesExist') ###(
1236    def picturesExist(self, ids,student_id=None):
1237        """check if pictures exist in the filesystem"""
1238        if student_id is None:
1239            student_id = self.getStudentId()
1240        if student_id is None:
1241            return False
1242        picture_path = getImagesDir(student_id)
1243        #picture_path = os.path.join(images_base,student_id)
1244        if not os.path.exists(picture_path):
1245            return False
1246        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1247        return set(ids).issubset(set(pictures))
1248    ###)
1249   
1250        security.declarePublic('picturePathExists') ###(
1251    def picturePathExists(self, student_id=None):
1252        """check if picture path exists in the filesystem"""
1253        if student_id is None:
1254            student_id = self.getStudentId()
1255        if student_id is None:
1256            return False
1257        picture_path = getImagesDir(student_id)
1258        if os.path.exists(picture_path):
1259            return True
1260        return False
1261    ###)
1262   
1263   
1264    security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1265    def removeUnusedImageFolders(self):
1266        """check if an unused image folders exists in the filesystem"""
1267        mtool = self.portal_membership
1268        member = mtool.getAuthenticatedMember()
1269        member_id = str(member)
1270        logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1271        abc = os.listdir(images_base)
1272        ifolders = []
1273        for i in abc:
1274            picture_path = os.path.join(images_base,i)
1275            ifolders.extend(os.listdir(picture_path))
1276        unused_ids = []
1277        for id in ifolders:
1278            res = self.students_catalog(id=id)
1279            if not res:
1280                unused_ids.append(id)
1281                #import pdb;pdb.set_trace()   
1282                if not id.endswith('removed'):
1283                    removed = self.waeup_tool.removePictureFolder(id) 
1284                    if removed:
1285                        logger.info('%s: image folder %s successfully removed' % (member_id,id))
1286                    else:
1287                        logger.info('%s: image folder %s could not be removed' % (member_id,id))
1288        return
1289       
1290    ###)   
1291
1292    security.declarePublic('picturesList') ###(
1293    def picturesList(self):
1294        """check if pictures exist in the filesystem"""
1295        path = 'images'
1296        student_id = self.getStudentId()
1297        #picture_path = os.path.join(i_home,path,student_id)
1298        picture_path = getImagesDir(student_id)
1299        if not os.path.exists(picture_path):
1300            return []
1301        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1302    ###)
1303
1304    security.declarePublic('showFsPicture') ###(
1305    def showFsPicture(self,path):
1306        """return a picture from the filesystem"""
1307        #picture_path = os.path.join(i_home,path)
1308        picture_path = os.path.join(images_base,path)
1309        response = self.REQUEST.RESPONSE
1310        #import pdb;pdb.set_trace()
1311        registry = getToolByName(self, 'mimetypes_registry')
1312        mimetype = str(registry.lookupExtension(path.lower()) or
1313                    registry.lookupExtension('file.bin'))
1314        if os.path.exists(picture_path):
1315            response.setHeader('Content-type',mimetype)
1316            return open(picture_path).read()
1317        picture_path = os.path.join(i_home,'import',path)
1318        if os.path.exists(picture_path):
1319            return open(picture_path).read()
1320    ###)
1321
1322    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1323    def deleteAllCourses(self,department="All"):
1324        ''' delete the courses'''
1325        pm = self.portal_membership
1326        member = pm.getAuthenticatedMember()
1327
1328        if str(member) not in ("henrik","joachim"):
1329            return "not possible"
1330        if department == "All":
1331            res = self.portal_catalog({'meta_type': 'Department'})
1332        if len(res) < 1:
1333            return "No Departments found"
1334
1335        deleted = []
1336        for dep in res:
1337            cf = dep.getObject().courses
1338            if cf:
1339                cf.manage_delObjects(ids=cf.objectIds())
1340                deleted.append("deleted Courses in %s" % dep.getId)
1341        return "\r".join(deleted)
1342    ###)
1343
1344    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1345    def getLogfileLines(self,filename="event.log",numlines=20):
1346        """Get last NUMLINES lines of logfile FILENAME.
1347
1348        Return last lines' of a file in the instances logfile directory as
1349        a list. The number of returned lines equals `numlines' or less. If
1350        less than `numlines' lines are available, the whole file ist
1351        returned. If the file can not be opened or some other error
1352        occurs, empty list is returend.
1353        """
1354        result = []
1355        lines_hit = 0
1356
1357        # We only handle files in instances' log directory...
1358        logpath = os.path.join(i_home, "log")
1359        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1360        if not filename.startswith( logpath ):
1361            # Attempt to access file outside log-dir...
1362            return []
1363
1364        try:
1365            fd = file( filename, "rb" )
1366        except IOError:
1367            return []
1368        if not fd:
1369            return []
1370
1371        if os.linesep == None:
1372            linesep = '\n'
1373        else:
1374            linesep = os.linesep
1375
1376        # Try to find 'numlines' times a lineseparator, searching from end
1377        # and moving to the beginning of file...
1378        fd.seek( 0, 2) # Move to end of file...
1379        while lines_hit < numlines:
1380            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1381                                          # one step forward...
1382                lines_hit += 1
1383            try:
1384                fd.seek( -2, 1) # Go two bytes back from current pos...
1385            except IOError:
1386                # We cannot go back two bytes. Maybe the file is too small...
1387                break
1388        fd.seek(2,1)
1389
1390        # Read all lines from current position...
1391        result = fd.readlines()
1392        # Remove line endings...
1393        result = [x.strip() for x in result]
1394        fd.close()
1395        return result
1396    ###)
1397
1398    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1399    def getCallbacksFromLog(self,filename):
1400        """fix Online Payment Transactions from Z2.log entries"""
1401        import transaction
1402        import random
1403        from cgi import parse_qs
1404        from urlparse import urlparse
1405        #from pdb import set_trace
1406        wftool = self.portal_workflow
1407        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1408        students_folder = self.portal_url.getPortalObject().campus.students
1409        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1410        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1411        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1412        data = re.compile(s)
1413        start = True
1414        tr_count = 1
1415        total = 0
1416        #name = 'pume_results'
1417        #name = 'epaymentsuccessful_z2log2'
1418        name = filename
1419        no_import = []
1420        imported = []
1421        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1422        try:
1423            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1424        except:
1425            logger.error('Error reading %s' % name)
1426            return
1427        tas = []
1428        for line in transactions:
1429            dict = {}
1430            items = data.search(line)
1431            dict['idict'] = idict = items.groupdict()
1432            #print idict
1433            #from pdb import set_trace;set_trace()
1434            urlparsed = urlparse(idict['get'][4:])
1435            #print urlparsed
1436            path = urlparsed[2].split('/')
1437            dict['student_id'] = student_id = path[8]
1438            dict['payment_id'] = payment_id = path[10]
1439            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1440            tas.append(dict)
1441            tr_count += 1
1442        return tas
1443    ###)
1444
1445    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1446    def importOnlinePaymentTransactions(self):
1447        """load Online Payment Transactions from CSV values"""
1448        import transaction
1449        import random
1450        #from pdb import set_trace
1451        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1452        opt = self.online_payments_import
1453        students_folder = self.portal_url.getPortalObject().campus.students
1454        start = True
1455        tr_count = 1
1456        total = 0
1457        #name = 'pume_results'
1458        name = 'OnlineTransactions'
1459        no_import = []
1460        imported = []
1461        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1462        try:
1463            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1464        except:
1465            logger.error('Error reading %s.csv' % name)
1466            return
1467        for pay_transaction in transactions:
1468            if start:
1469                start = False
1470                logger.info('Start loading from %s.csv' % name)
1471                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1472                no_import.append('%s,"Error"' % s)
1473                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1474                format_error = format + ',"%(Error)s"'
1475            data = {}
1476
1477            # format of the first file sent by Tayo
1478            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1479            #data['student_id'] = student_id = pay_transaction['Payer ID']
1480            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1481            #data['response_code'] = response_code = pay_transaction['Resp Code']
1482            #data['amount'] = amount = pay_transaction['Amount']
1483
1484            # format of the second file sent by Tayo
1485            #data['datetime'] = date = 0
1486            #data['student_id'] = student_id = pay_transaction['Payer ID']
1487            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1488            #data['response_code'] = response_code = '00'
1489            #data['amount'] = amount = pay_transaction['Amount']
1490
1491            # format of the third file sent by Kehinde
1492            data['datetime'] = date = 0
1493            data['student_id'] = student_id = pay_transaction['customer_id']
1494            data['order_id'] = order_id = pay_transaction['merchant_reference']
1495            data['response_code'] = response_code = '00'
1496            data['amount'] = amount = pay_transaction['Amount']
1497
1498            dup = False
1499            if response_code == "12":
1500                continue
1501            try:
1502                opt.addRecord(**data)
1503            except ValueError:
1504                dup = True
1505            #from pdb import set_trace;set_trace()
1506            if dup:
1507                if response_code == "00":
1508                    try:
1509                        opt.modifyRecord(**data)
1510                    except:
1511                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1512                        continue
1513                else:
1514                    pay_transaction['Error'] = "Duplicate order_id"
1515                    no_import.append( format_error % pay_transaction)
1516                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1517                    continue
1518            tr_count += 1
1519            if tr_count > 1000:
1520                if len(no_import) > 0:
1521                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1522                             '\n'.join(no_import) + '\n')
1523                    no_import = []
1524                em = '%d transactions committed\n' % (tr_count)
1525                transaction.commit()
1526                regs = []
1527                logger.info(em)
1528                total += tr_count
1529                tr_count = 0
1530        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1531                                                '\n'.join(no_import))
1532        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1533    ###)
1534
1535    security.declareProtected(ModifyPortalContent,"importData")###(
1536    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1537        """load data from CSV values"""
1538        import transaction
1539        import random
1540        students_folder = self.portal_url.getPortalObject().campus.students
1541        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1542        pending_only = False
1543        pend_str = '--'
1544        elapse = time.time()
1545        #
1546        # preparations
1547        #
1548        if filename == pend_str:
1549            pending_only = True
1550        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1551        importer = eval("%sImport" % importer_name)(self)
1552        logger = importer.logger
1553        if importer.init_errors:
1554            logger.info(importer.init_errors)
1555            return importer.init_errors
1556        member = importer.member
1557        #current = importer.current
1558        import_date = importer.import_date
1559        #
1560        # not_imported
1561        #
1562        info = importer.info
1563        data_keys = importer.data_keys
1564        csv_keys = importer.csv_keys
1565        #csv_keys.extend(info.keys())
1566        headline_mapping = dict((k,k) for k in csv_keys)
1567        #
1568        # pending
1569        #
1570        pending_path = importer.pending_path
1571        pending_tmp = importer.pending_tmp
1572        pending_backup = importer.pending_backup
1573        pending_fn = importer.pending_fn
1574        imported_path = importer.imported_path
1575        imported_fn = importer.imported_fn
1576        commit_after = importer.commit_after
1577        pending = []
1578        pending_digests = []
1579        #total_added_to_pending = 0
1580        if not pending_only:
1581            pending,pending_digests = importer.makeIdLists()
1582            pending_at_start = len(pending)
1583        datafile = open(pending_tmp,"w")
1584        pending_csv_writer = csv.DictWriter(datafile,
1585                                                    csv_keys,
1586                                                    extrasaction='ignore')
1587        pending_csv_writer.writerow(headline_mapping)
1588        datafile.close()
1589        #
1590        # imported
1591        #
1592        if not os.path.exists(imported_path):
1593            datafile = open(imported_path,"w")
1594            imported_csv_writer = csv.DictWriter(datafile,
1595                                                 csv_keys,
1596                                                 extrasaction='ignore')
1597            imported_csv_writer.writerow(headline_mapping)
1598            datafile.close()
1599        start = True
1600        tr_count = 0
1601        total = 0
1602        total_added_to_imported = 0
1603        total_pending = 0
1604        import_source_done = ""
1605        if pending_only:
1606            import_source_path = pending_path
1607        else:
1608            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1609            import_source_done = "%s/import/%s.done" % (i_home,filename)
1610        if not os.path.exists(import_source_path):
1611            fn = os.path.split(import_source_path)[1]
1612            em = 'no import file %(fn)s' % vars()
1613            return em
1614        import_source_fn = os.path.split(import_source_path)[1]
1615        if not pending_only:
1616            info['imported_from'] = import_source_fn
1617        headline = csv.reader(open(import_source_path,"rb")).next()
1618        if "import_mode" not in headline:
1619            msg = 'import_mode must be in heading'
1620            return msg
1621        invalid_keys = importer.checkHeadline(headline)
1622        if invalid_keys:
1623            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1624
1625        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1626                                                         or k.strip() in info.keys())]
1627        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1628        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1629        # if diff2schema and diff2schema != set(['id',]):
1630        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1631        #     return msg
1632        #
1633        # start importing
1634        #
1635        try:
1636            reader = csv.DictReader(open(import_source_path,"rb"))
1637        except:
1638            msg = 'Error reading %s.csv' % filename
1639            logger.error(msg)
1640            return msg
1641        items = [item for item in reader]
1642        total_to_import = len(items)
1643        tti_float = float(total_to_import)
1644        if pending_only:
1645            pending_at_start = total_to_import
1646        count = 0
1647        imported = []
1648        old_commit_count = 0
1649        error_count = imported_count = 0
1650        already_in = 0
1651        for record in items:
1652            item = {}
1653            empty_value_keys = []
1654            for k,v in record.items():
1655                if k is None:
1656                    continue
1657                if v:
1658                    if v == EMPTY:
1659                        empty_value_keys += k,
1660                        v = ''
1661                    item[k.strip()] = v.strip()
1662            count += 1
1663            if start:
1664                start = False
1665                adapters = [MappingStorageAdapter(importer.schema, item)]
1666                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1667            dm = DataModel(item, adapters,context=self)
1668            ds = DataStructure(data=item,datamodel=dm)
1669            error_string = ""
1670            total += 1
1671            import_mode = item.get('import_mode','')
1672            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1673            if import_method is None:
1674                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1675            elif (import_mode in importer.required_modes and
1676                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1677                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1678                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1679            else:
1680                for k in import_keys:
1681                    if k not in item.keys() or k not in importer.validators.keys():
1682                        continue
1683                    if not importer.validators[k](ds,mode=import_mode):
1684                        error_string += ' ++ '
1685                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1686                                                                           ds.getErrorMapping(k)))
1687            if error_string:
1688                error = error_string
1689                id = ''
1690                mapping = item
1691            else:
1692                temp_item = item.copy()
1693                temp_item.update(dm)
1694                results = import_method(temp_item)
1695                id = results[0]
1696                error = results[1]
1697                mapping = results[2]
1698            #
1699            # restore some values
1700            #
1701            for k in empty_value_keys:
1702                mapping[k] = EMPTY
1703            if mapping.has_key('sex'):
1704                #import pdb;pdb.set_trace()
1705                if mapping['sex'] == True:
1706                    mapping['sex'] = 'F'
1707                elif mapping['sex'] == False:
1708                    mapping['sex'] = 'M'
1709            if pending_only:
1710                info['imported_from'] = item['imported_from']
1711            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1712            info['error'] = error
1713            info['import_record_no'] = count + 1
1714            mapping.update(info)
1715            log_list = []
1716            if error:
1717                error_count += 1
1718                digest = makeDigest(mapping,data_keys)
1719                if digest not in pending_digests:
1720                    pending_digests += digest,
1721                    pending.append(mapping)
1722                    if not pending_only:
1723                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1724                else:
1725                    already_in += 1
1726                    pass
1727            else:
1728                imported_count += 1
1729                imported += mapping,
1730                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1731            if log_list:
1732                time_till_now = time.time() - elapse
1733                percent_finished = (error_count + imported_count)/tti_float*100
1734                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1735                logger.info(' '.join(log_list))
1736            finished = count > total_to_import - 1
1737            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1738            if must_commit:
1739                old_commit_count = imported_count
1740
1741            if must_commit or finished:
1742                if len(imported):
1743                    transaction.commit()
1744                    datafile = open(imported_path,"a")
1745                    writer = csv.DictWriter(datafile,
1746                                            csv_keys,
1747                                            extrasaction='ignore')
1748                    writer.writerows(imported)
1749                    datafile.close()
1750                    total_added_to_imported += len(imported)
1751                    imported = []
1752                if len(pending) > 0:
1753                    datafile = open(pending_tmp,"a")
1754                    writer = csv.DictWriter(datafile,
1755                                            csv_keys,
1756                                            extrasaction='ignore')
1757                    writer.writerows(pending)
1758                    datafile.close()
1759                    total_pending += len(pending)
1760                    #total_added_to_pending += len(pending)
1761                    pending = []
1762                if not finished:
1763                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1764                    logger.info(msg)
1765        elapse = time.time() - elapse
1766        if os.path.exists(pending_path):
1767            copy2(pending_path,pending_backup)
1768        copy2(pending_tmp,pending_path)
1769        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1770        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1771        if pending_only:
1772            removed_pending = pending_at_start - total_pending
1773            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1774        else:
1775            added_pending = total_pending - pending_at_start
1776            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1777        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1778        logger.info(msg)
1779        if import_source_done:
1780            copy(import_source_path,import_source_done)
1781            os.remove(import_source_path)
1782            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1783            if upload is not None:
1784                upload_doc = upload.getContent()
1785                mapping = {}
1786                #mapping['import_date'] = DateTime.DateTime()
1787                mapping['import_date'] = import_date
1788                mapping['imported_by'] = importer.imported_by
1789                mapping['import_message'] = msg
1790                upload_doc.edit(mapping = mapping)
1791        os.remove(pending_tmp)
1792        return msg
1793    ###)
1794
1795    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1796    def moveImagesToFS(self,student_id="O738726"):
1797        "move the images to the filesystem"
1798        images_dir = getImagesDir(student_id)
1799        #images_dir = os.path.join("%s" % images_base,student_id)
1800        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1801        stool = getToolByName(self, 'portal_schemas')
1802        schemas = ['student_application',
1803                   'student_clearance',
1804                   ]
1805        created = False
1806        for schema_id in schemas:
1807            schema = stool._getOb(schema_id)
1808            object = getattr(student_folder,schema_id[len('student_'):],None)
1809            if object is None:
1810                continue
1811            doc = object.getContent()
1812            for key in schema.keys():
1813                if schema[key].meta_type != "CPS Image Field":
1814                    continue
1815                #import pdb;pdb.set_trace()
1816                image = getattr(doc,key,None)
1817                if not image or not hasattr(image,"data"):
1818                    continue
1819                if not created:
1820                    if not os.path.exists(images_dir):
1821                        os.mkdir(images_dir)
1822                    created = True
1823                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1824                open(filename,"wb").write(str(image.data))
1825    ###)
1826
1827    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1828    def movePassportToFS(self,student_id="O738726"):
1829        "move the passports to the filesystem"
1830        images_dir = os.path.join("%s" % i_home,'passports')
1831        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1832        stool = getToolByName(self, 'portal_schemas')
1833        schemas = ['student_application',
1834                   #'student_clearance',
1835                   ]
1836        created = False
1837        for schema_id in schemas:
1838            schema = stool._getOb(schema_id)
1839            object = getattr(student_folder,schema_id[len('student_'):],None)
1840            if object is None:
1841                continue
1842            doc = object.getContent()
1843            for key in schema.keys():
1844                if schema[key].meta_type != "CPS Image Field":
1845                    continue
1846                #import pdb;pdb.set_trace()
1847                image = getattr(doc,key)
1848                if not hasattr(image,"data"):
1849                    continue
1850                if not created:
1851                    if not os.path.exists(images_dir):
1852                        os.mkdir(images_dir)
1853                    created = True
1854                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1855                open(filename,"wb").write(str(image.data))
1856    ###)
1857   
1858   
1859    security.declarePublic('getConfigParams')
1860    def getConfigParams(self,conf_id="configuration"):
1861        conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
1862        conf_obj = conf.getContent()
1863        stool = getToolByName(self, 'portal_schemas')
1864        schema = stool._getOb('configuration')
1865        d = {}
1866        for key in schema.keys():
1867            d[key] = getattr(conf_obj,key,None)
1868        return d
1869
1870
1871InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.