source: WAeUP_SRP/trunk/WAeUPTool.py @ 3841

Last change on this file since 3841 was 3840, checked in by Henrik Bettermann, 16 years ago

some security checks
  • Property svn:keywords set to Id
File size: 75.5 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3840 2009-01-07 07:09:15Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108    ###)
109
110    security.declareProtected(View,'updateRoleMappingsFor') ###(
111    def updateRoleMappingsFor(self,wf_definition,ob):
112        "do so for public"
113        wf_def = getattr(self.portal_workflow,wf_definition)
114        wf_def.updateRoleMappingsFor(ob)
115    ###)
116
117    security.declareProtected(View,'getStatesLgas') ###(
118    def getStatesLgas(self):
119        """return lga info"""
120        voc = getattr(self.portal_vocabularies,'local_gov_areas')
121        states = []
122        lgas  = []
123        d = {}
124        wd = {}
125        for k,v in voc.items():
126            parts = v.split(' / ')
127            if len(parts) == 1:
128                state = parts[0].lower()
129                lga = ""
130            elif len(parts) == 2:
131                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133            else:
134                continue
135            if state not in states:
136                states.append(state)
137            if lga not in lgas:
138                lgas.append(lga)
139            words = re.split('[^a-zA-Z0-9/]',k)
140            words.sort()
141            wd[k] = words
142            d[k] = v
143        mapping = {}
144        mapping['word_dict'] = wd
145        mapping['lga_dict'] = d
146        mapping['states'] = states
147        mapping['lgas'] = lgas
148        return mapping
149    ###)
150
151    security.declareProtected(View,'findLga') ###(
152    def findLga(self,words,words_dict):
153        words = re.split('[^a-zA-Z0-9/]',words)
154        lga_words = []
155        for word in words:
156            if word:
157                lga_words += word.strip().lower(),
158        lga_words.sort()
159        state_lga = ''
160        while not state_lga:
161            for k,l in words_dict.items():
162                if lga_words == l:
163                    state_lga = k
164                    break
165            break
166        return state_lga
167    ###)
168    security.declareProtected(View,'getAccessInfo') ###(
169    def getAccessInfo(self,context):
170        "return a dict with access_info"
171        logger = logging.getLogger('WAeUPTool.getAccessInfo')
172        mtool = self.portal_membership
173        member = mtool.getAuthenticatedMember()
174        member_id = str(member)
175        info = {}
176        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178        is_staff = info['is_staff'] = not is_anonymous and not is_student
179        roles = member.getRolesInContext(context)
180        info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181                                                        "SectionManager" in roles or
182                                                        "Manager" in roles)
183        info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif not is_allowed and (not is_staff or  member_id != requested_id):
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378    ###)
379
380    def generateStudentId(self,letter): ###(
381        import random
382        logger = logging.getLogger('WAeUPTool.generateStudentId')
383        r = random
384        if letter == '?':
385            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
386        sid = "%c%d" % (letter,r.randint(99999,1000000))
387        students = self.portal_url.getPortalObject().campus.students
388        while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid):
389            if self.waeup_tool.picturePathExists(sid):
390                logger.info('picture path %s exists' % sid)
391            sid = "%c%d" % (letter,r.randint(99999,1000000))
392        return sid
393    ###)
394
395    def generatePassword(self,s=None): ###(
396        import random
397        r = random
398        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
399        if s is None:
400            s = 'abcdefghklmnpqrstuvwxy23456789'
401        pw = ''
402        while len(pw) < 6:
403            pw += r.choice(s)
404        return pw
405    ###)
406
407    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
408    def dumpSchoolfeePayments(self):
409        "dump paid schoolfees"
410        mtool = self.portal_membership
411        member = mtool.getAuthenticatedMember()
412        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
413        aq_student = self.students_catalog.evalAdvancedQuery
414        query = In('review_state',('schoolfee_paid',
415                                   'courses_registered',
416                                   'courses_validated',
417                                   ))
418        res = aq_student(query)
419        #import pdb;pdb.set_trace()
420        l = []
421        logger.info("start for %d" % len(res))
422        count = 1
423        log_after = 100
424        for student in res:
425            if not count % log_after:
426                logger.info("processed %d total %d" % (log_after,count))
427            count += 1
428            fee_dict =self.getSchoolFee(student)
429            fulltime = student.mode.endswith('_ft')
430            d = {}
431            d['student_id'] = student.id
432            d['name'] = student.name
433            d['amount'] = fee_dict.get(new_returning)
434            l += d,
435        csv_name = self.dumpListToCSV(l,'payments')
436        logger.info('%s dumped payments to %s' % (member,export_file))
437    ###)
438
439    security.declarePublic('dumpListToCSV') ###(
440    def dumpListToCSV(self,l,filename,fields=None):
441        """dump a list of dicts to a CSV file"""
442        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
443        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
444        if fields is None:
445            fields = l[0].keys()
446        headline = ','.join(fields)
447        out = open(export_file,"wb")
448        out.write(headline +'\n')
449        out.close()
450        out = open(export_file,"a")
451        csv_writer = csv.DictWriter(out,fields,)
452        csv_writer.writerows(l)
453        return export_file
454    ###)
455
456    security.declareProtected(ManagePortal, 'listMembers') ###(
457    def listMembers(self):
458        "list all members"
459        mtool = self.portal_membership
460        member = mtool.getAuthenticatedMember()
461        logger = logging.getLogger('WAeUPTool.listMembers')
462        if str(member) not in ('admin','joachim'):
463            logger.info('%s tried to list members' % (member))
464            return None
465        members = self.portal_directories.members
466        all = members.listEntryIdsAndTitles()
467        l = []
468        for user_id,name in all:
469            d = {}
470            d['user_id'] = user_id
471            d['name'] = name
472            d['pw'] = getattr(getattr(members,user_id),'password')
473            d['email'] = getattr(getattr(members,user_id),'email')
474            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
475            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
476            l += d,
477        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
478        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
479        logger.info('%s dumped member list to %s' % (member,export_file))
480        fields = l[0].keys()
481        headline = ','.join(fields)
482        out = open(export_file,"wb")
483        out.write(headline +'\n')
484        out.close()
485        out = open(export_file,"a")
486        csv_writer = csv.DictWriter(out,fields,)
487        csv_writer.writerows(l)
488    ###)
489
490    security.declareProtected(ManagePortal, 'listStudents') ###(
491    def listStudents(self):
492        "list all students"
493        mtool = self.portal_membership
494        member = mtool.getAuthenticatedMember()
495        logger = logging.getLogger('WAeUPTool.listStudents')
496        if str(member) not in ('admin','joachim'):
497            logger.info('%s tried to list students' % (member))
498            return None
499        students = self.portal_directories.students
500        all = students.listEntryIdsAndTitles()
501        l = []
502        for user_id,name in all:
503            d = {}
504            d['user_id'] = user_id
505            d['name'] = name
506            d['pw'] = getattr(getattr(students,user_id),'password')
507            d['email'] = getattr(getattr(students,user_id),'email')
508            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
509            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
510            l += d,
511        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
512        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
513        logger.info('%s dumped student list to %s' % (member,export_file))
514        fields = l[0].keys()
515        headline = ','.join(fields)
516        out = open(export_file,"wb")
517        out.write(headline +'\n')
518        out.close()
519        out = open(export_file,"a")
520        csv_writer = csv.DictWriter(out,fields,)
521        csv_writer.writerows(l)
522    ###)
523
524    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
525    def removeDeletedDocIds(self, max=1000):
526        """
527        remove deleted docids from repository commit after max
528        """
529        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
530        repository = getToolByName(self, 'portal_repository')
531        pxtool = getToolByName(self, 'portal_proxies')
532        logger.info('start')
533        pxtool_infos = pxtool.getRevisionsUsed()
534        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
535
536        nb_revs = 0
537        docids_d = {} # all docids
538        unused_docids_d = {} # all docids that are unused
539        ids_unused_revs_docids = [] # ids for revs of unused docids
540        unused_ids = [] # ids for unused revs
541        total = 0
542        idlist = repository.objectIds()
543        to_delete = 0
544        found = False
545        for id in idlist:
546            docid, rev = repository._splitId(id)
547            if docid is None:
548                logger.info("invalid doc_id %s" % docid)
549                continue
550            nb_revs += 1
551            if not pxtool_infos.has_key(docid):
552                found = True
553                to_delete += 1
554                unused_ids.append(id)
555            elif not pxtool_infos[docid].has_key(rev):
556                found = True
557                to_delete += 1
558                unused_ids.append(id)
559            if found and not to_delete % max:
560                found = False
561                #import pdb;pdb.set_trace()
562                repository.manage_delObjects(unused_ids)
563                transaction.commit()
564                logger.info('removed %d total %d unused docids ' % (max,to_delete))
565        else:
566            if unused_ids:
567                repository.manage_delObjects(unused_ids)
568                transaction.commit()
569        logger.info('finished removing %d unused docids ' % (to_delete))
570    ###)
571
572    security.declareProtected(View,'getCredential') ###(
573    def getCredential(self,student_id):
574        student_entry = getattr(self.portal_directories.students,student_id,None)
575        if not self.isStaff():
576            mtool = self.portal_membership
577            member = mtool.getAuthenticatedMember()
578            logger = logging.getLogger('WAeUPTool.getCredential')
579            logger.info('%s tried to access password of %s' % (member,student_id))
580            return None
581        if student_entry is None:
582            return None
583        return getattr(student_entry,"password","not set")
584    ###)
585
586    security.declarePublic('checkPassword') ###(
587    def checkPassword(self,student_id,password):
588        student_entry = getattr(self.portal_directories.students,student_id,None)
589        if student_entry is None:
590            return False
591        return getattr(student_entry,"password","not set") == password
592    ###)
593
594    security.declarePublic('checkGenericPassword') ###(
595    def checkGenericPassword(self,member_id):
596        member_entry = getattr(self.portal_directories.members,member_id,None)
597        if member_entry is None:
598            return False
599        ltool = getToolByName(self, 'portal_layouts')
600        unsecure_words = ltool._getOb('members')['w__password'].check_words
601        password = getattr(member_entry,"password","not set")
602        is_unsecure = password in unsecure_words
603        if is_unsecure:
604            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
605            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
606        return is_unsecure
607    ###)
608
609    security.declareProtected(ModifyPortalContent,'editPassword') ###(
610    def editPassword(self,student_id,password):
611        "edit a student password"
612        student_entry = getattr(self.portal_directories.students,student_id,None)
613        if student_entry is None:
614            return
615        setattr(student_entry,'password',password)
616    ###)
617
618    security.declareProtected(ModifyPortalContent,'doCommit') ###(
619    def doCommit(self,logger=None):
620        "commit some transactions"
621        transaction.commit()
622    ###)
623
624    security.declarePublic('loadStudentFoto') ###(
625    def loadStudentFoto(self,student,filename,folder):
626        "return a student passport picture"
627        #import pdb;pdb.set_trace()
628        picture ="%s/import/%s/%s" % (i_home,folder,filename)
629        student_id = student.getId()
630        images_dir = getImagesDir(student_id)
631        if not os.path.exists(images_dir):
632            os.mkdir(images_dir)
633        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
634        for extension in ('.jpg','.JPG'):
635            fullname = "%(picture)s%(extension)s" % vars()
636            if os.path.exists(fullname):
637                copy2(fullname,image_name)
638                return "successfully copied passport picture"
639        return "passport picture not found: %s.jpg or .JPG" % picture
640    ###)
641
642
643    security.declareProtected(ModifyPortalContent,'createOne') ###(
644    def createOne(self,students_folder,student_brain,letter,commit=False):
645        sid = self.waeup_tool.generateStudentId(letter)
646        students_folder.invokeFactory('Student', sid)
647        student = getattr(students_folder,sid)
648        self.portal_workflow.doActionFor(student,'return')
649        student.manage_setLocalRoles(sid, ['Owner',])
650        matric_no = student_brain.matric_no
651        jamb_reg_no = student_brain.Entryregno
652        self.students_catalog.addRecord(id = sid,
653                                           matric_no = matric_no,
654                                           jamb_reg_no = jamb_reg_no,
655                                           sex = student_brain.Sex == "F",
656                                           name = "%s %s %s" % (student_brain.Firstname,
657                                                                student_brain.Middlename,
658                                                                student_brain.Lastname)
659                                        )
660        if commit:
661            transaction.commit()
662        return sid,jamb_reg_no
663    ###)
664
665    # Functions without docstring cannot be called from outside (via the URL),
666    # thus we can declare addStudent public.
667    security.declarePublic('addStudent') ###(
668    def addStudent(self,dict):
669        students_folder = self.portal_url.getPortalObject().campus.students
670        sid = self.waeup_tool.generateStudentId('?')
671        students_folder.invokeFactory('Student', sid)
672        student_obj = getattr(students_folder,sid)
673        f2t = StudentImport.field2types_student
674        #from pdb import set_trace; set_trace()
675        d = {}
676        #d['jamb_sex']  = 'M'
677        #if dict.get('sex'):
678        #    d['jamb_sex']  = 'F'
679
680        entry_session = dict.get('entry_session')
681        if entry_session == self.getSessionId()[0]:
682            wfaction = 'admit'
683            wft = 'wf_transition_admit'
684            password = None
685        else:
686            wfaction = 'return'
687            wft = 'wf_transition_return'
688            password = self.generatePassword()
689            self.makeStudentMember(sid,password)
690
691        for pt in f2t.keys():
692            student_obj.invokeFactory(pt,f2t[pt]['id'])
693            sub_obj = getattr(student_obj,f2t[pt]['id'])
694            sub_doc = sub_obj.getContent()
695            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
696            #d['Title'] = f2t[pt]['title']
697            for field in f2t[pt]['fields']:
698                d[field] = dict.get(field,'')
699            sub_doc.edit(mapping = d)
700            new_state = f2t[pt][wft]
701            if new_state != "remain":
702                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
703        self.portal_workflow.doActionFor(student_obj,wfaction)
704        student_obj.manage_setLocalRoles(sid, ['Owner',])
705        return sid,password
706    ###)
707
708    security.declarePublic('getCertificateBrain') ###(
709    def getCertificateBrain(self,cert_id):
710        "do it"
711        res = ZCatalog.searchResults(self.portal_catalog_real,
712                                {'portal_type':"Certificate",
713                                      'id': cert_id})
714        if res:
715            return res[0]
716        return None
717    ###)
718
719    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
720    def get_csv_filenames(self):
721        "do it"
722        files = [file for file in os.listdir("%s/import/" % (i_home))
723                 if file.endswith('.csv') and (file.find('imported') == -1 and
724                                               file.find('pending') == -1)]
725        return files
726    ###)
727
728    security.declarePublic('findStudentByMatricelNo') ###(
729    def findStudentByMatricelNo(self,matric_no):
730        "do it"
731        res = ZCatalog.searchResults(self.portal_catalog_real,
732                                {'portal_type':"StudentClearance",
733                                 'SearchableText': matric_no})
734        if res:
735            return res[0]
736        return None
737    ###)
738
739
740    security.declarePublic('getOfficerName') ###(
741    def getOfficerName(self,mid):
742        """get the real name of a member"""
743        membership = self.portal_membership
744        member_record = membership.getMemberById(mid)
745        return member_record.getProperty('fullname',None)
746    ###)
747
748
749    security.declarePublic('makeStudentMember') ###(
750    def makeStudentMember(self,sid,password='uNsEt'):
751        """make the student a member"""
752        membership = self.portal_membership
753        membership.addMember(sid,
754                             password ,
755                             roles=('Member',
756                                     'Student',
757                                     ),
758                             domains='',
759                             properties = {'memberareaCreationFlag': False,
760                                           'homeless': True},)
761        member = membership.getMemberById(sid)
762        self.portal_registration.afterAdd(member, sid, password, None)
763        #self.manage_setLocalRoles(sid, ['Owner',])
764    ###)
765
766    security.declareProtected(View,'makeStudentData') ###(
767    def makeStudentData(self,student_id,email=None,phone_nr=None):
768        "create Datastructure for a returning Student"
769        #import pdb;pdb.set_trace()
770        logger = logging.getLogger('WAeUPTool.makeStudentData')
771        students_folder = self.portal_url.getPortalObject().campus.students
772        #res = self.students_catalog(id=student_id)
773        #if res:
774        #    st = res[0]
775        #res = self.returning_import(matric_no = st.matric_no)
776        res = self.returning_import(id = student_id)
777        if res:
778            student = res[0]
779        else:
780            logger.info('Id %s not found in returning_import' % student_id)
781            return
782        logger.info('%s creates data structure' % student_id)
783        s_results = self.results_import(matric_no = student.matric_no)
784        if s_results:
785            lnr = self.getLevelFromResultsCosCode(s_results)
786            level = "%d00" % lnr
787            verdict,eligible = self.getVerdict(s_results[0].Verdict)
788            #if eligible:
789            #    level = "%d00" % (lnr + 1)
790        else:
791            logger.info('matric_no %s not found in results_import' % student.matric_no)
792            level = ''
793            verdict = ''
794        #student should not be allowed to perform this transition
795        #wftool = self.portal_workflow
796        #wftool.doActionFor(student,'return')
797        certcode_org = student.Coursemajorcode
798        certcode = makeCertificateCode(certcode_org)
799        certificate_brain = self.getCertificateBrain(certcode)
800        if not certificate_brain:
801            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
802            logger.info(em)
803        matric_no = student.matric_no
804        sid = student_id
805        student_obj = getattr(students_folder,sid)
806        if not getattr(student_obj,'application'):
807            student_obj.invokeFactory('StudentApplication','application')
808        application = student_obj.application
809        self.portal_workflow.doActionFor(application,'open',dest_container=application)
810        da = {'Title': 'Application Data'}
811        student_obj.invokeFactory('StudentPersonal','personal')
812        da['jamb_reg_no'] = student.Entryregno
813        em = self.getEntryMode(student.Entryregno)
814        da['entry_mode'] = em
815        personal = student_obj.personal
816        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
817        dp = {'Title': 'Personal Data'}
818        student_obj.invokeFactory('StudentClearance','clearance')
819        clearance = student_obj.clearance
820        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
821        dc = {'Title': 'Clearance/Eligibility Record'}
822        dc['matric_no'] = matric_no
823        state = student.State
824        lga = student.LGA
825        if state and lga:
826            lga =  state + ' / ' + lga
827        else:
828            lga = "None"
829        da['jamb_lga'] = dc['lga'] = lga
830        da['app_email'] = dp['email'] = email
831        da['app_mobile'] = dp['phone'] = phone_nr
832        dp['firstname'] = student.Firstname
833        dp['middlename'] = student.Middlename
834        dp['lastname'] = student.Lastname
835        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
836        da['jamb_sex'] = student.Sex
837        dp['sex'] = student.Sex == 'F'
838        dp['perm_address'] = student.Permanent_Address
839        application.getContent().edit(mapping=da)
840        self.portal_workflow.doActionFor(application,'close',dest_container=application)
841        personal.getContent().edit(mapping=dp)
842        clearance.getContent().edit(mapping=dc)
843        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
844        #
845        # Study Course
846        #
847        student_obj.invokeFactory('StudentStudyCourse','study_course')
848        studycourse = student_obj.study_course
849        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
850        dsc = {}
851        dsc['study_course'] = certcode
852        dsc['current_level'] = level
853        dsc['current_verdict'] = verdict
854        dsc['current_mode'] = em   #no longer used
855        dsc['current_session'] = '05'
856        studycourse.getContent().edit(mapping=dsc)
857        #
858        # Level
859        #
860        # l = getattr(studycourse,level,None)
861        # if l is None:
862        #     studycourse.invokeFactory('StudentStudyLevel', level)
863        #     l = getattr(studycourse, level)
864        #     self.portal_workflow.doActionFor(l,'open',dest_container=l)
865        #     l.getContent().edit(mapping={'Title': "Level %s" % level})
866        ###)
867
868    def init_timing(self): ###(
869        if self.with_timing:
870            if not hasattr(self,'_v_step_times'):
871                self._v_timer_count = 0
872                self._v_total = 0
873                self._v_step_times = {}
874                current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
875                self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
876            self.timer_step = 0
877            self.total_time = 0
878            self.elapse = time.time()
879            self.i_elapse = time.time()
880            self.c_elapse = time.clock()
881    ###)
882
883    def do_timing(self): ###(
884        if self.with_timing:
885            try:
886                raise 'dummy'
887            except:
888                frame = sys.exc_traceback.tb_frame.f_back
889                locals = frame.f_locals
890                globals = frame.f_globals
891                functionname = frame.f_code.co_name
892                filename = os.path.basename(frame.f_code.co_filename)
893                lineno = frame.f_lineno
894                mod_line = "%(functionname)s:%(lineno)s" % vars()
895            i_time = time.time() - self.i_elapse
896            td = {}
897            if self._v_step_times.has_key(mod_line):
898                a_time = self._v_step_times[mod_line]['a_time'] + i_time
899                td['a_time'] = a_time
900            else:
901                td['a_time'] = i_time
902            td['i_time'] = i_time
903            self._v_step_times[mod_line] = td
904            self.i_time = i_time
905            self.total_time += i_time
906            self.timer_step +=1
907            self.i_elapse = time.time()
908    ###)
909
910    security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
911    def print_timing(self):
912        if self.with_timing:
913            l = []
914            timer_count = self._v_timer_count + 1
915            mod_lines = self._v_step_times.keys()
916            mod_lines.sort(cmp,reverse=0)
917            for mod_line in mod_lines:
918                td = self._v_step_times[mod_line]
919                i_time = td['i_time']
920                a_time = td['a_time']/(self._v_timer_count + 1)
921                l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
922            total_time = self.total_time
923            total_avarage = self._v_total / timer_count
924            l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
925            print "\r\n".join(l)
926            out = open(self._v_timer_file,'a')
927            out.write("\r\n".join(l))
928            out.close()
929    ###)
930
931    security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
932    def get_timing_data(self):
933        if self.with_timing:
934            timer_count = self._v_timer_count + 1
935            results = {}
936            for k,d in self._v_step_times.items():
937                dd = {}
938                dd['a_time'] = d['a_time']/timer_count
939                dd['i_time'] = d['i_time']
940                dd['count'] = timer_count
941                results[k] = dd
942            dd = {}
943            dd['a_time'] = self._v_total / timer_count
944            dd['i_time'] = self.total_time
945            dd['count'] = timer_count
946            results["total"] = dd
947            return results
948    ###)
949
950    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
951    def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
952        "create Datastructure for an admitted Student"
953        #import pdb;pdb.set_trace()
954        logger = logging.getLogger('WAeUPTool.admitOneStudent')
955        self.with_timing = with_timing
956
957        if brain.screening_type in ('cest','sandwich',):
958            reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
959        else:
960            reg_no = brain.reg_no
961
962        #ignore argument entry_session
963        if not brain.entry_session:
964            logger.info('no entry_session for %s provided' % (reg_no))
965            return
966
967        if not hasattr(self,"_v_certificates"):
968            self._v_certificates = self.getCertificatesDict()
969        students_folder = self.portal_url.getPortalObject().campus.students
970
971        res = self.students_catalog(jamb_reg_no = reg_no)
972        if res:
973            logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
974            return
975        if brain.status != "admitted":
976            logger.info('status of %s is %s' % (reg_no,brain.status))
977            return
978        pin_parts = brain.pin.split('-')
979        if pin_parts and len(pin_parts) != 3:
980            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
981            return
982        if not brain.course_admitted:
983            logger.info('no course_admitted provided for %s' % (reg_no))
984            return           
985        if brain.course_admitted not in self._v_certificates:
986            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
987            return
988        if brain.sex not in (True,False):
989            logger.info('sex of %s not available' % (reg_no))
990            return
991        self.init_timing()
992        student_id = self.generateStudentId('?')
993        students_folder.invokeFactory('Student', student_id)
994        student_object = getattr(students_folder,student_id)
995        self.do_timing()
996        if pin_password:
997            password = pin_parts[2]
998            self.makeStudentMember(student_id,password = password)
999        student_object.manage_setLocalRoles(student_id, ['Owner',])
1000        self.do_timing()
1001        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
1002        #
1003        # application
1004        #
1005        student_object.invokeFactory('StudentApplication','application')
1006        application = student_object.application
1007        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
1008        #self.do_timing()
1009        da = {'Title': 'Application Data'}
1010        da['jamb_reg_no'] = reg_no
1011
1012        sex = 'M'
1013        if brain.sex:
1014            sex = 'F'
1015        da['jamb_sex'] = sex
1016        da['jamb_age'] = brain.jamb_age
1017        da['app_reg_pin'] = brain.pin
1018        da['jamb_lga'] = brain.jamb_lga
1019        da['jamb_state'] = brain.jamb_state
1020        da['jamb_score'] = brain.aggregate
1021        da['app_email'] = brain.email
1022        da['app_mobile'] = brain.phone
1023
1024        # entry_mode cannot be retrieved from the certtificate
1025        # because ug_ft has two different entry modes
1026                       
1027        if brain.entry_mode:                      # does not happen because import_application
1028            da['entry_mode'] = brain.entry_mode   # schema has no field entry_mode
1029        elif brain.screening_type == 'pume': 
1030            da['entry_mode'] = 'ume_ft' 
1031        elif brain.screening_type == 'pde': 
1032            da['entry_mode'] = 'de_ft' 
1033        else:
1034            da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1035       
1036        #elif brain.screening_type == 'pce': 
1037        #   da['entry_mode'] = 'pce' 
1038        #elif brain.screening_type == 'prence': 
1039        #   da['entry_mode'] = 'prence' 
1040        #else: 
1041        #   da['entry_mode'] = '' 
1042     
1043
1044        #da['entry_session'] = entry_session
1045        da['entry_session'] = brain.entry_session
1046        da['jamb_lastname'] = brain.lastname
1047        da['jamb_middlename'] = brain.middlenames   # different field names!
1048        da['jamb_firstname'] = brain.firstname
1049        da['screening_application_date'] = brain.application_date
1050        da['date_of_birth'] = brain.date_of_birth
1051        da['jamb_first_cos'] = brain.course1
1052        da['jamb_second_cos'] = brain.course2
1053        da['course3'] = brain.course3
1054        da['screening_type'] = brain.screening_type
1055        da['screening_score'] = brain.screening_score
1056        da['screening_date'] = brain.screening_date
1057        da['hq_type'] = brain.hq_type
1058        da['hq_grade'] = brain.hq_grade
1059        da['aos'] = brain.aos
1060
1061        application.getContent().edit(mapping=da)
1062        self.do_timing()
1063        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
1064        #
1065        # personal
1066        #
1067        student_object.invokeFactory('StudentPersonal','personal')
1068        personal = student_object.personal
1069        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1070        #self.do_timing()
1071        dp = {'Title': 'Personal Data'}
1072        dp['sex'] = brain.sex
1073        dp['email'] = brain.email
1074        dp['phone'] = brain.phone
1075        dp['lastname'] = brain.lastname
1076        dp['middlename'] = brain.middlenames   # different field names!
1077        dp['firstname'] = brain.firstname
1078        personal.getContent().edit(mapping=dp)
1079        self.do_timing()
1080        #
1081        # clearance
1082        #
1083        student_object.invokeFactory('StudentClearance','clearance')
1084        clearance = student_object.clearance
1085        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1086        dc = {'Title': 'Clearance/Eligibility Record'}
1087        dc['lga'] = brain.lga
1088        dc['birthday'] = brain.date_of_birth
1089        clearance.getContent().edit(mapping=dc)
1090        self.do_timing()
1091        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1092        #
1093        # study Course
1094        #
1095        student_object.invokeFactory('StudentStudyCourse','study_course')
1096        studycourse = student_object.study_course
1097        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1098        #self.do_timing()
1099        dsc = {}
1100        dsc['study_course'] = brain.course_admitted
1101        dsc['current_verdict'] = ''
1102        dsc['current_mode'] = da['entry_mode'] # no longer used
1103        if da['entry_mode'].startswith('de'):
1104            dsc['current_level'] = '200'
1105        elif da['entry_mode'].startswith('pre'):
1106            dsc['current_level'] = '000'
1107        else:
1108            dsc['current_level'] = '100'
1109        dsc['current_session'] = brain.entry_session
1110        studycourse.getContent().edit(mapping=dsc)
1111        self.do_timing()
1112        #
1113        # payments folder
1114        student_object.invokeFactory('PaymentsFolder','payments')
1115        payments = getattr(student_object,'payments')
1116        #self.do_timing()
1117        dpay = {}
1118        dpay['Title'] = 'Payments'
1119        payments.getContent().edit(mapping=dpay)
1120        self.portal_workflow.doActionFor(payments,'open')
1121        self.do_timing()
1122        #
1123        # passport foto
1124        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1125                                                              brain.screening_type,
1126                                                              brain.reg_no)
1127        images_dir = getImagesDir(student_id)
1128        #images_dir = os.path.join("%s" % images_base,student_id)
1129        letter_dir,student_dir = os.path.split(images_dir)
1130        if not os.path.exists(letter_dir):
1131            os.mkdir(letter_dir)
1132        if not os.path.exists(images_dir):
1133            os.mkdir(images_dir)
1134        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1135        if os.path.exists(app_picture):
1136            copy2(app_picture,image_name)
1137        else:
1138            logger.info('passport of %s/%s not found: %s' % (student_id,
1139                                                             brain.reg_no,
1140                                                             app_picture))
1141
1142        self.do_timing()
1143        self.print_timing()
1144        if with_timing:
1145            self.timer_step = 0
1146            self._v_timer_count += 1
1147            self._v_total += self.total_time
1148        return student_id
1149    ###)
1150
1151    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1152    def makeStudentLevel(self,student_id):
1153        "create the StudyLevel for a returning Student"
1154        #import pdb;pdb.set_trace()
1155        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1156        students_folder = self.portal_url.getPortalObject().campus.students
1157        res = self.students_catalog(id=student_id)
1158        if res:
1159            st = res[0]
1160        course = st.course
1161        matric_no = st.matric_no
1162        level = st.level
1163        res = self.results_import(matric_no = matric_no)
1164        if res:
1165            results = res
1166        logger.info('%s creating Level %s' % (student_id,level))
1167        #
1168        # Level
1169        #
1170        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1171        studycourse = getattr(student_obj,"study_course",None)
1172        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1173        l = getattr(studycourse,level,None)
1174        if l is None:
1175            studycourse.invokeFactory('StudentStudyLevel', level)
1176            l = getattr(studycourse, level)
1177            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1178            l.getContent().edit(mapping={'Title': "Level %s" % level})
1179        ###)
1180
1181    security.declarePublic('getHallInfo') ###(
1182    def getHallInfo(self,bed):
1183        """return Hall Info"""
1184        info = {}
1185        bedsplit = bed.split('_')
1186        if len(bedsplit) == 4:
1187            hall,block,room,letter = bed.split('_')
1188        else:
1189            info['maintenance_code'] = 'None'
1190            return info
1191        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1192        if res and len(res) == 1:
1193            hall_brain = res[0]
1194            hall_doc = hall_brain.getObject().getContent()
1195        else:
1196            return info
1197        info['hall_title'] = hall_brain.Title
1198        info['maintenance_code'] = hall_doc.maintenance_code
1199        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1200        batch_doc = None
1201        for brain in res:
1202            if brain.id.startswith(info['maintenance_code']):
1203                batch_doc = brain.getObject().getContent()
1204                break
1205        if batch_doc is None:
1206            info['maintenance_fee'] = ''
1207        else:
1208            info['maintenance_fee'] = batch_doc.cost
1209        return info
1210    ###)
1211
1212    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1213    def removePictureFolder(self,student_id):
1214        """remove picture_folder by renaming it"""
1215        picture_path = getImagesDir(student_id)
1216        dest_path = os.path.join("%s" % images_base,'removed',student_id)
1217        dest_path = dest_path + "_removed"
1218        if os.path.exists(dest_path) or not os.path.exists(picture_path):
1219            return False
1220        os.rename(picture_path,dest_path)
1221        return True
1222    ###)
1223
1224    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1225    def restorePictureFolder(self,student_id):
1226        """restore picture_folder by renaming it"""
1227        picture_path = getImagesDir(student_id)
1228        orig_path = os.path.join("%s" % images_base,'removed',student_id)
1229        orig_path = orig_path + "_removed"
1230        if os.path.exists(picture_path) or not os.path.exists(orig_path):
1231            return False       
1232        os.rename(picture_path + "_removed",picture_path)
1233        return True
1234    ###)
1235
1236    security.declarePublic('picturesExist') ###(
1237    def picturesExist(self, ids,student_id=None):
1238        """check if pictures exist in the filesystem"""
1239        if student_id is None:
1240            student_id = self.getStudentId()
1241        if student_id is None:
1242            return False
1243        picture_path = getImagesDir(student_id)
1244        #picture_path = os.path.join(images_base,student_id)
1245        if not os.path.exists(picture_path):
1246            return False
1247        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1248        return set(ids).issubset(set(pictures))
1249    ###)
1250   
1251        security.declarePublic('picturePathExists') ###(
1252    def picturePathExists(self, student_id=None):
1253        """check if picture path exists in the filesystem"""
1254        if student_id is None:
1255            student_id = self.getStudentId()
1256        if student_id is None:
1257            return False
1258        picture_path = getImagesDir(student_id)
1259        if os.path.exists(picture_path):
1260            return True
1261        return False
1262    ###)
1263   
1264   
1265    security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1266    def removeUnusedImageFolders(self):
1267        """check if an unused image folders exists in the filesystem"""
1268        mtool = self.portal_membership
1269        member = mtool.getAuthenticatedMember()
1270        member_id = str(member)
1271        logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1272        abc = os.listdir(images_base)
1273        ifolders = []
1274        for i in abc:
1275            picture_path = os.path.join(images_base,i)
1276            ifolders.extend(os.listdir(picture_path))
1277        unused_ids = []
1278        for id in ifolders:
1279            res = self.students_catalog(id=id)
1280            if not res:
1281                unused_ids.append(id)
1282                #import pdb;pdb.set_trace()   
1283                if not id.endswith('removed'):
1284                    removed = self.waeup_tool.removePictureFolder(id) 
1285                    if removed:
1286                        logger.info('%s: image folder %s successfully removed' % (member_id,id))
1287                    else:
1288                        logger.info('%s: image folder %s could not be removed' % (member_id,id))
1289        return
1290       
1291    ###)   
1292
1293    security.declarePublic('picturesList') ###(
1294    def picturesList(self):
1295        """check if pictures exist in the filesystem"""
1296        path = 'images'
1297        student_id = self.getStudentId()
1298        #picture_path = os.path.join(i_home,path,student_id)
1299        picture_path = getImagesDir(student_id)
1300        if not os.path.exists(picture_path):
1301            return []
1302        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1303    ###)
1304
1305    security.declarePublic('showFsPicture') ###(
1306    def showFsPicture(self,path):
1307        """return a picture from the filesystem"""
1308        #picture_path = os.path.join(i_home,path)
1309        picture_path = os.path.join(images_base,path)
1310        response = self.REQUEST.RESPONSE
1311        #import pdb;pdb.set_trace()
1312        registry = getToolByName(self, 'mimetypes_registry')
1313        mimetype = str(registry.lookupExtension(path.lower()) or
1314                    registry.lookupExtension('file.bin'))
1315        if os.path.exists(picture_path):
1316            response.setHeader('Content-type',mimetype)
1317            return open(picture_path).read()
1318        picture_path = os.path.join(i_home,'import',path)
1319        if os.path.exists(picture_path):
1320            return open(picture_path).read()
1321    ###)
1322
1323    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1324    def deleteAllCourses(self,department="All"):
1325        ''' delete the courses'''
1326        pm = self.portal_membership
1327        member = pm.getAuthenticatedMember()
1328
1329        if str(member) not in ("henrik","joachim"):
1330            return "not possible"
1331        if department == "All":
1332            res = self.portal_catalog({'meta_type': 'Department'})
1333        if len(res) < 1:
1334            return "No Departments found"
1335
1336        deleted = []
1337        for dep in res:
1338            cf = dep.getObject().courses
1339            if cf:
1340                cf.manage_delObjects(ids=cf.objectIds())
1341                deleted.append("deleted Courses in %s" % dep.getId)
1342        return "\r".join(deleted)
1343    ###)
1344
1345    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1346    def getLogfileLines(self,filename="event.log",numlines=20):
1347        """Get last NUMLINES lines of logfile FILENAME.
1348
1349        Return last lines' of a file in the instances logfile directory as
1350        a list. The number of returned lines equals `numlines' or less. If
1351        less than `numlines' lines are available, the whole file ist
1352        returned. If the file can not be opened or some other error
1353        occurs, empty list is returend.
1354        """
1355        result = []
1356        lines_hit = 0
1357
1358        # We only handle files in instances' log directory...
1359        logpath = os.path.join(i_home, "log")
1360        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1361        if not filename.startswith( logpath ):
1362            # Attempt to access file outside log-dir...
1363            return []
1364
1365        try:
1366            fd = file( filename, "rb" )
1367        except IOError:
1368            return []
1369        if not fd:
1370            return []
1371
1372        if os.linesep == None:
1373            linesep = '\n'
1374        else:
1375            linesep = os.linesep
1376
1377        # Try to find 'numlines' times a lineseparator, searching from end
1378        # and moving to the beginning of file...
1379        fd.seek( 0, 2) # Move to end of file...
1380        while lines_hit < numlines:
1381            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1382                                          # one step forward...
1383                lines_hit += 1
1384            try:
1385                fd.seek( -2, 1) # Go two bytes back from current pos...
1386            except IOError:
1387                # We cannot go back two bytes. Maybe the file is too small...
1388                break
1389        fd.seek(2,1)
1390
1391        # Read all lines from current position...
1392        result = fd.readlines()
1393        # Remove line endings...
1394        result = [x.strip() for x in result]
1395        fd.close()
1396        return result
1397    ###)
1398
1399    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1400    def getCallbacksFromLog(self,filename):
1401        """fix Online Payment Transactions from Z2.log entries"""
1402        import transaction
1403        import random
1404        from cgi import parse_qs
1405        from urlparse import urlparse
1406        #from pdb import set_trace
1407        wftool = self.portal_workflow
1408        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1409        students_folder = self.portal_url.getPortalObject().campus.students
1410        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1411        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1412        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1413        data = re.compile(s)
1414        start = True
1415        tr_count = 1
1416        total = 0
1417        #name = 'pume_results'
1418        #name = 'epaymentsuccessful_z2log2'
1419        name = filename
1420        no_import = []
1421        imported = []
1422        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1423        try:
1424            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1425        except:
1426            logger.error('Error reading %s' % name)
1427            return
1428        tas = []
1429        for line in transactions:
1430            dict = {}
1431            items = data.search(line)
1432            dict['idict'] = idict = items.groupdict()
1433            #print idict
1434            #from pdb import set_trace;set_trace()
1435            urlparsed = urlparse(idict['get'][4:])
1436            #print urlparsed
1437            path = urlparsed[2].split('/')
1438            dict['student_id'] = student_id = path[8]
1439            dict['payment_id'] = payment_id = path[10]
1440            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1441            tas.append(dict)
1442            tr_count += 1
1443        return tas
1444    ###)
1445
1446    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1447    def importOnlinePaymentTransactions(self):
1448        """load Online Payment Transactions from CSV values"""
1449        import transaction
1450        import random
1451        #from pdb import set_trace
1452        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1453        opt = self.online_payments_import
1454        students_folder = self.portal_url.getPortalObject().campus.students
1455        start = True
1456        tr_count = 1
1457        total = 0
1458        #name = 'pume_results'
1459        name = 'OnlineTransactions'
1460        no_import = []
1461        imported = []
1462        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1463        try:
1464            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1465        except:
1466            logger.error('Error reading %s.csv' % name)
1467            return
1468        for pay_transaction in transactions:
1469            if start:
1470                start = False
1471                logger.info('Start loading from %s.csv' % name)
1472                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1473                no_import.append('%s,"Error"' % s)
1474                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1475                format_error = format + ',"%(Error)s"'
1476            data = {}
1477
1478            # format of the first file sent by Tayo
1479            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1480            #data['student_id'] = student_id = pay_transaction['Payer ID']
1481            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1482            #data['response_code'] = response_code = pay_transaction['Resp Code']
1483            #data['amount'] = amount = pay_transaction['Amount']
1484
1485            # format of the second file sent by Tayo
1486            #data['datetime'] = date = 0
1487            #data['student_id'] = student_id = pay_transaction['Payer ID']
1488            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1489            #data['response_code'] = response_code = '00'
1490            #data['amount'] = amount = pay_transaction['Amount']
1491
1492            # format of the third file sent by Kehinde
1493            data['datetime'] = date = 0
1494            data['student_id'] = student_id = pay_transaction['customer_id']
1495            data['order_id'] = order_id = pay_transaction['merchant_reference']
1496            data['response_code'] = response_code = '00'
1497            data['amount'] = amount = pay_transaction['Amount']
1498
1499            dup = False
1500            if response_code == "12":
1501                continue
1502            try:
1503                opt.addRecord(**data)
1504            except ValueError:
1505                dup = True
1506            #from pdb import set_trace;set_trace()
1507            if dup:
1508                if response_code == "00":
1509                    try:
1510                        opt.modifyRecord(**data)
1511                    except:
1512                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1513                        continue
1514                else:
1515                    pay_transaction['Error'] = "Duplicate order_id"
1516                    no_import.append( format_error % pay_transaction)
1517                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1518                    continue
1519            tr_count += 1
1520            if tr_count > 1000:
1521                if len(no_import) > 0:
1522                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1523                             '\n'.join(no_import) + '\n')
1524                    no_import = []
1525                em = '%d transactions committed\n' % (tr_count)
1526                transaction.commit()
1527                regs = []
1528                logger.info(em)
1529                total += tr_count
1530                tr_count = 0
1531        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1532                                                '\n'.join(no_import))
1533        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1534    ###)
1535
1536    security.declareProtected(ModifyPortalContent,"importData")###(
1537    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1538        """load data from CSV values"""
1539        import transaction
1540        import random
1541        students_folder = self.portal_url.getPortalObject().campus.students
1542        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1543        pending_only = False
1544        pend_str = '--'
1545        elapse = time.time()
1546        #
1547        # preparations
1548        #
1549        if filename == pend_str:
1550            pending_only = True
1551        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1552        importer = eval("%sImport" % importer_name)(self)
1553        logger = importer.logger
1554        if importer.init_errors:
1555            logger.info(importer.init_errors)
1556            return importer.init_errors
1557        member = importer.member
1558        #current = importer.current
1559        import_date = importer.import_date
1560        #
1561        # not_imported
1562        #
1563        info = importer.info
1564        data_keys = importer.data_keys
1565        csv_keys = importer.csv_keys
1566        #csv_keys.extend(info.keys())
1567        headline_mapping = dict((k,k) for k in csv_keys)
1568        #
1569        # pending
1570        #
1571        pending_path = importer.pending_path
1572        pending_tmp = importer.pending_tmp
1573        pending_backup = importer.pending_backup
1574        pending_fn = importer.pending_fn
1575        imported_path = importer.imported_path
1576        imported_fn = importer.imported_fn
1577        commit_after = importer.commit_after
1578        pending = []
1579        pending_digests = []
1580        #total_added_to_pending = 0
1581        if not pending_only:
1582            pending,pending_digests = importer.makeIdLists()
1583            pending_at_start = len(pending)
1584        datafile = open(pending_tmp,"w")
1585        pending_csv_writer = csv.DictWriter(datafile,
1586                                                    csv_keys,
1587                                                    extrasaction='ignore')
1588        pending_csv_writer.writerow(headline_mapping)
1589        datafile.close()
1590        #
1591        # imported
1592        #
1593        if not os.path.exists(imported_path):
1594            datafile = open(imported_path,"w")
1595            imported_csv_writer = csv.DictWriter(datafile,
1596                                                 csv_keys,
1597                                                 extrasaction='ignore')
1598            imported_csv_writer.writerow(headline_mapping)
1599            datafile.close()
1600        start = True
1601        tr_count = 0
1602        total = 0
1603        total_added_to_imported = 0
1604        total_pending = 0
1605        import_source_done = ""
1606        if pending_only:
1607            import_source_path = pending_path
1608        else:
1609            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1610            import_source_done = "%s/import/%s.done" % (i_home,filename)
1611        if not os.path.exists(import_source_path):
1612            fn = os.path.split(import_source_path)[1]
1613            em = 'no import file %(fn)s' % vars()
1614            return em
1615        import_source_fn = os.path.split(import_source_path)[1]
1616        if not pending_only:
1617            info['imported_from'] = import_source_fn
1618        headline = csv.reader(open(import_source_path,"rb")).next()
1619        if "import_mode" not in headline:
1620            msg = 'import_mode must be in heading'
1621            return msg
1622        invalid_keys = importer.checkHeadline(headline)
1623        if invalid_keys:
1624            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1625
1626        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1627                                                         or k.strip() in info.keys())]
1628        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1629        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1630        # if diff2schema and diff2schema != set(['id',]):
1631        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1632        #     return msg
1633        #
1634        # start importing
1635        #
1636        try:
1637            reader = csv.DictReader(open(import_source_path,"rb"))
1638        except:
1639            msg = 'Error reading %s.csv' % filename
1640            logger.error(msg)
1641            return msg
1642        items = [item for item in reader]
1643        total_to_import = len(items)
1644        tti_float = float(total_to_import)
1645        if pending_only:
1646            pending_at_start = total_to_import
1647        count = 0
1648        imported = []
1649        old_commit_count = 0
1650        error_count = imported_count = 0
1651        already_in = 0
1652        for record in items:
1653            item = {}
1654            empty_value_keys = []
1655            for k,v in record.items():
1656                if k is None:
1657                    continue
1658                if v:
1659                    if v == EMPTY:
1660                        empty_value_keys += k,
1661                        v = ''
1662                    item[k.strip()] = v.strip()
1663            count += 1
1664            if start:
1665                start = False
1666                adapters = [MappingStorageAdapter(importer.schema, item)]
1667                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1668            dm = DataModel(item, adapters,context=self)
1669            ds = DataStructure(data=item,datamodel=dm)
1670            error_string = ""
1671            total += 1
1672            import_mode = item.get('import_mode','')
1673            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1674            if import_method is None:
1675                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1676            elif (import_mode in importer.required_modes and
1677                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1678                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1679                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1680            else:
1681                for k in import_keys:
1682                    if k not in item.keys() or k not in importer.validators.keys():
1683                        continue
1684                    if not importer.validators[k](ds,mode=import_mode):
1685                        error_string += ' ++ '
1686                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1687                                                                           ds.getErrorMapping(k)))
1688            if error_string:
1689                error = error_string
1690                id = ''
1691                mapping = item
1692            else:
1693                temp_item = item.copy()
1694                temp_item.update(dm)
1695                results = import_method(temp_item)
1696                id = results[0]
1697                error = results[1]
1698                mapping = results[2]
1699            #
1700            # restore some values
1701            #
1702            for k in empty_value_keys:
1703                mapping[k] = EMPTY
1704            if mapping.has_key('sex'):
1705                #import pdb;pdb.set_trace()
1706                if mapping['sex'] == True:
1707                    mapping['sex'] = 'F'
1708                elif mapping['sex'] == False:
1709                    mapping['sex'] = 'M'
1710            if pending_only:
1711                info['imported_from'] = item['imported_from']
1712            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1713            info['error'] = error
1714            info['import_record_no'] = count + 1
1715            mapping.update(info)
1716            log_list = []
1717            if error:
1718                error_count += 1
1719                digest = makeDigest(mapping,data_keys)
1720                if digest not in pending_digests:
1721                    pending_digests += digest,
1722                    pending.append(mapping)
1723                    if not pending_only:
1724                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1725                else:
1726                    already_in += 1
1727                    pass
1728            else:
1729                imported_count += 1
1730                imported += mapping,
1731                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1732            if log_list:
1733                time_till_now = time.time() - elapse
1734                percent_finished = (error_count + imported_count)/tti_float*100
1735                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1736                logger.info(' '.join(log_list))
1737            finished = count > total_to_import - 1
1738            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1739            if must_commit:
1740                old_commit_count = imported_count
1741
1742            if must_commit or finished:
1743                if len(imported):
1744                    transaction.commit()
1745                    datafile = open(imported_path,"a")
1746                    writer = csv.DictWriter(datafile,
1747                                            csv_keys,
1748                                            extrasaction='ignore')
1749                    writer.writerows(imported)
1750                    datafile.close()
1751                    total_added_to_imported += len(imported)
1752                    imported = []
1753                if len(pending) > 0:
1754                    datafile = open(pending_tmp,"a")
1755                    writer = csv.DictWriter(datafile,
1756                                            csv_keys,
1757                                            extrasaction='ignore')
1758                    writer.writerows(pending)
1759                    datafile.close()
1760                    total_pending += len(pending)
1761                    #total_added_to_pending += len(pending)
1762                    pending = []
1763                if not finished:
1764                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1765                    logger.info(msg)
1766        elapse = time.time() - elapse
1767        if os.path.exists(pending_path):
1768            copy2(pending_path,pending_backup)
1769        copy2(pending_tmp,pending_path)
1770        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1771        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1772        if pending_only:
1773            removed_pending = pending_at_start - total_pending
1774            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1775        else:
1776            added_pending = total_pending - pending_at_start
1777            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1778        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1779        logger.info(msg)
1780        if import_source_done:
1781            copy(import_source_path,import_source_done)
1782            os.remove(import_source_path)
1783            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1784            if upload is not None:
1785                upload_doc = upload.getContent()
1786                mapping = {}
1787                #mapping['import_date'] = DateTime.DateTime()
1788                mapping['import_date'] = import_date
1789                mapping['imported_by'] = importer.imported_by
1790                mapping['import_message'] = msg
1791                upload_doc.edit(mapping = mapping)
1792        os.remove(pending_tmp)
1793        return msg
1794    ###)
1795
1796    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1797    def moveImagesToFS(self,student_id="O738726"):
1798        "move the images to the filesystem"
1799        images_dir = getImagesDir(student_id)
1800        #images_dir = os.path.join("%s" % images_base,student_id)
1801        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1802        stool = getToolByName(self, 'portal_schemas')
1803        schemas = ['student_application',
1804                   'student_clearance',
1805                   ]
1806        created = False
1807        for schema_id in schemas:
1808            schema = stool._getOb(schema_id)
1809            object = getattr(student_folder,schema_id[len('student_'):],None)
1810            if object is None:
1811                continue
1812            doc = object.getContent()
1813            for key in schema.keys():
1814                if schema[key].meta_type != "CPS Image Field":
1815                    continue
1816                #import pdb;pdb.set_trace()
1817                image = getattr(doc,key,None)
1818                if not image or not hasattr(image,"data"):
1819                    continue
1820                if not created:
1821                    if not os.path.exists(images_dir):
1822                        os.mkdir(images_dir)
1823                    created = True
1824                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1825                open(filename,"wb").write(str(image.data))
1826    ###)
1827
1828    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1829    def movePassportToFS(self,student_id="O738726"):
1830        "move the passports to the filesystem"
1831        images_dir = os.path.join("%s" % i_home,'passports')
1832        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1833        stool = getToolByName(self, 'portal_schemas')
1834        schemas = ['student_application',
1835                   #'student_clearance',
1836                   ]
1837        created = False
1838        for schema_id in schemas:
1839            schema = stool._getOb(schema_id)
1840            object = getattr(student_folder,schema_id[len('student_'):],None)
1841            if object is None:
1842                continue
1843            doc = object.getContent()
1844            for key in schema.keys():
1845                if schema[key].meta_type != "CPS Image Field":
1846                    continue
1847                #import pdb;pdb.set_trace()
1848                image = getattr(doc,key)
1849                if not hasattr(image,"data"):
1850                    continue
1851                if not created:
1852                    if not os.path.exists(images_dir):
1853                        os.mkdir(images_dir)
1854                    created = True
1855                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1856                open(filename,"wb").write(str(image.data))
1857    ###)
1858   
1859   
1860    security.declarePublic('getConfigParams')
1861    def getConfigParams(self,conf_id="configuration"):
1862        conf = getattr(self.portal_url.getPortalObject().campus,conf_id)
1863        conf_obj = conf.getContent()
1864        stool = getToolByName(self, 'portal_schemas')
1865        schema = stool._getOb('configuration')
1866        d = {}
1867        for key in schema.keys():
1868            d[key] = getattr(conf_obj,key,None)
1869        return d
1870
1871
1872InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.