source: WAeUP_SRP/trunk/WAeUPTool.py @ 3814

Last change on this file since 3814 was 3812, checked in by Henrik Bettermann, 16 years ago

resolve Uniben #589
  • Property svn:keywords set to Id
File size: 75.8 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3812 2008-12-11 07:24:16Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108    ###)
109
110    security.declareProtected(View,'updateRoleMappingsFor') ###(
111    def updateRoleMappingsFor(self,wf_definition,ob):
112        "do so for public"
113        wf_def = getattr(self.portal_workflow,wf_definition)
114        wf_def.updateRoleMappingsFor(ob)
115    ###)
116
117    security.declareProtected(View,'getStatesLgas') ###(
118    def getStatesLgas(self):
119        """return lga info"""
120        voc = getattr(self.portal_vocabularies,'local_gov_areas')
121        states = []
122        lgas  = []
123        d = {}
124        wd = {}
125        for k,v in voc.items():
126            parts = v.split(' / ')
127            if len(parts) == 1:
128                state = parts[0].lower()
129                lga = ""
130            elif len(parts) == 2:
131                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
132                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
133            else:
134                continue
135            if state not in states:
136                states.append(state)
137            if lga not in lgas:
138                lgas.append(lga)
139            words = re.split('[^a-zA-Z0-9/]',k)
140            words.sort()
141            wd[k] = words
142            d[k] = v
143        mapping = {}
144        mapping['word_dict'] = wd
145        mapping['lga_dict'] = d
146        mapping['states'] = states
147        mapping['lgas'] = lgas
148        return mapping
149    ###)
150
151    security.declareProtected(View,'findLga') ###(
152    def findLga(self,words,words_dict):
153        words = re.split('[^a-zA-Z0-9/]',words)
154        lga_words = []
155        for word in words:
156            if word:
157                lga_words += word.strip().lower(),
158        lga_words.sort()
159        state_lga = ''
160        while not state_lga:
161            for k,l in words_dict.items():
162                if lga_words == l:
163                    state_lga = k
164                    break
165            break
166        return state_lga
167    ###)
168    security.declareProtected(View,'getAccessInfo') ###(
169    def getAccessInfo(self,context):
170        "return a dict with access_info"
171        logger = logging.getLogger('WAeUPTool.getAccessInfo')
172        mtool = self.portal_membership
173        member = mtool.getAuthenticatedMember()
174        member_id = str(member)
175        info = {}
176        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
177        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
178        is_staff = info['is_staff'] = not is_anonymous and not is_student
179        roles = member.getRolesInContext(context)
180        info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
181                                                        "SectionManager" in roles or
182                                                        "Manager" in roles)
183        info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif not is_allowed and (not is_staff or  member_id != requested_id):
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378    ###)
379
380    def generateStudentId(self,letter): ###(
381        import random
382        logger = logging.getLogger('WAeUPTool.generateStudentId')
383        r = random
384        if letter == '?':
385            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
386        sid = "%c%d" % (letter,r.randint(99999,1000000))
387        students = self.portal_url.getPortalObject().campus.students
388        while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid):
389            if self.waeup_tool.picturePathExists(sid):
390                logger.info('picture path %s exists' % sid)
391            sid = "%c%d" % (letter,r.randint(99999,1000000))
392        return sid
393    ###)
394
395    def generatePassword(self,s=None): ###(
396        import random
397        r = random
398        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
399        if s is None:
400            s = 'abcdefghklmnpqrstuvwxy23456789'
401        pw = ''
402        while len(pw) < 6:
403            pw += r.choice(s)
404        return pw
405    ###)
406
407    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
408    def dumpSchoolfeePayments(self):
409        "dump paid schoolfees"
410        mtool = self.portal_membership
411        member = mtool.getAuthenticatedMember()
412        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
413        aq_student = self.students_catalog.evalAdvancedQuery
414        query = In('review_state',('schoolfee_paid',
415                                   'courses_registered',
416                                   'courses_validated',
417                                   ))
418        res = aq_student(query)
419        #import pdb;pdb.set_trace()
420        l = []
421        logger.info("start for %d" % len(res))
422        count = 1
423        log_after = 100
424        for student in res:
425            if not count % log_after:
426                logger.info("processed %d total %d" % (log_after,count))
427            count += 1
428            fee_dict =self.getSchoolFee(student)
429            fulltime = student.mode.endswith('_ft')
430            d = {}
431            d['student_id'] = student.id
432            d['name'] = student.name
433            d['amount'] = fee_dict.get(new_returning)
434            l += d,
435        csv_name = self.dumpListToCSV(l,'payments')
436        logger.info('%s dumped payments to %s' % (member,export_file))
437    ###)
438
439    security.declarePublic('dumpListToCSV') ###(
440    def dumpListToCSV(self,l,filename,fields=None):
441        """dump a list of dicts to a CSV file"""
442        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
443        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
444        if fields is None:
445            fields = l[0].keys()
446        headline = ','.join(fields)
447        out = open(export_file,"wb")
448        out.write(headline +'\n')
449        out.close()
450        out = open(export_file,"a")
451        csv_writer = csv.DictWriter(out,fields,)
452        csv_writer.writerows(l)
453        return export_file
454    ###)
455
456    security.declareProtected(ManagePortal, 'listMembers') ###(
457    def listMembers(self):
458        "list all members"
459        mtool = self.portal_membership
460        member = mtool.getAuthenticatedMember()
461        logger = logging.getLogger('WAeUPTool.listMembers')
462        if str(member) not in ('admin','joachim'):
463            logger.info('%s tried to list members' % (member))
464            return None
465        members = self.portal_directories.members
466        all = members.listEntryIdsAndTitles()
467        l = []
468        for user_id,name in all:
469            d = {}
470            d['user_id'] = user_id
471            d['name'] = name
472            d['pw'] = getattr(getattr(members,user_id),'password')
473            d['email'] = getattr(getattr(members,user_id),'email')
474            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
475            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
476            l += d,
477        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
478        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
479        logger.info('%s dumped member list to %s' % (member,export_file))
480        fields = l[0].keys()
481        headline = ','.join(fields)
482        out = open(export_file,"wb")
483        out.write(headline +'\n')
484        out.close()
485        out = open(export_file,"a")
486        csv_writer = csv.DictWriter(out,fields,)
487        csv_writer.writerows(l)
488    ###)
489
490    security.declareProtected(ManagePortal, 'listStudents') ###(
491    def listStudents(self):
492        "list all students"
493        mtool = self.portal_membership
494        member = mtool.getAuthenticatedMember()
495        logger = logging.getLogger('WAeUPTool.listStudents')
496        if str(member) not in ('admin','joachim'):
497            logger.info('%s tried to list students' % (member))
498            return None
499        students = self.portal_directories.students
500        all = students.listEntryIdsAndTitles()
501        l = []
502        for user_id,name in all:
503            d = {}
504            d['user_id'] = user_id
505            d['name'] = name
506            d['pw'] = getattr(getattr(students,user_id),'password')
507            d['email'] = getattr(getattr(students,user_id),'email')
508            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
509            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
510            l += d,
511        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
512        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
513        logger.info('%s dumped student list to %s' % (member,export_file))
514        fields = l[0].keys()
515        headline = ','.join(fields)
516        out = open(export_file,"wb")
517        out.write(headline +'\n')
518        out.close()
519        out = open(export_file,"a")
520        csv_writer = csv.DictWriter(out,fields,)
521        csv_writer.writerows(l)
522    ###)
523
524    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
525    def removeDeletedDocIds(self, max=1000):
526        """
527        remove deleted docids from repository commit after max
528        """
529        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
530        repository = getToolByName(self, 'portal_repository')
531        pxtool = getToolByName(self, 'portal_proxies')
532        logger.info('start')
533        pxtool_infos = pxtool.getRevisionsUsed()
534        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
535
536        nb_revs = 0
537        docids_d = {} # all docids
538        unused_docids_d = {} # all docids that are unused
539        ids_unused_revs_docids = [] # ids for revs of unused docids
540        unused_ids = [] # ids for unused revs
541        total = 0
542        idlist = repository.objectIds()
543        to_delete = 0
544        found = False
545        for id in idlist:
546            docid, rev = repository._splitId(id)
547            if docid is None:
548                logger.info("invalid doc_id %s" % docid)
549                continue
550            nb_revs += 1
551            if not pxtool_infos.has_key(docid):
552                found = True
553                to_delete += 1
554                unused_ids.append(id)
555            elif not pxtool_infos[docid].has_key(rev):
556                found = True
557                to_delete += 1
558                unused_ids.append(id)
559            if found and not to_delete % max:
560                found = False
561                #import pdb;pdb.set_trace()
562                repository.manage_delObjects(unused_ids)
563                transaction.commit()
564                logger.info('removed %d total %d unused docids ' % (max,to_delete))
565        else:
566            if unused_ids:
567                repository.manage_delObjects(unused_ids)
568                transaction.commit()
569        logger.info('finished removing %d unused docids ' % (to_delete))
570    ###)
571
572    security.declareProtected(View,'getCredential') ###(
573    def getCredential(self,student_id):
574        student_entry = getattr(self.portal_directories.students,student_id,None)
575        if not self.isStaff():
576            mtool = self.portal_membership
577            member = mtool.getAuthenticatedMember()
578            logger = logging.getLogger('WAeUPTool.getCredential')
579            logger.info('%s tried to access password of %s' % (member,student_id))
580            return None
581        if student_entry is None:
582            return None
583        return getattr(student_entry,"password","not set")
584    ###)
585
586    security.declarePublic('checkPassword') ###(
587    def checkPassword(self,student_id,password):
588        student_entry = getattr(self.portal_directories.students,student_id,None)
589        if student_entry is None:
590            return False
591        return getattr(student_entry,"password","not set") == password
592    ###)
593
594    security.declarePublic('checkGenericPassword') ###(
595    def checkGenericPassword(self,member_id):
596        member_entry = getattr(self.portal_directories.members,member_id,None)
597        if member_entry is None:
598            return False
599        ltool = getToolByName(self, 'portal_layouts')
600        unsecure_words = ltool._getOb('members')['w__password'].check_words
601        password = getattr(member_entry,"password","not set")
602        is_unsecure = password in unsecure_words
603        if is_unsecure:
604            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
605            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
606        return is_unsecure
607    ###)
608
609    security.declareProtected(ModifyPortalContent,'editPassword') ###(
610    def editPassword(self,student_id,password):
611        "edit a student password"
612        student_entry = getattr(self.portal_directories.students,student_id,None)
613        if student_entry is None:
614            return
615        setattr(student_entry,'password',password)
616    ###)
617
618    security.declareProtected(ModifyPortalContent,'doCommit') ###(
619    def doCommit(self,logger=None):
620        "commit some transactions"
621        transaction.commit()
622    ###)
623
624    security.declarePublic('loadStudentFoto') ###(
625    def loadStudentFoto(self,student,filename,folder):
626        "return a student passport picture"
627        #import pdb;pdb.set_trace()
628        picture ="%s/import/%s/%s" % (i_home,folder,filename)
629        student_id = student.getId()
630        images_dir = getImagesDir(student_id)
631        if not os.path.exists(images_dir):
632            os.mkdir(images_dir)
633        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
634        for extension in ('.jpg','.JPG'):
635            fullname = "%(picture)s%(extension)s" % vars()
636            if os.path.exists(fullname):
637                copy2(fullname,image_name)
638                return "successfully copied passport picture"
639        return "passport picture not found: %s.jpg or .JPG" % picture
640    ###)
641
642    def old____loadStudentFoto(self,student,filename,folder): ###(
643        "return a student passport picture"
644        app = student.application
645        app_doc = app.getContent()
646        #clear = student.clearance
647        #clear_doc = clear.getContent()
648        #matric_no = clear_doc.matric_no.upper()
649        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
650        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
651        #import pdb;pdb.set_trace()
652        if os.path.exists(picture1):
653            file = open(picture1)
654        elif os.path.exists(picture2):
655            file = open(picture2)
656        else:
657            return "passport picture not found %s" % picture1
658        reopened = False
659        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
660            self.portal_workflow.doActionFor(app,'open')
661            reopened = True
662        outfile = file.read()
663        app_doc.manage_addFile('passport',
664                               file=outfile,
665                               title="%s.jpg" % filename)
666        if reopened:
667            self.portal_workflow.doActionFor(app,'close')
668        return "successfully loaded passport picture"
669    ###)
670
671    security.declareProtected(ModifyPortalContent,'createOne') ###(
672    def createOne(self,students_folder,student_brain,letter,commit=False):
673        sid = self.waeup_tool.generateStudentId(letter)
674        students_folder.invokeFactory('Student', sid)
675        student = getattr(students_folder,sid)
676        self.portal_workflow.doActionFor(student,'return')
677        student.manage_setLocalRoles(sid, ['Owner',])
678        matric_no = student_brain.matric_no
679        jamb_reg_no = student_brain.Entryregno
680        self.students_catalog.addRecord(id = sid,
681                                           matric_no = matric_no,
682                                           jamb_reg_no = jamb_reg_no,
683                                           sex = student_brain.Sex == "F",
684                                           name = "%s %s %s" % (student_brain.Firstname,
685                                                                student_brain.Middlename,
686                                                                student_brain.Lastname)
687                                        )
688        if commit:
689            transaction.commit()
690        return sid,jamb_reg_no
691    ###)
692
693    security.declareProtected(ModifyPortalContent,'addStudent') ###(
694    def addStudent(self,dict):
695        students_folder = self.portal_url.getPortalObject().campus.students
696        sid = self.waeup_tool.generateStudentId('?')
697        students_folder.invokeFactory('Student', sid)
698        student_obj = getattr(students_folder,sid)
699        f2t = StudentImport.field2types_student
700        #from pdb import set_trace; set_trace()
701        d = {}
702        #d['jamb_sex']  = 'M'
703        #if dict.get('sex'):
704        #    d['jamb_sex']  = 'F'
705
706        entry_session = dict.get('entry_session')
707        if entry_session == self.getSessionId()[0]:
708            wfaction = 'admit'
709            wft = 'wf_transition_admit'
710            password = None
711        else:
712            wfaction = 'return'
713            wft = 'wf_transition_return'
714            password = self.generatePassword()
715            self.makeStudentMember(sid,password)
716
717        for pt in f2t.keys():
718            student_obj.invokeFactory(pt,f2t[pt]['id'])
719            sub_obj = getattr(student_obj,f2t[pt]['id'])
720            sub_doc = sub_obj.getContent()
721            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
722            #d['Title'] = f2t[pt]['title']
723            for field in f2t[pt]['fields']:
724                d[field] = dict.get(field,'')
725            sub_doc.edit(mapping = d)
726            new_state = f2t[pt][wft]
727            if new_state != "remain":
728                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
729        self.portal_workflow.doActionFor(student_obj,wfaction)
730        student_obj.manage_setLocalRoles(sid, ['Owner',])
731        return sid,password
732    ###)
733
734    security.declarePublic('getCertificateBrain') ###(
735    def getCertificateBrain(self,cert_id):
736        "do it"
737        res = ZCatalog.searchResults(self.portal_catalog_real,
738                                {'portal_type':"Certificate",
739                                      'id': cert_id})
740        if res:
741            return res[0]
742        return None
743    ###)
744
745    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
746    def get_csv_filenames(self):
747        "do it"
748        files = [file for file in os.listdir("%s/import/" % (i_home))
749                 if file.endswith('.csv') and (file.find('imported') == -1 and
750                                               file.find('pending') == -1)]
751        return files
752    ###)
753
754    security.declarePublic('findStudentByMatricelNo') ###(
755    def findStudentByMatricelNo(self,matric_no):
756        "do it"
757        res = ZCatalog.searchResults(self.portal_catalog_real,
758                                {'portal_type':"StudentClearance",
759                                 'SearchableText': matric_no})
760        if res:
761            return res[0]
762        return None
763    ###)
764
765    security.declarePublic('makeStudentMember') ###(
766    def makeStudentMember(self,sid,password='uNsEt'):
767        """make the student a member"""
768        membership = self.portal_membership
769        membership.addMember(sid,
770                             password ,
771                             roles=('Member',
772                                     'Student',
773                                     ),
774                             domains='',
775                             properties = {'memberareaCreationFlag': False,
776                                           'homeless': True},)
777        member = membership.getMemberById(sid)
778        self.portal_registration.afterAdd(member, sid, password, None)
779        #self.manage_setLocalRoles(sid, ['Owner',])
780    ###)
781
782    security.declareProtected(View,'makeStudentData') ###(
783    def makeStudentData(self,student_id,email=None,phone_nr=None):
784        "create Datastructure for a returning Student"
785        #import pdb;pdb.set_trace()
786        logger = logging.getLogger('WAeUPTool.makeStudentData')
787        students_folder = self.portal_url.getPortalObject().campus.students
788        #res = self.students_catalog(id=student_id)
789        #if res:
790        #    st = res[0]
791        #res = self.returning_import(matric_no = st.matric_no)
792        res = self.returning_import(id = student_id)
793        if res:
794            student = res[0]
795        else:
796            logger.info('Id %s not found in returning_import' % student_id)
797            return
798        logger.info('%s creates data structure' % student_id)
799        s_results = self.results_import(matric_no = student.matric_no)
800        if s_results:
801            lnr = self.getLevelFromResultsCosCode(s_results)
802            level = "%d00" % lnr
803            verdict,eligible = self.getVerdict(s_results[0].Verdict)
804            #if eligible:
805            #    level = "%d00" % (lnr + 1)
806        else:
807            logger.info('matric_no %s not found in results_import' % student.matric_no)
808            level = ''
809            verdict = ''
810        #student should not be allowed to perform this transition
811        #wftool = self.portal_workflow
812        #wftool.doActionFor(student,'return')
813        certcode_org = student.Coursemajorcode
814        certcode = makeCertificateCode(certcode_org)
815        certificate_brain = self.getCertificateBrain(certcode)
816        if not certificate_brain:
817            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
818            logger.info(em)
819        matric_no = student.matric_no
820        sid = student_id
821        student_obj = getattr(students_folder,sid)
822        if not getattr(student_obj,'application'):
823            student_obj.invokeFactory('StudentApplication','application')
824        application = student_obj.application
825        self.portal_workflow.doActionFor(application,'open',dest_container=application)
826        da = {'Title': 'Application Data'}
827        student_obj.invokeFactory('StudentPersonal','personal')
828        da['jamb_reg_no'] = student.Entryregno
829        em = self.getEntryMode(student.Entryregno)
830        da['entry_mode'] = em
831        personal = student_obj.personal
832        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
833        dp = {'Title': 'Personal Data'}
834        student_obj.invokeFactory('StudentClearance','clearance')
835        clearance = student_obj.clearance
836        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
837        dc = {'Title': 'Clearance/Eligibility Record'}
838        dc['matric_no'] = matric_no
839        state = student.State
840        lga = student.LGA
841        if state and lga:
842            lga =  state + ' / ' + lga
843        else:
844            lga = "None"
845        da['jamb_lga'] = dc['lga'] = lga
846        da['app_email'] = dp['email'] = email
847        da['app_mobile'] = dp['phone'] = phone_nr
848        dp['firstname'] = student.Firstname
849        dp['middlename'] = student.Middlename
850        dp['lastname'] = student.Lastname
851        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
852        da['jamb_sex'] = student.Sex
853        dp['sex'] = student.Sex == 'F'
854        dp['perm_address'] = student.Permanent_Address
855        application.getContent().edit(mapping=da)
856        self.portal_workflow.doActionFor(application,'close',dest_container=application)
857        personal.getContent().edit(mapping=dp)
858        clearance.getContent().edit(mapping=dc)
859        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
860        #
861        # Study Course
862        #
863        student_obj.invokeFactory('StudentStudyCourse','study_course')
864        studycourse = student_obj.study_course
865        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
866        dsc = {}
867        dsc['study_course'] = certcode
868        dsc['current_level'] = level
869        dsc['current_verdict'] = verdict
870        dsc['current_mode'] = em   #no longer used
871        dsc['current_session'] = '05'
872        studycourse.getContent().edit(mapping=dsc)
873        #
874        # Level
875        #
876        # l = getattr(studycourse,level,None)
877        # if l is None:
878        #     studycourse.invokeFactory('StudentStudyLevel', level)
879        #     l = getattr(studycourse, level)
880        #     self.portal_workflow.doActionFor(l,'open',dest_container=l)
881        #     l.getContent().edit(mapping={'Title': "Level %s" % level})
882        ###)
883
884    def init_timing(self): ###(
885        if self.with_timing:
886            if not hasattr(self,'_v_step_times'):
887                self._v_timer_count = 0
888                self._v_total = 0
889                self._v_step_times = {}
890                current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
891                self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,)
892            self.timer_step = 0
893            self.total_time = 0
894            self.elapse = time.time()
895            self.i_elapse = time.time()
896            self.c_elapse = time.clock()
897    ###)
898
899    def do_timing(self): ###(
900        if self.with_timing:
901            try:
902                raise 'dummy'
903            except:
904                frame = sys.exc_traceback.tb_frame.f_back
905                locals = frame.f_locals
906                globals = frame.f_globals
907                functionname = frame.f_code.co_name
908                filename = os.path.basename(frame.f_code.co_filename)
909                lineno = frame.f_lineno
910                mod_line = "%(functionname)s:%(lineno)s" % vars()
911            i_time = time.time() - self.i_elapse
912            td = {}
913            if self._v_step_times.has_key(mod_line):
914                a_time = self._v_step_times[mod_line]['a_time'] + i_time
915                td['a_time'] = a_time
916            else:
917                td['a_time'] = i_time
918            td['i_time'] = i_time
919            self._v_step_times[mod_line] = td
920            self.i_time = i_time
921            self.total_time += i_time
922            self.timer_step +=1
923            self.i_elapse = time.time()
924    ###)
925
926    security.declareProtected(ModifyPortalContent,'print_timing') ###( ###(
927    def print_timing(self):
928        if self.with_timing:
929            l = []
930            timer_count = self._v_timer_count + 1
931            mod_lines = self._v_step_times.keys()
932            mod_lines.sort(cmp,reverse=0)
933            for mod_line in mod_lines:
934                td = self._v_step_times[mod_line]
935                i_time = td['i_time']
936                a_time = td['a_time']/(self._v_timer_count + 1)
937                l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()),
938            total_time = self.total_time
939            total_avarage = self._v_total / timer_count
940            l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()),
941            print "\r\n".join(l)
942            out = open(self._v_timer_file,'a')
943            out.write("\r\n".join(l))
944            out.close()
945    ###)
946
947    security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###(
948    def get_timing_data(self):
949        if self.with_timing:
950            timer_count = self._v_timer_count + 1
951            results = {}
952            for k,d in self._v_step_times.items():
953                dd = {}
954                dd['a_time'] = d['a_time']/timer_count
955                dd['i_time'] = d['i_time']
956                dd['count'] = timer_count
957                results[k] = dd
958            dd = {}
959            dd['a_time'] = self._v_total / timer_count
960            dd['i_time'] = self.total_time
961            dd['count'] = timer_count
962            results["total"] = dd
963            return results
964    ###)
965
966    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
967    def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False):
968        "create Datastructure for an admitted Student"
969        #import pdb;pdb.set_trace()
970        logger = logging.getLogger('WAeUPTool.admitOneStudent')
971        self.with_timing = with_timing
972
973        if brain.screening_type in ('cest','sandwich',):
974            reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session)
975        else:
976            reg_no = brain.reg_no
977
978        #ignore argument entry_session
979        if not brain.entry_session:
980            logger.info('no entry_session for %s provided' % (reg_no))
981            return
982
983        if not hasattr(self,"_v_certificates"):
984            self._v_certificates = self.getCertificatesDict()
985        students_folder = self.portal_url.getPortalObject().campus.students
986
987        res = self.students_catalog(jamb_reg_no = reg_no)
988        if res:
989            logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id))
990            return
991        if brain.status != "admitted":
992            logger.info('status of %s is %s' % (reg_no,brain.status))
993            return
994        pin_parts = brain.pin.split('-')
995        if pin_parts and len(pin_parts) != 3:
996            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
997            return
998        if not brain.course_admitted:
999            logger.info('no course_admitted provided for %s' % (reg_no))
1000            return           
1001        if brain.course_admitted not in self._v_certificates:
1002            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
1003            return
1004        if brain.sex not in (True,False):
1005            logger.info('sex of %s not available' % (reg_no))
1006            return
1007        self.init_timing()
1008        student_id = self.generateStudentId('?')
1009        students_folder.invokeFactory('Student', student_id)
1010        student_object = getattr(students_folder,student_id)
1011        self.do_timing()
1012        if pin_password:
1013            password = pin_parts[2]
1014            self.makeStudentMember(student_id,password = password)
1015        student_object.manage_setLocalRoles(student_id, ['Owner',])
1016        self.do_timing()
1017        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
1018        #
1019        # application
1020        #
1021        student_object.invokeFactory('StudentApplication','application')
1022        application = student_object.application
1023        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
1024        #self.do_timing()
1025        da = {'Title': 'Application Data'}
1026        da['jamb_reg_no'] = reg_no
1027
1028        sex = 'M'
1029        if brain.sex:
1030            sex = 'F'
1031        da['jamb_sex'] = sex
1032        da['jamb_age'] = brain.jamb_age
1033        da['app_reg_pin'] = brain.pin
1034        da['jamb_lga'] = brain.jamb_lga
1035        da['jamb_state'] = brain.jamb_state
1036        da['jamb_score'] = brain.aggregate
1037        da['app_email'] = brain.email
1038        da['app_mobile'] = brain.phone
1039
1040        # entry_mode cannot be retrieved from the certtificate
1041        # because ug_ft has two different entry modes
1042                       
1043        if brain.entry_mode:                      # does not happen because import_application
1044            da['entry_mode'] = brain.entry_mode   # schema has no field entry_mode
1045        elif brain.screening_type == 'pume': 
1046            da['entry_mode'] = 'ume_ft' 
1047        elif brain.screening_type == 'pde': 
1048            da['entry_mode'] = 'de_ft' 
1049        else:
1050            da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
1051       
1052        #elif brain.screening_type == 'pce': 
1053        #   da['entry_mode'] = 'pce' 
1054        #elif brain.screening_type == 'prence': 
1055        #   da['entry_mode'] = 'prence' 
1056        #else: 
1057        #   da['entry_mode'] = '' 
1058     
1059
1060        #da['entry_session'] = entry_session
1061        da['entry_session'] = brain.entry_session
1062        da['jamb_lastname'] = brain.lastname
1063        da['jamb_middlename'] = brain.middlenames   # different field names!
1064        da['jamb_firstname'] = brain.firstname
1065        da['screening_application_date'] = brain.application_date
1066        da['date_of_birth'] = brain.date_of_birth
1067        da['jamb_first_cos'] = brain.course1
1068        da['jamb_second_cos'] = brain.course2
1069        da['course3'] = brain.course3
1070        da['screening_type'] = brain.screening_type
1071        da['screening_score'] = brain.screening_score
1072        da['screening_date'] = brain.screening_date
1073        da['hq_type'] = brain.hq_type
1074        da['hq_grade'] = brain.hq_grade
1075        da['aos'] = brain.aos
1076
1077        application.getContent().edit(mapping=da)
1078        self.do_timing()
1079        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
1080        #
1081        # personal
1082        #
1083        student_object.invokeFactory('StudentPersonal','personal')
1084        personal = student_object.personal
1085        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
1086        #self.do_timing()
1087        dp = {'Title': 'Personal Data'}
1088        dp['sex'] = brain.sex
1089        dp['email'] = brain.email
1090        dp['phone'] = brain.phone
1091        dp['lastname'] = brain.lastname
1092        dp['middlename'] = brain.middlenames   # different field names!
1093        dp['firstname'] = brain.firstname
1094        personal.getContent().edit(mapping=dp)
1095        self.do_timing()
1096        #
1097        # clearance
1098        #
1099        student_object.invokeFactory('StudentClearance','clearance')
1100        clearance = student_object.clearance
1101        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
1102        dc = {'Title': 'Clearance/Eligibility Record'}
1103        dc['lga'] = brain.lga
1104        dc['birthday'] = brain.date_of_birth
1105        clearance.getContent().edit(mapping=dc)
1106        self.do_timing()
1107        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1108        #
1109        # study Course
1110        #
1111        student_object.invokeFactory('StudentStudyCourse','study_course')
1112        studycourse = student_object.study_course
1113        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1114        #self.do_timing()
1115        dsc = {}
1116        dsc['study_course'] = brain.course_admitted
1117        dsc['current_verdict'] = ''
1118        dsc['current_mode'] = da['entry_mode'] # no longer used
1119        if da['entry_mode'].startswith('de'):
1120            dsc['current_level'] = '200'
1121        elif da['entry_mode'].startswith('pre'):
1122            dsc['current_level'] = '000'
1123        else:
1124            dsc['current_level'] = '100'
1125        dsc['current_session'] = brain.entry_session
1126        studycourse.getContent().edit(mapping=dsc)
1127        self.do_timing()
1128        #
1129        # payments folder
1130        student_object.invokeFactory('PaymentsFolder','payments')
1131        payments = getattr(student_object,'payments')
1132        #self.do_timing()
1133        dpay = {}
1134        dpay['Title'] = 'Payments'
1135        payments.getContent().edit(mapping=dpay)
1136        self.portal_workflow.doActionFor(payments,'open')
1137        self.do_timing()
1138        #
1139        # passport foto
1140        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1141                                                              brain.screening_type,
1142                                                              brain.reg_no)
1143        images_dir = getImagesDir(student_id)
1144        #images_dir = os.path.join("%s" % images_base,student_id)
1145        letter_dir,student_dir = os.path.split(images_dir)
1146        if not os.path.exists(letter_dir):
1147            os.mkdir(letter_dir)
1148        if not os.path.exists(images_dir):
1149            os.mkdir(images_dir)
1150        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1151        if os.path.exists(app_picture):
1152            copy2(app_picture,image_name)
1153        else:
1154            logger.info('passport of %s/%s not found: %s' % (student_id,
1155                                                             brain.reg_no,
1156                                                             app_picture))
1157
1158        self.do_timing()
1159        self.print_timing()
1160        if with_timing:
1161            self.timer_step = 0
1162            self._v_timer_count += 1
1163            self._v_total += self.total_time
1164        return student_id
1165    ###)
1166
1167    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1168    def makeStudentLevel(self,student_id):
1169        "create the StudyLevel for a returning Student"
1170        #import pdb;pdb.set_trace()
1171        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1172        students_folder = self.portal_url.getPortalObject().campus.students
1173        res = self.students_catalog(id=student_id)
1174        if res:
1175            st = res[0]
1176        course = st.course
1177        matric_no = st.matric_no
1178        level = st.level
1179        res = self.results_import(matric_no = matric_no)
1180        if res:
1181            results = res
1182        logger.info('%s creating Level %s' % (student_id,level))
1183        #
1184        # Level
1185        #
1186        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1187        studycourse = getattr(student_obj,"study_course",None)
1188        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1189        l = getattr(studycourse,level,None)
1190        if l is None:
1191            studycourse.invokeFactory('StudentStudyLevel', level)
1192            l = getattr(studycourse, level)
1193            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1194            l.getContent().edit(mapping={'Title': "Level %s" % level})
1195        ###)
1196
1197    security.declarePublic('getHallInfo') ###(
1198    def getHallInfo(self,bed):
1199        """return Hall Info"""
1200        info = {}
1201        bedsplit = bed.split('_')
1202        if len(bedsplit) == 4:
1203            hall,block,room,letter = bed.split('_')
1204        else:
1205            info['maintenance_code'] = 'None'
1206            return info
1207        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1208        if res and len(res) == 1:
1209            hall_brain = res[0]
1210            hall_doc = hall_brain.getObject().getContent()
1211        else:
1212            return info
1213        info['hall_title'] = hall_brain.Title
1214        info['maintenance_code'] = hall_doc.maintenance_code
1215        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1216        batch_doc = None
1217        for brain in res:
1218            if brain.id.startswith(info['maintenance_code']):
1219                batch_doc = brain.getObject().getContent()
1220                break
1221        if batch_doc is None:
1222            info['maintenance_fee'] = ''
1223        else:
1224            info['maintenance_fee'] = batch_doc.cost
1225        return info
1226    ###)
1227
1228    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1229    def removePictureFolder(self,student_id):
1230        """remove picture_folder by renaming it"""
1231        picture_path = getImagesDir(student_id)
1232        dest_path = os.path.join("%s" % images_base,'removed',student_id)
1233        dest_path = dest_path + "_removed"
1234        if os.path.exists(dest_path) or not os.path.exists(picture_path):
1235            return False
1236        os.rename(picture_path,dest_path)
1237        return True
1238    ###)
1239
1240    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1241    def restorePictureFolder(self,student_id):
1242        """restore picture_folder by renaming it"""
1243        picture_path = getImagesDir(student_id)
1244        orig_path = os.path.join("%s" % images_base,'removed',student_id)
1245        orig_path = orig_path + "_removed"
1246        if os.path.exists(picture_path) or not os.path.exists(orig_path):
1247            return False       
1248        os.rename(picture_path + "_removed",picture_path)
1249        return True
1250    ###)
1251
1252    security.declarePublic('picturesExist') ###(
1253    def picturesExist(self, ids,student_id=None):
1254        """check if pictures exist in the filesystem"""
1255        if student_id is None:
1256            student_id = self.getStudentId()
1257        if student_id is None:
1258            return False
1259        picture_path = getImagesDir(student_id)
1260        #picture_path = os.path.join(images_base,student_id)
1261        if not os.path.exists(picture_path):
1262            return False
1263        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1264        return set(ids).issubset(set(pictures))
1265    ###)
1266   
1267        security.declarePublic('picturePathExists') ###(
1268    def picturePathExists(self, student_id=None):
1269        """check if picture path exists in the filesystem"""
1270        if student_id is None:
1271            student_id = self.getStudentId()
1272        if student_id is None:
1273            return False
1274        picture_path = getImagesDir(student_id)
1275        if os.path.exists(picture_path):
1276            return True
1277        return False
1278    ###)
1279   
1280   
1281    security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###(
1282    def removeUnusedImageFolders(self):
1283        """check if an unused image folders exists in the filesystem"""
1284        mtool = self.portal_membership
1285        member = mtool.getAuthenticatedMember()
1286        member_id = str(member)
1287        logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders')
1288        abc = os.listdir(images_base)
1289        ifolders = []
1290        for i in abc:
1291            picture_path = os.path.join(images_base,i)
1292            ifolders.extend(os.listdir(picture_path))
1293        unused_ids = []
1294        for id in ifolders:
1295            res = self.students_catalog(id=id)
1296            if not res:
1297                unused_ids.append(id)
1298                #import pdb;pdb.set_trace()   
1299                if not id.endswith('removed'):
1300                    removed = self.waeup_tool.removePictureFolder(id) 
1301                    if removed:
1302                        logger.info('%s: image folder %s successfully removed' % (member_id,id))
1303                    else:
1304                        logger.info('%s: image folder %s could not be removed' % (member_id,id))
1305        return
1306       
1307    ###)   
1308
1309    security.declarePublic('picturesList') ###(
1310    def picturesList(self):
1311        """check if pictures exist in the filesystem"""
1312        path = 'images'
1313        student_id = self.getStudentId()
1314        #picture_path = os.path.join(i_home,path,student_id)
1315        picture_path = getImagesDir(student_id)
1316        if not os.path.exists(picture_path):
1317            return []
1318        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1319    ###)
1320
1321    security.declarePublic('showFsPicture') ###(
1322    def showFsPicture(self,path):
1323        """return a picture from the filesystem"""
1324        #picture_path = os.path.join(i_home,path)
1325        picture_path = os.path.join(images_base,path)
1326        response = self.REQUEST.RESPONSE
1327        #import pdb;pdb.set_trace()
1328        registry = getToolByName(self, 'mimetypes_registry')
1329        mimetype = str(registry.lookupExtension(path.lower()) or
1330                    registry.lookupExtension('file.bin'))
1331        if os.path.exists(picture_path):
1332            response.setHeader('Content-type',mimetype)
1333            return open(picture_path).read()
1334        picture_path = os.path.join(i_home,'import',path)
1335        if os.path.exists(picture_path):
1336            return open(picture_path).read()
1337    ###)
1338
1339    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1340    def deleteAllCourses(self,department="All"):
1341        ''' delete the courses'''
1342        pm = self.portal_membership
1343        member = pm.getAuthenticatedMember()
1344
1345        if str(member) not in ("henrik","joachim"):
1346            return "not possible"
1347        if department == "All":
1348            res = self.portal_catalog({'meta_type': 'Department'})
1349        if len(res) < 1:
1350            return "No Departments found"
1351
1352        deleted = []
1353        for dep in res:
1354            cf = dep.getObject().courses
1355            if cf:
1356                cf.manage_delObjects(ids=cf.objectIds())
1357                deleted.append("deleted Courses in %s" % dep.getId)
1358        return "\r".join(deleted)
1359    ###)
1360
1361    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1362    def getLogfileLines(self,filename="event.log",numlines=20):
1363        """Get last NUMLINES lines of logfile FILENAME.
1364
1365        Return last lines' of a file in the instances logfile directory as
1366        a list. The number of returned lines equals `numlines' or less. If
1367        less than `numlines' lines are available, the whole file ist
1368        returned. If the file can not be opened or some other error
1369        occurs, empty list is returend.
1370        """
1371        result = []
1372        lines_hit = 0
1373
1374        # We only handle files in instances' log directory...
1375        logpath = os.path.join(i_home, "log")
1376        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1377        if not filename.startswith( logpath ):
1378            # Attempt to access file outside log-dir...
1379            return []
1380
1381        try:
1382            fd = file( filename, "rb" )
1383        except IOError:
1384            return []
1385        if not fd:
1386            return []
1387
1388        if os.linesep == None:
1389            linesep = '\n'
1390        else:
1391            linesep = os.linesep
1392
1393        # Try to find 'numlines' times a lineseparator, searching from end
1394        # and moving to the beginning of file...
1395        fd.seek( 0, 2) # Move to end of file...
1396        while lines_hit < numlines:
1397            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1398                                          # one step forward...
1399                lines_hit += 1
1400            try:
1401                fd.seek( -2, 1) # Go two bytes back from current pos...
1402            except IOError:
1403                # We cannot go back two bytes. Maybe the file is too small...
1404                break
1405        fd.seek(2,1)
1406
1407        # Read all lines from current position...
1408        result = fd.readlines()
1409        # Remove line endings...
1410        result = [x.strip() for x in result]
1411        fd.close()
1412        return result
1413    ###)
1414
1415    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1416    def getCallbacksFromLog(self,filename):
1417        """fix Online Payment Transactions from Z2.log entries"""
1418        import transaction
1419        import random
1420        from cgi import parse_qs
1421        from urlparse import urlparse
1422        #from pdb import set_trace
1423        wftool = self.portal_workflow
1424        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1425        students_folder = self.portal_url.getPortalObject().campus.students
1426        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1427        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1428        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1429        data = re.compile(s)
1430        start = True
1431        tr_count = 1
1432        total = 0
1433        #name = 'pume_results'
1434        #name = 'epaymentsuccessful_z2log2'
1435        name = filename
1436        no_import = []
1437        imported = []
1438        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1439        try:
1440            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1441        except:
1442            logger.error('Error reading %s' % name)
1443            return
1444        tas = []
1445        for line in transactions:
1446            dict = {}
1447            items = data.search(line)
1448            dict['idict'] = idict = items.groupdict()
1449            #print idict
1450            #from pdb import set_trace;set_trace()
1451            urlparsed = urlparse(idict['get'][4:])
1452            #print urlparsed
1453            path = urlparsed[2].split('/')
1454            dict['student_id'] = student_id = path[8]
1455            dict['payment_id'] = payment_id = path[10]
1456            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1457            tas.append(dict)
1458            tr_count += 1
1459        return tas
1460    ###)
1461
1462    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1463    def importOnlinePaymentTransactions(self):
1464        """load Online Payment Transactions from CSV values"""
1465        import transaction
1466        import random
1467        #from pdb import set_trace
1468        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1469        opt = self.online_payments_import
1470        students_folder = self.portal_url.getPortalObject().campus.students
1471        start = True
1472        tr_count = 1
1473        total = 0
1474        #name = 'pume_results'
1475        name = 'OnlineTransactions'
1476        no_import = []
1477        imported = []
1478        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1479        try:
1480            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1481        except:
1482            logger.error('Error reading %s.csv' % name)
1483            return
1484        for pay_transaction in transactions:
1485            if start:
1486                start = False
1487                logger.info('Start loading from %s.csv' % name)
1488                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1489                no_import.append('%s,"Error"' % s)
1490                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1491                format_error = format + ',"%(Error)s"'
1492            data = {}
1493
1494            # format of the first file sent by Tayo
1495            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1496            #data['student_id'] = student_id = pay_transaction['Payer ID']
1497            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1498            #data['response_code'] = response_code = pay_transaction['Resp Code']
1499            #data['amount'] = amount = pay_transaction['Amount']
1500
1501            # format of the second file sent by Tayo
1502            #data['datetime'] = date = 0
1503            #data['student_id'] = student_id = pay_transaction['Payer ID']
1504            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1505            #data['response_code'] = response_code = '00'
1506            #data['amount'] = amount = pay_transaction['Amount']
1507
1508            # format of the third file sent by Kehinde
1509            data['datetime'] = date = 0
1510            data['student_id'] = student_id = pay_transaction['customer_id']
1511            data['order_id'] = order_id = pay_transaction['merchant_reference']
1512            data['response_code'] = response_code = '00'
1513            data['amount'] = amount = pay_transaction['Amount']
1514
1515            dup = False
1516            if response_code == "12":
1517                continue
1518            try:
1519                opt.addRecord(**data)
1520            except ValueError:
1521                dup = True
1522            #from pdb import set_trace;set_trace()
1523            if dup:
1524                if response_code == "00":
1525                    try:
1526                        opt.modifyRecord(**data)
1527                    except:
1528                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1529                        continue
1530                else:
1531                    pay_transaction['Error'] = "Duplicate order_id"
1532                    no_import.append( format_error % pay_transaction)
1533                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1534                    continue
1535            tr_count += 1
1536            if tr_count > 1000:
1537                if len(no_import) > 0:
1538                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1539                             '\n'.join(no_import) + '\n')
1540                    no_import = []
1541                em = '%d transactions committed\n' % (tr_count)
1542                transaction.commit()
1543                regs = []
1544                logger.info(em)
1545                total += tr_count
1546                tr_count = 0
1547        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1548                                                '\n'.join(no_import))
1549        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1550    ###)
1551
1552    security.declareProtected(ModifyPortalContent,"importData")###(
1553    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1554        """load data from CSV values"""
1555        import transaction
1556        import random
1557        students_folder = self.portal_url.getPortalObject().campus.students
1558        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1559        pending_only = False
1560        pend_str = '--'
1561        elapse = time.time()
1562        #
1563        # preparations
1564        #
1565        if filename == pend_str:
1566            pending_only = True
1567        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1568        importer = eval("%sImport" % importer_name)(self)
1569        logger = importer.logger
1570        if importer.init_errors:
1571            logger.info(importer.init_errors)
1572            return importer.init_errors
1573        member = importer.member
1574        #current = importer.current
1575        import_date = importer.import_date
1576        #
1577        # not_imported
1578        #
1579        info = importer.info
1580        data_keys = importer.data_keys
1581        csv_keys = importer.csv_keys
1582        #csv_keys.extend(info.keys())
1583        headline_mapping = dict((k,k) for k in csv_keys)
1584        #
1585        # pending
1586        #
1587        pending_path = importer.pending_path
1588        pending_tmp = importer.pending_tmp
1589        pending_backup = importer.pending_backup
1590        pending_fn = importer.pending_fn
1591        imported_path = importer.imported_path
1592        imported_fn = importer.imported_fn
1593        commit_after = importer.commit_after
1594        pending = []
1595        pending_digests = []
1596        #total_added_to_pending = 0
1597        if not pending_only:
1598            pending,pending_digests = importer.makeIdLists()
1599            pending_at_start = len(pending)
1600        datafile = open(pending_tmp,"w")
1601        pending_csv_writer = csv.DictWriter(datafile,
1602                                                    csv_keys,
1603                                                    extrasaction='ignore')
1604        pending_csv_writer.writerow(headline_mapping)
1605        datafile.close()
1606        #
1607        # imported
1608        #
1609        if not os.path.exists(imported_path):
1610            datafile = open(imported_path,"w")
1611            imported_csv_writer = csv.DictWriter(datafile,
1612                                                 csv_keys,
1613                                                 extrasaction='ignore')
1614            imported_csv_writer.writerow(headline_mapping)
1615            datafile.close()
1616        start = True
1617        tr_count = 0
1618        total = 0
1619        total_added_to_imported = 0
1620        total_pending = 0
1621        import_source_done = ""
1622        if pending_only:
1623            import_source_path = pending_path
1624        else:
1625            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1626            import_source_done = "%s/import/%s.done" % (i_home,filename)
1627        if not os.path.exists(import_source_path):
1628            fn = os.path.split(import_source_path)[1]
1629            em = 'no import file %(fn)s' % vars()
1630            return em
1631        import_source_fn = os.path.split(import_source_path)[1]
1632        if not pending_only:
1633            info['imported_from'] = import_source_fn
1634        headline = csv.reader(open(import_source_path,"rb")).next()
1635        if "import_mode" not in headline:
1636            msg = 'import_mode must be in heading'
1637            return msg
1638        invalid_keys = importer.checkHeadline(headline)
1639        if invalid_keys:
1640            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1641
1642        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1643                                                         or k.strip() in info.keys())]
1644        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1645        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1646        # if diff2schema and diff2schema != set(['id',]):
1647        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1648        #     return msg
1649        #
1650        # start importing
1651        #
1652        try:
1653            reader = csv.DictReader(open(import_source_path,"rb"))
1654        except:
1655            msg = 'Error reading %s.csv' % filename
1656            logger.error(msg)
1657            return msg
1658        items = [item for item in reader]
1659        total_to_import = len(items)
1660        tti_float = float(total_to_import)
1661        if pending_only:
1662            pending_at_start = total_to_import
1663        count = 0
1664        imported = []
1665        old_commit_count = 0
1666        error_count = imported_count = 0
1667        already_in = 0
1668        for record in items:
1669            item = {}
1670            empty_value_keys = []
1671            for k,v in record.items():
1672                if k is None:
1673                    continue
1674                if v:
1675                    if v == EMPTY:
1676                        empty_value_keys += k,
1677                        v = ''
1678                    item[k.strip()] = v.strip()
1679            count += 1
1680            if start:
1681                start = False
1682                adapters = [MappingStorageAdapter(importer.schema, item)]
1683                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1684            dm = DataModel(item, adapters,context=self)
1685            ds = DataStructure(data=item,datamodel=dm)
1686            error_string = ""
1687            total += 1
1688            import_mode = item.get('import_mode','')
1689            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1690            if import_method is None:
1691                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1692            elif (import_mode in importer.required_modes and
1693                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1694                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1695                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1696            else:
1697                for k in import_keys:
1698                    if k not in item.keys() or k not in importer.validators.keys():
1699                        continue
1700                    if not importer.validators[k](ds,mode=import_mode):
1701                        error_string += ' ++ '
1702                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1703                                                                           ds.getErrorMapping(k)))
1704            if error_string:
1705                error = error_string
1706                id = ''
1707                mapping = item
1708            else:
1709                temp_item = item.copy()
1710                temp_item.update(dm)
1711                results = import_method(temp_item)
1712                id = results[0]
1713                error = results[1]
1714                mapping = results[2]
1715            #
1716            # restore some values
1717            #
1718            for k in empty_value_keys:
1719                mapping[k] = EMPTY
1720            if mapping.has_key('sex'):
1721                #import pdb;pdb.set_trace()
1722                if mapping['sex'] == True:
1723                    mapping['sex'] = 'F'
1724                elif mapping['sex'] == False:
1725                    mapping['sex'] = 'M'
1726            if pending_only:
1727                info['imported_from'] = item['imported_from']
1728            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1729            info['error'] = error
1730            info['import_record_no'] = count + 1
1731            mapping.update(info)
1732            log_list = []
1733            if error:
1734                error_count += 1
1735                digest = makeDigest(mapping,data_keys)
1736                if digest not in pending_digests:
1737                    pending_digests += digest,
1738                    pending.append(mapping)
1739                    if not pending_only:
1740                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1741                else:
1742                    already_in += 1
1743                    pass
1744            else:
1745                imported_count += 1
1746                imported += mapping,
1747                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1748            if log_list:
1749                time_till_now = time.time() - elapse
1750                percent_finished = (error_count + imported_count)/tti_float*100
1751                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1752                logger.info(' '.join(log_list))
1753            finished = count > total_to_import - 1
1754            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1755            if must_commit:
1756                old_commit_count = imported_count
1757
1758            if must_commit or finished:
1759                if len(imported):
1760                    transaction.commit()
1761                    datafile = open(imported_path,"a")
1762                    writer = csv.DictWriter(datafile,
1763                                            csv_keys,
1764                                            extrasaction='ignore')
1765                    writer.writerows(imported)
1766                    datafile.close()
1767                    total_added_to_imported += len(imported)
1768                    imported = []
1769                if len(pending) > 0:
1770                    datafile = open(pending_tmp,"a")
1771                    writer = csv.DictWriter(datafile,
1772                                            csv_keys,
1773                                            extrasaction='ignore')
1774                    writer.writerows(pending)
1775                    datafile.close()
1776                    total_pending += len(pending)
1777                    #total_added_to_pending += len(pending)
1778                    pending = []
1779                if not finished:
1780                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1781                    logger.info(msg)
1782        elapse = time.time() - elapse
1783        if os.path.exists(pending_path):
1784            copy2(pending_path,pending_backup)
1785        copy2(pending_tmp,pending_path)
1786        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1787        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1788        if pending_only:
1789            removed_pending = pending_at_start - total_pending
1790            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1791        else:
1792            added_pending = total_pending - pending_at_start
1793            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1794        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1795        logger.info(msg)
1796        if import_source_done:
1797            copy(import_source_path,import_source_done)
1798            os.remove(import_source_path)
1799            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1800            if upload is not None:
1801                upload_doc = upload.getContent()
1802                mapping = {}
1803                #mapping['import_date'] = DateTime.DateTime()
1804                mapping['import_date'] = import_date
1805                mapping['imported_by'] = importer.imported_by
1806                mapping['import_message'] = msg
1807                upload_doc.edit(mapping = mapping)
1808        os.remove(pending_tmp)
1809        return msg
1810    ###)
1811
1812    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1813    def moveImagesToFS(self,student_id="O738726"):
1814        "move the images to the filesystem"
1815        images_dir = getImagesDir(student_id)
1816        #images_dir = os.path.join("%s" % images_base,student_id)
1817        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1818        stool = getToolByName(self, 'portal_schemas')
1819        schemas = ['student_application',
1820                   'student_clearance',
1821                   ]
1822        created = False
1823        for schema_id in schemas:
1824            schema = stool._getOb(schema_id)
1825            object = getattr(student_folder,schema_id[len('student_'):],None)
1826            if object is None:
1827                continue
1828            doc = object.getContent()
1829            for key in schema.keys():
1830                if schema[key].meta_type != "CPS Image Field":
1831                    continue
1832                #import pdb;pdb.set_trace()
1833                image = getattr(doc,key,None)
1834                if not image or not hasattr(image,"data"):
1835                    continue
1836                if not created:
1837                    if not os.path.exists(images_dir):
1838                        os.mkdir(images_dir)
1839                    created = True
1840                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1841                open(filename,"wb").write(str(image.data))
1842    ###)
1843
1844    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1845    def movePassportToFS(self,student_id="O738726"):
1846        "move the passports to the filesystem"
1847        images_dir = os.path.join("%s" % i_home,'passports')
1848        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1849        stool = getToolByName(self, 'portal_schemas')
1850        schemas = ['student_application',
1851                   #'student_clearance',
1852                   ]
1853        created = False
1854        for schema_id in schemas:
1855            schema = stool._getOb(schema_id)
1856            object = getattr(student_folder,schema_id[len('student_'):],None)
1857            if object is None:
1858                continue
1859            doc = object.getContent()
1860            for key in schema.keys():
1861                if schema[key].meta_type != "CPS Image Field":
1862                    continue
1863                #import pdb;pdb.set_trace()
1864                image = getattr(doc,key)
1865                if not hasattr(image,"data"):
1866                    continue
1867                if not created:
1868                    if not os.path.exists(images_dir):
1869                        os.mkdir(images_dir)
1870                    created = True
1871                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1872                open(filename,"wb").write(str(image.data))
1873    ###)
1874
1875InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.