source: WAeUP_SRP/base/WAeUPTool.py @ 3272

Last change on this file since 3272 was 3272, checked in by Henrik Bettermann, 17 years ago

implement export of students directory

  • Property svn:keywords set to Id
File size: 68.5 KB
RevLine 
[3219]1# -*- mode: python; mode: fold; -*-
[197]2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3272 2008-03-05 21:57:35Z henrik $
[1174]20"""The WAeUP Tool Box.
[197]21"""
22
23from AccessControl import ClassSecurityInfo
[828]24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
[2283]29from zExceptions import BadRequest
[197]30
[1890]31from Products.CMFCore.utils import getToolByName
[1747]32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
[828]35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
[1890]39from Products.CMFCore.permissions import ManagePortal
[197]40from Products.CMFCore.utils import UniqueObject
[1194]41from Products.CMFCore.URLTool import URLTool
[1747]42from Products.CMFCore.utils import getToolByName
[1151]43from Students import makeCertificateCode
[1285]44from Globals import package_home,INSTANCE_HOME
[3172]45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
[2094]49import DateTime,time
[1620]50import logging
[1170]51import transaction
[2193]52import csv,re,os,sys
[3172]53import md5
[2413]54from shutil import copy2
[1707]55from Products.AdvancedQuery import Eq, Between, Le,In
[197]56
[2413]57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
[3198]60EMPTY = 'XXX'
[2413]61
[2675]62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
[2413]64
[1707]65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
[1720]69
[828]70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
[197]71    """WAeUP tool"""
72
[828]73    id = 'waeup_tool'
[197]74    meta_type = 'WAeUP Tool'
[828]75    _actions = ()
[197]76    security = ClassSecurityInfo()
[828]77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
[2999]82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
[3014]85    ###)
[2999]86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
[3014]90    ###)
[2999]91
[1818]92    def rwrite(self,s): ###(
[1707]93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
[1818]96    ###)
[1174]97
[2695]98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
[1818]103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
[1827]108
[1818]109###)
110
[2695]111    security.declareProtected(View,'updateRoleMappingsFor') ###(
112    def updateRoleMappingsFor(self,wf_definition,ob):
113        "do so for public"
114        wf_def = getattr(self.portal_workflow,wf_definition)
115        wf_def.updateRoleMappingsFor(ob)
116    ###)
[2699]117
[2999]118    security.declareProtected(View,'getStatesLgas') ###(
119    def getStatesLgas(self):
120        """return lga info"""
121        voc = getattr(self.portal_vocabularies,'local_gov_areas')
122        states = []
123        lgas  = []
124        d = {}
125        wd = {}
126        for k,v in voc.items():
127            parts = v.split(' / ')
128            if len(parts) == 1:
129                state = parts[0].lower()
130                lga = ""
131            elif len(parts) == 2:
132                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134            else:
135                continue
136            if state not in states:
137                states.append(state)
138            if lga not in lgas:
139                lgas.append(lga)
140            words = re.split('[^a-zA-Z0-9/]',k)
141            words.sort()
142            wd[k] = words
143            d[k] = v
144        mapping = {}
145        mapping['word_dict'] = wd
146        mapping['lga_dict'] = d
147        mapping['states'] = states
148        mapping['lgas'] = lgas
149        return mapping
150    ###)
151
[3014]152    security.declareProtected(View,'findLga') ###(
153    def findLga(self,words,words_dict):
154        words = re.split('[^a-zA-Z0-9/]',words)
155        lga_words = []
156        for word in words:
157            if word:
158                lga_words += word.strip().lower(),
159        lga_words.sort()
160        state_lga = ''
161        while not state_lga:
162            for k,l in words_dict.items():
163                if lga_words == l:
164                    state_lga = k
165                    break
166            break
167        return state_lga
168    ###)
[2975]169    security.declareProtected(View,'getAccessInfo') ###(
170    def getAccessInfo(self,context):
171        "return a dict with access_info"
172        logger = logging.getLogger('WAeUPTool.getAccessInfo')
173        mtool = self.portal_membership
174        member = mtool.getAuthenticatedMember()
175        member_id = str(member)
176        info = {}
177        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179        is_staff = info['is_staff'] = not is_anonymous and not is_student
180        roles = member.getRolesInContext(context)
181        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182                                                                            "SectionManager" in roles or
183                                                                            "Manager" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
[3025]187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif is_anonymous or member_id != requested_id:
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
[2975]194        info['student_id'] = student_id
195        return info
196    ###)
197
[1818]198    security.declareProtected(ModifyPortalContent,'openLog') ###(
[1716]199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
[2259]208    ###)
[1716]209
[2259]210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
[1818]215
[2259]216    ###)
217
[2094]218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
[2119]335                    t_min = t_per
[2094]336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
[2119]342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
[2094]344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
[2119]363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
[2094]364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
[1818]374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
[1716]375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378
[1818]379###)
[1720]380
[1151]381    def generateStudentId(self,letter): ###(
382        import random
383        r = random
[1194]384        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385        if letter == '?':
[2695]386            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
[1151]387        sid = "%c%d" % (letter,r.randint(99999,1000000))
[1720]388        students = self.portal_url.getPortalObject().campus.students
[1721]389##        while hasattr(students, sid):
390##            sid = "%c%d" % (letter,r.randint(99999,1000000))
391        while self.students_catalog(id = sid):
[1720]392            sid = "%c%d" % (letter,r.randint(99999,1000000))
[1151]393        return sid
[1460]394    ###)
[1415]395
396    def generatePassword(self,s=None): ###(
397        import random
398        r = random
399        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400        if s is None:
401            s = 'abcdefghklmnpqrstuvwxy23456789'
402        pw = ''
403        while len(pw) < 6:
404            pw += r.choice(s)
405        return pw
[1151]406    ###)
[828]407
[2645]408    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409    def dumpSchoolfeePayments(self):
410        "dump paid schoolfees"
[2666]411        mtool = self.portal_membership
412        member = mtool.getAuthenticatedMember()
[2645]413        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414        aq_student = self.students_catalog.evalAdvancedQuery
415        query = In('review_state',('schoolfee_paid',
416                                   'courses_registered',
417                                   'courses_validated',
418                                   ))
419        res = aq_student(query)
[2666]420        #import pdb;pdb.set_trace()
[2645]421        l = []
422        logger.info("start for %d" % len(res))
423        count = 1
424        log_after = 100
425        for student in res:
[2656]426            if not count % log_after:
[2645]427                logger.info("processed %d total %d" % (log_after,count))
428            count += 1
429            fee_dict =self.getSchoolFee(student)
430            fulltime = student.mode.endswith('_ft')
431            d = {}
432            d['student_id'] = student.id
[2666]433            d['name'] = student.name
434            d['amount'] = fee_dict.get(new_returning)
[2645]435            l += d,
[2666]436        csv_name = self.dumpListToCSV(l,'payments')
437        logger.info('%s dumped payments to %s' % (member,export_file))
[2645]438    ###)
439
[2666]440    security.declarePublic('dumpListToCSV') ###(
441    def dumpListToCSV(self,l,filename,fields=None):
442        """dump a list of dicts to a CSV file"""
443        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445        if fields is None:
446            fields = l[0].keys()
447        headline = ','.join(fields)
448        out = open(export_file,"wb")
449        out.write(headline +'\n')
450        out.close()
451        out = open(export_file,"a")
452        csv_writer = csv.DictWriter(out,fields,)
453        csv_writer.writerows(l)
454        return export_file
455    ###)
456
[2643]457    security.declareProtected(ManagePortal, 'listMembers') ###(
458    def listMembers(self):
459        "list all members"
460        mtool = self.portal_membership
461        member = mtool.getAuthenticatedMember()
462        logger = logging.getLogger('WAeUPTool.listMembers')
463        if str(member) not in ('admin','joachim'):
464            logger.info('%s tried to list members' % (member))
465            return None
466        members = self.portal_directories.members
467        all = members.listEntryIdsAndTitles()
468        l = []
469        for user_id,name in all:
470            d = {}
471            d['user_id'] = user_id
472            d['name'] = name
473            d['pw'] = getattr(getattr(members,user_id),'password')
[2645]474            d['email'] = getattr(getattr(members,user_id),'email')
475            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
[2643]477            l += d,
478        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
[3272]480        logger.info('%s dumped member list to %s' % (member,export_file))
[2643]481        fields = l[0].keys()
482        headline = ','.join(fields)
483        out = open(export_file,"wb")
484        out.write(headline +'\n')
485        out.close()
486        out = open(export_file,"a")
487        csv_writer = csv.DictWriter(out,fields,)
488        csv_writer.writerows(l)
489    ###)
[3272]490   
491    security.declareProtected(ManagePortal, 'listStudents') ###(
492    def listStudents(self):
493        "list all students"
494        mtool = self.portal_membership
495        member = mtool.getAuthenticatedMember()
496        logger = logging.getLogger('WAeUPTool.listStudents')
497        if str(member) not in ('admin','joachim'):
498            logger.info('%s tried to list students' % (member))
499            return None
500        students = self.portal_directories.students
501        all = students.listEntryIdsAndTitles()
502        l = []
503        for user_id,name in all:
504            d = {}
505            d['user_id'] = user_id
506            d['name'] = name
507            d['pw'] = getattr(getattr(students,user_id),'password')
508            d['email'] = getattr(getattr(students,user_id),'email')
509            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511            l += d,
512        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514        logger.info('%s dumped student list to %s' % (member,export_file))
515        fields = l[0].keys()
516        headline = ','.join(fields)
517        out = open(export_file,"wb")
518        out.write(headline +'\n')
519        out.close()
520        out = open(export_file,"a")
521        csv_writer = csv.DictWriter(out,fields,)
522        csv_writer.writerows(l)
523    ###)   
[2656]524
[1890]525    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526    def removeDeletedDocIds(self, max=1000):
527        """
528        remove deleted docids from repository commit after max
529        """
530        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531        repository = getToolByName(self, 'portal_repository')
532        pxtool = getToolByName(self, 'portal_proxies')
[2632]533        logger.info('start')
[1890]534        pxtool_infos = pxtool.getRevisionsUsed()
[2632]535        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
[1895]536
[1890]537        nb_revs = 0
538        docids_d = {} # all docids
539        unused_docids_d = {} # all docids that are unused
540        ids_unused_revs_docids = [] # ids for revs of unused docids
[2632]541        unused_ids = [] # ids for unused revs
[1890]542        total = 0
[1895]543        idlist = repository.objectIds()
[2632]544        to_delete = 0
545        found = False
[1895]546        for id in idlist:
[1890]547            docid, rev = repository._splitId(id)
548            if docid is None:
549                logger.info("invalid doc_id %s" % docid)
550                continue
551            nb_revs += 1
552            if not pxtool_infos.has_key(docid):
[2632]553                found = True
554                to_delete += 1
555                unused_ids.append(id)
[1890]556            elif not pxtool_infos[docid].has_key(rev):
[2632]557                found = True
558                to_delete += 1
559                unused_ids.append(id)
560            if found and not to_delete % max:
561                found = False
[1890]562                #import pdb;pdb.set_trace()
[2632]563                repository.manage_delObjects(unused_ids)
[1890]564                transaction.commit()
[2632]565                logger.info('removed %d total %d unused docids ' % (max,to_delete))
566        else:
567            if unused_ids:
568                repository.manage_delObjects(unused_ids)
569                transaction.commit()
570        logger.info('finished removing %d unused docids ' % (to_delete))
[1890]571
[1895]572
[1890]573###)
574
[2562]575    security.declareProtected(View,'getCredential') ###(
[1250]576    def getCredential(self,student_id):
577        student_entry = getattr(self.portal_directories.students,student_id,None)
[2555]578        if not self.isStaff():
[2563]579            mtool = self.portal_membership
580            member = mtool.getAuthenticatedMember()
581            logger = logging.getLogger('WAeUPTool.getCredential')
582            logger.info('%s tried to access password of %s' % (member,student_id))
[2555]583            return None
[1250]584        if student_entry is None:
585            return None
[1263]586        return getattr(student_entry,"password","not set")
[1261]587    ###)
[1250]588
[2747]589    security.declarePublic('checkPassword') ###(
[1467]590    def checkPassword(self,student_id,password):
[1460]591        student_entry = getattr(self.portal_directories.students,student_id,None)
592        if student_entry is None:
593            return False
594        return getattr(student_entry,"password","not set") == password
[2747]595    ###)
[2712]596
[2747]597    security.declarePublic('checkGenericPassword') ###(
[2710]598    def checkGenericPassword(self,member_id):
599        member_entry = getattr(self.portal_directories.members,member_id,None)
600        if member_entry is None:
601            return False
[2712]602        ltool = getToolByName(self, 'portal_layouts')
[2710]603        unsecure_words = ltool._getOb('members')['w__password'].check_words
[2712]604        password = getattr(member_entry,"password","not set")
605        is_unsecure = password in unsecure_words
606        if is_unsecure:
[2713]607            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
[2712]608            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609        return is_unsecure
[2747]610    ###)
[1460]611
[2556]612    security.declareProtected(ModifyPortalContent,'editPassword') ###(
[1467]613    def editPassword(self,student_id,password):
614        "edit a student password"
615        student_entry = getattr(self.portal_directories.students,student_id,None)
616        if student_entry is None:
[1571]617            return
[1467]618        setattr(student_entry,'password',password)
619    ###)
620
[2555]621    security.declareProtected(ModifyPortalContent,'doCommit') ###(
[1401]622    def doCommit(self,logger=None):
623        "commit some transactions"
624        transaction.commit()
625    ###)
626
[1285]627    security.declarePublic('loadStudentFoto') ###(
[1813]628    def loadStudentFoto(self,student,filename,folder):
[1285]629        "return a student passport picture"
[2413]630        #import pdb;pdb.set_trace()
631        picture ="%s/import/%s/%s" % (i_home,folder,filename)
632        student_id = student.getId()
[2675]633        images_dir = getImagesDir(student_id)
[2413]634        if not os.path.exists(images_dir):
635            os.mkdir(images_dir)
636        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637        for extension in ('.jpg','.JPG'):
638            fullname = "%(picture)s%(extension)s" % vars()
639            if os.path.exists(fullname):
640                copy2(fullname,image_name)
641                return "successfully copied passport picture"
[2467]642        return "passport picture not found: %s.jpg or .JPG" % picture
[2413]643    ###)
644
[2420]645    def old____loadStudentFoto(self,student,filename,folder): ###(
[2413]646        "return a student passport picture"
[1921]647        app = student.application
648        app_doc = app.getContent()
[1813]649        #clear = student.clearance
650        #clear_doc = clear.getContent()
651        #matric_no = clear_doc.matric_no.upper()
652        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
[1285]654        #import pdb;pdb.set_trace()
[1286]655        if os.path.exists(picture1):
656            file = open(picture1)
657        elif os.path.exists(picture2):
[1287]658            file = open(picture2)
[1286]659        else:
[1287]660            return "passport picture not found %s" % picture1
[1921]661        reopened = False
662        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663            self.portal_workflow.doActionFor(app,'open')
664            reopened = True
[1285]665        outfile = file.read()
666        app_doc.manage_addFile('passport',
667                               file=outfile,
[1818]668                               title="%s.jpg" % filename)
[1921]669        if reopened:
670            self.portal_workflow.doActionFor(app,'close')
[1286]671        return "successfully loaded passport picture"
[1285]672    ###)
673
[1170]674    security.declareProtected(ModifyPortalContent,'createOne') ###(
[1194]675    def createOne(self,students_folder,student_brain,letter,commit=False):
676        sid = self.waeup_tool.generateStudentId(letter)
[1170]677        students_folder.invokeFactory('Student', sid)
678        student = getattr(students_folder,sid)
679        self.portal_workflow.doActionFor(student,'return')
680        student.manage_setLocalRoles(sid, ['Owner',])
681        matric_no = student_brain.matric_no
682        jamb_reg_no = student_brain.Entryregno
683        self.students_catalog.addRecord(id = sid,
684                                           matric_no = matric_no,
685                                           jamb_reg_no = jamb_reg_no,
686                                           sex = student_brain.Sex == "F",
687                                           name = "%s %s %s" % (student_brain.Firstname,
688                                                                student_brain.Middlename,
689                                                                student_brain.Lastname)
690                                        )
691        if commit:
692            transaction.commit()
693        return sid,jamb_reg_no
694    ###)
695
[1752]696    security.declareProtected(ModifyPortalContent,'addStudent') ###(
697    def addStudent(self,dict):
[1415]698        students_folder = self.portal_url.getPortalObject().campus.students
699        sid = self.waeup_tool.generateStudentId('?')
700        students_folder.invokeFactory('Student', sid)
701        student_obj = getattr(students_folder,sid)
[3027]702        f2t = self.field2types_student
[1772]703        #from pdb import set_trace; set_trace()
704        d = {}
[2297]705        #d['jamb_sex']  = 'M'
706        #if dict.get('sex'):
707        #    d['jamb_sex']  = 'F'
[1904]708
[1898]709        entry_session = dict.get('entry_session')
[2454]710        if entry_session == self.getSessionId()[0]:
[1898]711            wfaction = 'admit'
[1991]712            wft = 'wf_transition_admit'
[1900]713            password = None
[1904]714        else:
[1900]715            wfaction = 'return'
[1991]716            wft = 'wf_transition_return'
[1900]717            password = self.generatePassword()
[1904]718            self.makeStudentMember(sid,password)
719
[1728]720        for pt in f2t.keys():
721            student_obj.invokeFactory(pt,f2t[pt]['id'])
722            sub_obj = getattr(student_obj,f2t[pt]['id'])
723            sub_doc = sub_obj.getContent()
[1898]724            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
[1772]725            d['Title'] = f2t[pt]['title']
[1728]726            for field in f2t[pt]['fields']:
[1749]727                d[field] = dict.get(field,'')
[1728]728            sub_doc.edit(mapping = d)
[1898]729            new_state = f2t[pt][wft]
[1772]730            if new_state != "remain":
731                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
[1415]732        self.portal_workflow.doActionFor(student_obj,wfaction)
733        student_obj.manage_setLocalRoles(sid, ['Owner',])
734        return sid,password
735    ###)
736
[1151]737    security.declarePublic('getCertificateBrain') ###(
738    def getCertificateBrain(self,cert_id):
739        "do it"
[1849]740        res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]741                                {'portal_type':"Certificate",
742                                      'id': cert_id})
743        if res:
744            return res[0]
745        return None
746    ###)
[1160]747
[1756]748    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749    def get_csv_filenames(self):
750        "do it"
[1759]751        files = [file for file in os.listdir("%s/import/" % (i_home))
[3143]752                 if file.endswith('.csv') and (file.find('imported') == -1 and
753                                               file.find('pending') == -1)]
[1756]754        return files
755    ###)
756
[1151]757    security.declarePublic('findStudentByMatricelNo') ###(
758    def findStudentByMatricelNo(self,matric_no):
759        "do it"
[1849]760        res = ZCatalog.searchResults(self.portal_catalog_real,
[1151]761                                {'portal_type':"StudentClearance",
762                                 'SearchableText': matric_no})
763        if res:
764            return res[0]
765        return None
766    ###)
767
[2557]768    security.declarePublic('makeStudentMember') ###(
[1151]769    def makeStudentMember(self,sid,password='uNsEt'):
770        """make the student a member"""
771        membership = self.portal_membership
772        membership.addMember(sid,
773                             password ,
774                             roles=('Member',
775                                     'Student',
776                                     ),
777                             domains='',
778                             properties = {'memberareaCreationFlag': False,
779                                           'homeless': True},)
780        member = membership.getMemberById(sid)
781        self.portal_registration.afterAdd(member, sid, password, None)
[1261]782        #self.manage_setLocalRoles(sid, ['Owner',])
[1151]783    ###)
[1160]784
[2560]785    security.declareProtected(View,'makeStudentData') ###(
[1158]786    def makeStudentData(self,student_id,email=None,phone_nr=None):
[1151]787        "create Datastructure for a returning Student"
[1406]788        #import pdb;pdb.set_trace()
[1571]789        logger = logging.getLogger('WAeUPTool.makeStudentData')
[1151]790        students_folder = self.portal_url.getPortalObject().campus.students
[1794]791        #res = self.students_catalog(id=student_id)
792        #if res:
793        #    st = res[0]
794        #res = self.returning_import(matric_no = st.matric_no)
[1798]795        res = self.returning_import(id = student_id)
[1151]796        if res:
[1160]797            student = res[0]
[1794]798        else:
[1816]799            logger.info('Id %s not found in returning_import' % student_id)
[1794]800            return
[1580]801        logger.info('%s creates data structure' % student_id)
[1794]802        s_results = self.results_import(matric_no = student.matric_no)
[1816]803        if s_results:
804            lnr = self.getLevelFromResultsCosCode(s_results)
805            level = "%d00" % lnr
806            verdict,eligible = self.getVerdict(s_results[0].Verdict)
[2454]807            #if eligible:
808            #    level = "%d00" % (lnr + 1)
[1816]809        else:
810            logger.info('matric_no %s not found in results_import' % student.matric_no)
[2656]811            level = ''
812            verdict = ''
[1171]813        #student should not be allowed to perform this transition
[1174]814        #wftool = self.portal_workflow
815        #wftool.doActionFor(student,'return')
[1151]816        certcode_org = student.Coursemajorcode
817        certcode = makeCertificateCode(certcode_org)
818        certificate_brain = self.getCertificateBrain(certcode)
819        if not certificate_brain:
820            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821            logger.info(em)
822        matric_no = student.matric_no
823        sid = student_id
824        student_obj = getattr(students_folder,sid)
[3114]825        if not getattr(student_obj,'application'):
826            student_obj.invokeFactory('StudentApplication','application')
[1151]827        application = student_obj.application
[1169]828        self.portal_workflow.doActionFor(application,'open',dest_container=application)
[1151]829        da = {'Title': 'Application Data'}
830        student_obj.invokeFactory('StudentPersonal','personal')
831        da['jamb_reg_no'] = student.Entryregno
[1462]832        em = self.getEntryMode(student.Entryregno)
[1401]833        da['entry_mode'] = em
[1151]834        personal = student_obj.personal
835        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836        dp = {'Title': 'Personal Data'}
837        student_obj.invokeFactory('StudentClearance','clearance')
838        clearance = student_obj.clearance
[1169]839        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[1151]840        dc = {'Title': 'Clearance/Eligibility Record'}
841        dc['matric_no'] = matric_no
842        state = student.State
843        lga = student.LGA
844        if state and lga:
845            lga =  state + ' / ' + lga
846        else:
847            lga = "None"
[1174]848        da['jamb_lga'] = dc['lga'] = lga
[1173]849        da['app_email'] = dp['email'] = email
850        da['app_mobile'] = dp['phone'] = phone_nr
[1411]851        dp['firstname'] = student.Firstname
852        dp['middlename'] = student.Middlename
853        dp['lastname'] = student.Lastname
854        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
[1174]855        da['jamb_sex'] = student.Sex
[1151]856        dp['sex'] = student.Sex == 'F'
857        dp['perm_address'] = student.Permanent_Address
858        application.getContent().edit(mapping=da)
[1169]859        self.portal_workflow.doActionFor(application,'close',dest_container=application)
[1151]860        personal.getContent().edit(mapping=dp)
861        clearance.getContent().edit(mapping=dc)
[1169]862        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[1151]863        #
864        # Study Course
865        #
866        student_obj.invokeFactory('StudentStudyCourse','study_course')
867        studycourse = student_obj.study_course
868        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869        dsc = {}
870        dsc['study_course'] = certcode
[1401]871        dsc['current_level'] = level
872        dsc['current_verdict'] = verdict
[1827]873        dsc['current_mode'] = em
[2466]874        dsc['current_session'] = '05'
[1151]875        studycourse.getContent().edit(mapping=dsc)
876        #
877        # Level
878        #
[1198]879##        l = getattr(studycourse,level,None)
880##        if l is None:
881##            studycourse.invokeFactory('StudentStudyLevel', level)
882##            l = getattr(studycourse, level)
883##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
884##            l.getContent().edit(mapping={'Title': "Level %s" % level})
[1194]885###)
[1160]886
[2556]887    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
[2576]888    def admitOneStudent(self,brain,entry_session,pin_password):
[2514]889        "create Datastructure for an admitted Student"
890        #import pdb;pdb.set_trace()
[2727]891        if not hasattr(self,"_v_certificates"):
892            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
[2514]893        students_folder = self.portal_url.getPortalObject().campus.students
[2716]894        logger = logging.getLogger('WAeUPTool.admitOneStudent')
[3264]895        if brain.screening_type in ('cest','sandwich',):
896            reg_no = "%s%s/07" % (brain.course1,brain.serial)
897        else:
898            reg_no = brain.reg_no
899        res = self.students_catalog(jamb_reg_no = reg_no)
[3198]900        if res:
[3264]901            logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
[3198]902            return
[2514]903        if brain.status != "admitted":
[3264]904            logger.info('status of %s is %s' % (reg_no,brain.status))
[2540]905            return
[2514]906        pin_parts = brain.pin.split('-')
[2755]907        if pin_parts and len(pin_parts) != 3:
[3264]908            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
[2540]909            return
[2727]910        if brain.course_admitted not in self._v_certificates:
[3264]911            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
[2727]912            return
[2959]913        if brain.sex not in (True,False):
[3264]914            logger.info('sex of %s not available' % (reg_no))
[3002]915            return
[2540]916        student_id = self.generateStudentId('?')
[2514]917        students_folder.invokeFactory('Student', student_id)
918        student_object = getattr(students_folder,student_id)
[2576]919        if pin_password:
920            password = pin_parts[2]
921            self.makeStudentMember(student_id,password = password)
[2540]922        student_object.manage_setLocalRoles(student_id, ['Owner',])
[3264]923        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
[2514]924        #
925        # application
926        #
[2540]927        student_object.invokeFactory('StudentApplication','application')
928        application = student_object.application
[2575]929        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
[2514]930        da = {'Title': 'Application Data'}
[3264]931        da['jamb_reg_no'] = reg_no
[2576]932
[2540]933        sex = 'M'
934        if brain.sex:
935            sex = 'F'
936        da['jamb_sex'] = sex
[2670]937        da['jamb_age'] = brain.jamb_age
[2575]938        #da['app_ac_pin'] = brain.pin
[2670]939        da['jamb_lga'] = brain.jamb_lga
940        da['jamb_state'] = brain.jamb_state
[2575]941        da['jamb_score'] = brain.aggregate
[2514]942        da['app_email'] = brain.email
[2540]943        da['app_mobile'] = brain.phone
[2575]944        if brain.entry_mode:
945            da['entry_mode'] = brain.entry_mode
946        elif brain.screening_type == 'pume':
947            da['entry_mode'] = 'ume_ft'
948        elif brain.screening_type == 'pde':
949            da['entry_mode'] = 'de_ft'
[2950]950        elif brain.screening_type == 'pce':
[2959]951            da['entry_mode'] = 'pce'
[2950]952        elif brain.screening_type == 'prence':
[2959]953            da['entry_mode'] = 'prence'
[2575]954        else:
[2921]955            da['entry_mode'] = ''
[2575]956        da['entry_session'] = entry_session
957        da['jamb_lastname'] = brain.lastname
958        da['jamb_middlename'] = brain.middlenames   # different field names!
[2670]959        da['jamb_firstname'] = brain.firstname
[2575]960        da['screening_application_date'] = brain.application_date
961        da['date_of_birth'] = brain.date_of_birth
962        da['jamb_first_cos'] = brain.course1
963        da['jamb_second_cos'] = brain.course2
964        da['course3'] = brain.course3
965        da['screening_type'] = brain.screening_type
966        da['screening_score'] = brain.screening_score
967        da['screening_date'] = brain.screening_date
968        da['hq_type'] = brain.hq_type
969        da['hq_grade'] = brain.hq_grade
970        da['aos'] = brain.aos
[2576]971
[2514]972        application.getContent().edit(mapping=da)
[2575]973        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
[2514]974        #
975        # personal
976        #
[2540]977        student_object.invokeFactory('StudentPersonal','personal')
978        personal = student_object.personal
[2575]979        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
[2514]980        dp = {'Title': 'Personal Data'}
981        dp['sex'] = brain.sex
982        dp['email'] = brain.email
[2540]983        dp['phone'] = brain.phone
[2514]984        dp['lastname'] = brain.lastname
[2575]985        dp['middlename'] = brain.middlenames   # different field names!
[2576]986        dp['firstname'] = brain.firstname
[2514]987        personal.getContent().edit(mapping=dp)
988        #
989        # clearance
990        #
[2540]991        student_object.invokeFactory('StudentClearance','clearance')
992        clearance = student_object.clearance
[2575]993        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
[2514]994        dc = {'Title': 'Clearance/Eligibility Record'}
995        dc['lga'] = brain.lga
[2670]996        dc['birthday'] = brain.date_of_birth
[2514]997        clearance.getContent().edit(mapping=dc)
[2575]998        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
[2514]999        #
1000        # study Course
1001        #
[2540]1002        student_object.invokeFactory('StudentStudyCourse','study_course')
1003        studycourse = student_object.study_course
[2575]1004        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
[2514]1005        dsc = {}
1006        dsc['study_course'] = brain.course_admitted
1007        dsc['current_verdict'] = ''
[2575]1008        dsc['current_mode'] = da['entry_mode']
1009        if da['entry_mode'].startswith('de'):
1010            dsc['current_level'] = '200'
[3040]1011        elif da['entry_mode'].startswith('pre'):
1012            dsc['current_level'] = '000'
[2575]1013        else:
1014            dsc['current_level'] = '100'
1015        dsc['current_session'] = entry_session
[2514]1016        studycourse.getContent().edit(mapping=dsc)
1017        #
1018        # payments folder
[2540]1019        student_object.invokeFactory('PaymentsFolder','payments')
1020        payments = getattr(student_object,'payments')
[2514]1021        dpay = {}
1022        dpay['Title'] = 'Payments'
1023        payments.getContent().edit(mapping=dpay)
[2540]1024        self.portal_workflow.doActionFor(payments,'open')
[2580]1025        #
1026        # passport foto
1027        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1028                                                              brain.screening_type,
1029                                                              brain.reg_no)
[2675]1030        images_dir = getImagesDir(student_id)
1031        #images_dir = os.path.join("%s" % images_base,student_id)
1032        letter_dir,student_dir = os.path.split(images_dir)
1033        if not os.path.exists(letter_dir):
1034            os.mkdir(letter_dir)
[2580]1035        if not os.path.exists(images_dir):
1036            os.mkdir(images_dir)
1037        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1038        if os.path.exists(app_picture):
1039            copy2(app_picture,image_name)
1040        else:
1041            logger.info('passport of %s/%s not found: %s' % (student_id,
1042                                                             brain.reg_no,
1043                                                             app_picture))
[2656]1044
[2514]1045        return student_id
1046    ###)
1047
[2556]1048    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
[1194]1049    def makeStudentLevel(self,student_id):
1050        "create the StudyLevel for a returning Student"
1051        #import pdb;pdb.set_trace()
[1571]1052        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
[1194]1053        students_folder = self.portal_url.getPortalObject().campus.students
1054        res = self.students_catalog(id=student_id)
1055        if res:
1056            st = res[0]
1057        course = st.course
1058        matric_no = st.matric_no
1059        level = st.level
1060        res = self.results_import(matric_no = matric_no)
1061        if res:
1062            results = res
[1571]1063        logger.info('%s creating Level %s' % (student_id,level))
[1194]1064        #
1065        # Level
1066        #
1067        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1068        studycourse = getattr(student_obj,"study_course",None)
1069        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1070        l = getattr(studycourse,level,None)
1071        if l is None:
1072            studycourse.invokeFactory('StudentStudyLevel', level)
1073            l = getattr(studycourse, level)
1074            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1075            l.getContent().edit(mapping={'Title': "Level %s" % level})
1076        ###)
1077
[2000]1078    security.declarePublic('getHallInfo') ###(
1079    def getHallInfo(self,bed):
1080        """return Hall Info"""
[828]1081        info = {}
1082        hall,block,room,letter = bed.split('_')
[1846]1083        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
[828]1084        if res and len(res) == 1:
1085            hall_brain = res[0]
1086            hall_doc = hall_brain.getObject().getContent()
1087        else:
1088            return info
1089        info['hall_title'] = hall_brain.Title
1090        info['maintenance_code'] = hall_doc.maintenance_code
[1846]1091        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
[828]1092        batch_doc = None
1093        for brain in res:
1094            if brain.id.startswith(info['maintenance_code']):
1095                batch_doc = brain.getObject().getContent()
1096                break
1097        if batch_doc is None:
[2913]1098            info['maintenance_fee'] = ''
[828]1099        else:
1100            info['maintenance_fee'] = batch_doc.cost
1101        return info
[1151]1102    ###)
[828]1103
[2396]1104    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1105    def removePictureFolder(self,student_id):
1106        """remove picture_folder by renaming it"""
1107        path = 'images'
1108        picture_path = os.path.join(i_home,path,student_id)
1109        if not os.path.exists(picture_path):
1110            return False
1111        os.rename(picture_path,picture_path + "_removed")
1112        return True
1113    ###)
1114
1115    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1116    def restorePictureFolder(self,student_id):
1117        """restore picture_folder by renaming it"""
1118        path = 'images'
1119        picture_path = os.path.join(i_home,path,student_id)
1120        if not os.path.exists(picture_path + "_removed"):
1121            return False
1122        os.rename(picture_path + "_removed",picture_path)
1123        return True
1124    ###)
1125
[2363]1126    security.declarePublic('picturesExist') ###(
[2420]1127    def picturesExist(self, ids,student_id=None):
[2363]1128        """check if pictures exist in the filesystem"""
[2420]1129        if student_id is None:
1130            student_id = self.getStudentId()
1131        if student_id is None:
1132            return False
[2675]1133        picture_path = getImagesDir(student_id)
1134        #picture_path = os.path.join(images_base,student_id)
[2363]1135        if not os.path.exists(picture_path):
1136            return False
1137        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1138        return set(ids).issubset(set(pictures))
1139    ###)
1140
[2364]1141    security.declarePublic('picturesList') ###(
1142    def picturesList(self):
1143        """check if pictures exist in the filesystem"""
1144        path = 'images'
1145        student_id = self.getStudentId()
[2677]1146        #picture_path = os.path.join(i_home,path,student_id)
1147        picture_path = getImagesDir(student_id)
[2364]1148        if not os.path.exists(picture_path):
1149            return []
1150        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1151    ###)
1152
[2114]1153    security.declarePublic('showFsPicture') ###(
1154    def showFsPicture(self,path):
1155        """return a picture from the filesystem"""
[2675]1156        #picture_path = os.path.join(i_home,path)
1157        picture_path = os.path.join(images_base,path)
[2335]1158        response = self.REQUEST.RESPONSE
[2342]1159        #import pdb;pdb.set_trace()
[2354]1160        registry = getToolByName(self, 'mimetypes_registry')
1161        mimetype = str(registry.lookupExtension(path.lower()) or
1162                    registry.lookupExtension('file.bin'))
[2114]1163        if os.path.exists(picture_path):
[2354]1164            response.setHeader('Content-type',mimetype)
[2114]1165            return open(picture_path).read()
[2335]1166        picture_path = os.path.join(i_home,'import',path)
1167        if os.path.exists(picture_path):
1168            return open(picture_path).read()
[2114]1169    ###)
1170
[1151]1171    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1172    def deleteAllCourses(self,department="All"):
1173        ''' delete the courses'''
1174        pm = self.portal_membership
1175        member = pm.getAuthenticatedMember()
[1160]1176
[1151]1177        if str(member) not in ("henrik","joachim"):
1178            return "not possible"
1179        if department == "All":
1180            res = self.portal_catalog({'meta_type': 'Department'})
1181        if len(res) < 1:
1182            return "No Departments found"
[1160]1183
[1151]1184        deleted = []
1185        for dep in res:
1186            cf = dep.getObject().courses
1187            if cf:
1188                cf.manage_delObjects(ids=cf.objectIds())
1189                deleted.append("deleted Courses in %s" % dep.getId)
1190        return "\r".join(deleted)
[1160]1191    ###)
[1151]1192
[1572]1193    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1194    def getLogfileLines(self,filename="event.log",numlines=20):
1195        """Get last NUMLINES lines of logfile FILENAME.
[1160]1196
[1572]1197        Return last lines' of a file in the instances logfile directory as
1198        a list. The number of returned lines equals `numlines' or less. If
1199        less than `numlines' lines are available, the whole file ist
1200        returned. If the file can not be opened or some other error
1201        occurs, empty list is returend.
1202        """
1203        result = []
1204        lines_hit = 0
1205
1206        # We only handle files in instances' log directory...
1207        logpath = os.path.join(i_home, "log")
1208        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1209        if not filename.startswith( logpath ):
1210            # Attempt to access file outside log-dir...
1211            return []
1212
1213        try:
1214            fd = file( filename, "rb" )
1215        except IOError:
1216            return []
1217        if not fd:
1218            return []
1219
1220        if os.linesep == None:
1221            linesep = '\n'
1222        else:
1223            linesep = os.linesep
1224
1225        # Try to find 'numlines' times a lineseparator, searching from end
1226        # and moving to the beginning of file...
1227        fd.seek( 0, 2) # Move to end of file...
1228        while lines_hit < numlines:
1229            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1230                                          # one step forward...
1231                lines_hit += 1
1232            try:
1233                fd.seek( -2, 1) # Go two bytes back from current pos...
1234            except IOError:
1235                # We cannot go back two bytes. Maybe the file is too small...
1236                break
1237        fd.seek(2,1)
1238
1239        # Read all lines from current position...
1240        result = fd.readlines()
1241        # Remove line endings...
1242        result = [x.strip() for x in result]
1243        fd.close()
1244        return result
1245    ###)
1246
[1700]1247    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
[1665]1248    def getCallbacksFromLog(self,filename):
1249        """fix Online Payment Transactions from Z2.log entries"""
1250        import transaction
1251        import random
1252        from cgi import parse_qs
1253        from urlparse import urlparse
1254        #from pdb import set_trace
1255        wftool = self.portal_workflow
1256        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1257        students_folder = self.portal_url.getPortalObject().campus.students
1258        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1259        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1260        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1261        data = re.compile(s)
1262        start = True
1263        tr_count = 1
1264        total = 0
1265        #name = 'pume_results'
1266        #name = 'epaymentsuccessful_z2log2'
1267        name = filename
1268        no_import = []
1269        imported = []
1270        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1271        try:
1272            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1273        except:
1274            logger.error('Error reading %s' % name)
1275            return
1276        tas = []
1277        for line in transactions:
1278            dict = {}
1279            items = data.search(line)
1280            dict['idict'] = idict = items.groupdict()
1281            #print idict
1282            #from pdb import set_trace;set_trace()
1283            urlparsed = urlparse(idict['get'][4:])
1284            #print urlparsed
1285            path = urlparsed[2].split('/')
1286            dict['student_id'] = student_id = path[8]
1287            dict['payment_id'] = payment_id = path[10]
1288            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1289            tas.append(dict)
1290            tr_count += 1
1291        return tas
1292    ###)
1293
[1620]1294    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1295    def importOnlinePaymentTransactions(self):
1296        """load Online Payment Transactions from CSV values"""
1297        import transaction
1298        import random
1299        #from pdb import set_trace
1300        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
[1625]1301        opt = self.online_payments_import
[1620]1302        students_folder = self.portal_url.getPortalObject().campus.students
1303        start = True
1304        tr_count = 1
1305        total = 0
1306        #name = 'pume_results'
1307        name = 'OnlineTransactions'
1308        no_import = []
1309        imported = []
1310        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1311        try:
1312            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1313        except:
1314            logger.error('Error reading %s.csv' % name)
1315            return
1316        for pay_transaction in transactions:
1317            if start:
1318                start = False
1319                logger.info('Start loading from %s.csv' % name)
1320                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1321                no_import.append('%s,"Error"' % s)
1322                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1323                format_error = format + ',"%(Error)s"'
1324            data = {}
[1644]1325
[1781]1326            # format of the first file sent by Tayo
[1643]1327            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1328            #data['student_id'] = student_id = pay_transaction['Payer ID']
1329            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1330            #data['response_code'] = response_code = pay_transaction['Resp Code']
1331            #data['amount'] = amount = pay_transaction['Amount']
[1644]1332
[1781]1333            # format of the second file sent by Tayo
[1798]1334            #data['datetime'] = date = 0
1335            #data['student_id'] = student_id = pay_transaction['Payer ID']
1336            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1337            #data['response_code'] = response_code = '00'
1338            #data['amount'] = amount = pay_transaction['Amount']
[1813]1339
[1798]1340            # format of the third file sent by Kehinde
[1643]1341            data['datetime'] = date = 0
[1798]1342            data['student_id'] = student_id = pay_transaction['customer_id']
1343            data['order_id'] = order_id = pay_transaction['merchant_reference']
[1644]1344            data['response_code'] = response_code = '00'
[1813]1345            data['amount'] = amount = pay_transaction['Amount']
[1644]1346
[1620]1347            dup = False
1348            if response_code == "12":
1349                continue
1350            try:
1351                opt.addRecord(**data)
1352            except ValueError:
1353                dup = True
1354            #from pdb import set_trace;set_trace()
1355            if dup:
1356                if response_code == "00":
[1798]1357                    try:
1358                        opt.modifyRecord(**data)
1359                    except:
1360                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1361                        continue
[1620]1362                else:
[1674]1363                    pay_transaction['Error'] = "Duplicate order_id"
[1620]1364                    no_import.append( format_error % pay_transaction)
[1674]1365                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
[1620]1366                    continue
1367            tr_count += 1
1368            if tr_count > 1000:
1369                if len(no_import) > 0:
1370                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1371                             '\n'.join(no_import) + '\n')
1372                    no_import = []
[1645]1373                em = '%d transactions committed\n' % (tr_count)
[1620]1374                transaction.commit()
1375                regs = []
1376                logger.info(em)
1377                total += tr_count
1378                tr_count = 0
1379        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1380                                                '\n'.join(no_import))
1381        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1382    ###)
[2237]1383
[3172]1384    security.declareProtected(ModifyPortalContent,"importData")###(
1385    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1386        """load data from CSV values"""
1387        import transaction
1388        import random
1389        students_folder = self.portal_url.getPortalObject().campus.students
1390        pending_only = False
[3177]1391        pend_str = '--'
[3172]1392        elapse = time.time()
1393        #
1394        # preparations
1395        #
[3177]1396        if filename == pend_str:
[3172]1397            pending_only = True
[3191]1398        # mode = "create"
1399        # if edit:
1400        #     if filename.endswith('_toDelete'):
1401        #         mode = "delete"
1402        #     else:
1403        #         mode = "edit"
[3172]1404        importer_name = ''.join([part.capitalize() for part in name.split('_')])
[3191]1405        importer = eval("%sImport" % importer_name)(self)
[3172]1406        logger = importer.logger
1407        if importer.init_errors:
[3191]1408            logger.info(importer.init_errors)
1409            return importer.init_errors
[3172]1410        member = importer.member
1411        current = importer.current
1412        import_date = importer.import_date
1413        #
1414        # not_imported
1415        #
1416        info = importer.info
1417        data_keys = importer.data_keys
[3181]1418        csv_keys = importer.csv_keys
[3172]1419        #csv_keys.extend(info.keys())
1420        headline_mapping = dict((k,k) for k in csv_keys)
1421        #
1422        # pending
1423        #
1424        pending_path = importer.pending_path
1425        pending_tmp = importer.pending_tmp
1426        pending_backup = importer.pending_backup
1427        pending_fn = importer.pending_fn
1428        imported_path = importer.imported_path
1429        imported_fn = importer.imported_fn
[3180]1430        commit_after = importer.commit_after
[3172]1431        pending = []
1432        pending_digests = []
1433        #total_added_to_pending = 0
1434        if not pending_only:
1435            pending,pending_digests = importer.makeIdLists()
1436            pending_at_start = len(pending)
1437        datafile = open(pending_tmp,"w")
1438        pending_csv_writer = csv.DictWriter(datafile,
1439                                                    csv_keys,
1440                                                    extrasaction='ignore')
1441        pending_csv_writer.writerow(headline_mapping)
1442        datafile.close()
1443        #
1444        # imported
1445        #
1446        if not os.path.exists(imported_path):
1447            datafile = open(imported_path,"w")
1448            imported_csv_writer = csv.DictWriter(datafile,
1449                                                 csv_keys,
1450                                                 extrasaction='ignore')
1451            imported_csv_writer.writerow(headline_mapping)
1452            datafile.close()
1453        start = True
1454        tr_count = 0
1455        total = 0
1456        total_added_to_imported = 0
1457        total_pending = 0
1458        if pending_only:
1459            import_source_path = pending_path
[2193]1460        else:
[3172]1461            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1462        if not os.path.exists(import_source_path):
[3195]1463            fn = os.path.split(import_source_path)[1]
[3268]1464            em = 'no import file %(fn)s' % vars()
[3172]1465            return em
1466        import_source_fn = os.path.split(import_source_path)[1]
[3267]1467        if not pending_only:
1468            info['imported_from'] = import_source_fn
[3172]1469        attrs = csv.reader(open(import_source_path,"rb")).next()
[3191]1470        if "import_mode" not in attrs:
1471            msg = 'import_mode must be in heading'
1472            return msg
[3201]1473
[3172]1474        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore')
1475                                                        or k.strip() in info.keys())]
1476        diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1477        diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1478        if diff2schema and diff2schema != set(['id',]):
1479            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1480            return msg
1481        #
1482        # start importing
1483        #
[2193]1484        try:
[3172]1485            reader = csv.DictReader(open(import_source_path,"rb"))
1486        except:
1487            msg = 'Error reading %s.csv' % filename
1488            logger.error(msg)
1489            return msg
1490        items = [item for item in reader]
1491        total_to_import = len(items)
[3181]1492        tti_float = float(total_to_import)
[3172]1493        if pending_only:
1494            pending_at_start = total_to_import
1495        count = 0
1496        imported = []
1497        old_commit_count = 0
[3204]1498        error_count = imported_count = 0
[3172]1499        already_in = 0
1500        for record in items:
1501            item = {}
[3198]1502            empty_value_keys = []
[3172]1503            for k,v in record.items():
1504                if v:
[3198]1505                    if v == EMPTY:
1506                        empty_value_keys += k,
1507                        v = ''
[3172]1508                    item[k.strip()] = v.strip()
1509            count += 1
1510            if start:
1511                start = False
1512                adapters = [MappingStorageAdapter(importer.schema, item)]
[3191]1513                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
[3172]1514            dm = DataModel(item, adapters,context=self)
1515            ds = DataStructure(data=item,datamodel=dm)
1516            error_string = ""
1517            total += 1
[3271]1518            import_mode = item.get('import_mode','')
[3191]1519            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1520            if import_method is None:
[3271]1521                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
[3251]1522            elif (import_mode in importer.required_modes and
[3248]1523                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1524                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1525                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
[3191]1526            else:
1527                for k in import_keys:
1528                    if k not in item.keys() or k not in importer.validators.keys():
1529                        continue
1530                    if not importer.validators[k](ds,mode=import_mode):
[3201]1531                        error_string += ' ++ '
1532                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
[3172]1533                                                                           ds.getErrorMapping(k)))
1534            if error_string:
1535                error = error_string
1536                id = ''
1537                mapping = item
[2699]1538            else:
[3172]1539                temp_item = item.copy()
1540                temp_item.update(dm)
[3185]1541                results = import_method(temp_item)
[3172]1542                id = results[0]
1543                error = results[1]
1544                mapping = results[2]
[3267]1545            #
1546            # restore some values
1547            #
[3198]1548            for k in empty_value_keys:
1549                mapping[k] = EMPTY
[3186]1550            if mapping.has_key('sex'):
1551                #import pdb;pdb.set_trace()
[3267]1552                if mapping['sex'] == True:
[3186]1553                    mapping['sex'] = 'F'
[3267]1554                elif mapping['sex'] == False:
[3186]1555                    mapping['sex'] = 'M'
[3267]1556            if pending_only:
1557                info['imported_from'] = item['imported_from']
[3181]1558            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
[3172]1559            info['error'] = error
1560            mapping.update(info)
[3178]1561            log_list = []
[3172]1562            if error:
[3204]1563                error_count += 1
[3172]1564                digest = makeDigest(mapping,data_keys)
1565                if digest not in pending_digests:
1566                    pending_digests += digest,
1567                    pending.append(mapping)
1568                    if not pending_only:
[3251]1569                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
[3172]1570                else:
1571                    already_in += 1
1572                    pass
1573            else:
1574                imported_count += 1
1575                imported += mapping,
[3251]1576                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
[3178]1577            if log_list:
1578                time_till_now = time.time() - elapse
[3204]1579                percent_finished = (error_count + imported_count)/tti_float*100
[3181]1580                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
[3178]1581                logger.info(' '.join(log_list))
[3172]1582            finished = count > total_to_import - 1
1583            must_commit = False
1584            if len(imported) != old_commit_count:
[3180]1585                if not len(imported) % commit_after:
[3172]1586                    must_commit = True
1587                    old_commit_count = len(imported)
[3181]1588
[3172]1589            if must_commit or finished:
1590                if len(imported):
1591                    transaction.commit()
1592                    datafile = open(imported_path,"a")
1593                    writer = csv.DictWriter(datafile,
1594                                            csv_keys,
1595                                            extrasaction='ignore')
1596                    writer.writerows(imported)
1597                    datafile.close()
1598                    total_added_to_imported += len(imported)
1599                    imported = []
1600                if len(pending) > 0:
1601                    datafile = open(pending_tmp,"a")
1602                    writer = csv.DictWriter(datafile,
1603                                            csv_keys,
1604                                            extrasaction='ignore')
1605                    writer.writerows(pending)
1606                    datafile.close()
1607                    total_pending += len(pending)
1608                    #total_added_to_pending += len(pending)
1609                    pending = []
1610                if not finished:
1611                    msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
1612                    logger.info(msg)
1613        elapse = time.time() - elapse
[3271]1614        if os.path.exists(pending_path):
1615            copy2(pending_path,pending_backup)
[3172]1616        copy2(pending_tmp,pending_path)
[3191]1617        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
[3181]1618        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
[3172]1619        if pending_only:
1620            removed_pending = pending_at_start - total_pending
[3184]1621            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
[2699]1622        else:
[3181]1623            added_pending = total_pending - pending_at_start
1624            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1625        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
[3172]1626        logger.info(msg)
[3178]1627        os.remove(pending_tmp)
[3172]1628        return msg
[2514]1629    ###)
1630
[1775]1631
[3251]1632
[2292]1633    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1634    def moveImagesToFS(self,student_id="O738726"):
1635        "move the images to the filesystem"
[2675]1636        images_dir = getImagesDir(student_id)
1637        #images_dir = os.path.join("%s" % images_base,student_id)
[2292]1638        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1639        stool = getToolByName(self, 'portal_schemas')
1640        schemas = ['student_application',
1641                   'student_clearance',
1642                   ]
1643        created = False
1644        for schema_id in schemas:
1645            schema = stool._getOb(schema_id)
1646            object = getattr(student_folder,schema_id[len('student_'):],None)
1647            if object is None:
1648                continue
1649            doc = object.getContent()
1650            for key in schema.keys():
1651                if schema[key].meta_type != "CPS Image Field":
1652                    continue
1653                #import pdb;pdb.set_trace()
[2351]1654                image = getattr(doc,key,None)
1655                if not image or not hasattr(image,"data"):
[2292]1656                    continue
1657                if not created:
1658                    if not os.path.exists(images_dir):
1659                        os.mkdir(images_dir)
1660                    created = True
1661                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1662                open(filename,"wb").write(str(image.data))
[2500]1663    ###)
[2292]1664
[2300]1665    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1666    def movePassportToFS(self,student_id="O738726"):
1667        "move the passports to the filesystem"
1668        images_dir = os.path.join("%s" % i_home,'passports')
1669        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1670        stool = getToolByName(self, 'portal_schemas')
1671        schemas = ['student_application',
1672                   #'student_clearance',
1673                   ]
1674        created = False
1675        for schema_id in schemas:
1676            schema = stool._getOb(schema_id)
1677            object = getattr(student_folder,schema_id[len('student_'):],None)
1678            if object is None:
1679                continue
1680            doc = object.getContent()
1681            for key in schema.keys():
1682                if schema[key].meta_type != "CPS Image Field":
1683                    continue
1684                #import pdb;pdb.set_trace()
1685                image = getattr(doc,key)
1686                if not hasattr(image,"data"):
1687                    continue
1688                if not created:
1689                    if not os.path.exists(images_dir):
1690                        os.mkdir(images_dir)
1691                    created = True
1692                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1693                open(filename,"wb").write(str(image.data))
[2500]1694    ###)
[2300]1695
[828]1696InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.