source: WAeUP_SRP/base/WAeUPTool.py @ 3347

Last change on this file since 3347 was 3346, checked in by joachim, 17 years ago

fix for #108 fceokene: additional commas in csv-files are ignored
getCertificatesDict now returns a dictionary of dictinaries
admitOne uses it.
  • Property svn:keywords set to Id
File size: 69.4 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3346 2008-03-17 10:54:14Z joachim $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108
109###)
110
111    security.declareProtected(View,'updateRoleMappingsFor') ###(
112    def updateRoleMappingsFor(self,wf_definition,ob):
113        "do so for public"
114        wf_def = getattr(self.portal_workflow,wf_definition)
115        wf_def.updateRoleMappingsFor(ob)
116    ###)
117
118    security.declareProtected(View,'getStatesLgas') ###(
119    def getStatesLgas(self):
120        """return lga info"""
121        voc = getattr(self.portal_vocabularies,'local_gov_areas')
122        states = []
123        lgas  = []
124        d = {}
125        wd = {}
126        for k,v in voc.items():
127            parts = v.split(' / ')
128            if len(parts) == 1:
129                state = parts[0].lower()
130                lga = ""
131            elif len(parts) == 2:
132                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134            else:
135                continue
136            if state not in states:
137                states.append(state)
138            if lga not in lgas:
139                lgas.append(lga)
140            words = re.split('[^a-zA-Z0-9/]',k)
141            words.sort()
142            wd[k] = words
143            d[k] = v
144        mapping = {}
145        mapping['word_dict'] = wd
146        mapping['lga_dict'] = d
147        mapping['states'] = states
148        mapping['lgas'] = lgas
149        return mapping
150    ###)
151
152    security.declareProtected(View,'findLga') ###(
153    def findLga(self,words,words_dict):
154        words = re.split('[^a-zA-Z0-9/]',words)
155        lga_words = []
156        for word in words:
157            if word:
158                lga_words += word.strip().lower(),
159        lga_words.sort()
160        state_lga = ''
161        while not state_lga:
162            for k,l in words_dict.items():
163                if lga_words == l:
164                    state_lga = k
165                    break
166            break
167        return state_lga
168    ###)
169    security.declareProtected(View,'getAccessInfo') ###(
170    def getAccessInfo(self,context):
171        "return a dict with access_info"
172        logger = logging.getLogger('WAeUPTool.getAccessInfo')
173        mtool = self.portal_membership
174        member = mtool.getAuthenticatedMember()
175        member_id = str(member)
176        info = {}
177        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179        is_staff = info['is_staff'] = not is_anonymous and not is_student
180        roles = member.getRolesInContext(context)
181        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182                                                                            "SectionManager" in roles or
183                                                                            "Manager" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif is_anonymous or member_id != requested_id:
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378
379###)
380
381    def generateStudentId(self,letter): ###(
382        import random
383        r = random
384        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385        if letter == '?':
386            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387        sid = "%c%d" % (letter,r.randint(99999,1000000))
388        students = self.portal_url.getPortalObject().campus.students
389##        while hasattr(students, sid):
390##            sid = "%c%d" % (letter,r.randint(99999,1000000))
391        while self.students_catalog(id = sid):
392            sid = "%c%d" % (letter,r.randint(99999,1000000))
393        return sid
394    ###)
395
396    def generatePassword(self,s=None): ###(
397        import random
398        r = random
399        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400        if s is None:
401            s = 'abcdefghklmnpqrstuvwxy23456789'
402        pw = ''
403        while len(pw) < 6:
404            pw += r.choice(s)
405        return pw
406    ###)
407
408    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409    def dumpSchoolfeePayments(self):
410        "dump paid schoolfees"
411        mtool = self.portal_membership
412        member = mtool.getAuthenticatedMember()
413        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414        aq_student = self.students_catalog.evalAdvancedQuery
415        query = In('review_state',('schoolfee_paid',
416                                   'courses_registered',
417                                   'courses_validated',
418                                   ))
419        res = aq_student(query)
420        #import pdb;pdb.set_trace()
421        l = []
422        logger.info("start for %d" % len(res))
423        count = 1
424        log_after = 100
425        for student in res:
426            if not count % log_after:
427                logger.info("processed %d total %d" % (log_after,count))
428            count += 1
429            fee_dict =self.getSchoolFee(student)
430            fulltime = student.mode.endswith('_ft')
431            d = {}
432            d['student_id'] = student.id
433            d['name'] = student.name
434            d['amount'] = fee_dict.get(new_returning)
435            l += d,
436        csv_name = self.dumpListToCSV(l,'payments')
437        logger.info('%s dumped payments to %s' % (member,export_file))
438    ###)
439
440    security.declarePublic('dumpListToCSV') ###(
441    def dumpListToCSV(self,l,filename,fields=None):
442        """dump a list of dicts to a CSV file"""
443        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445        if fields is None:
446            fields = l[0].keys()
447        headline = ','.join(fields)
448        out = open(export_file,"wb")
449        out.write(headline +'\n')
450        out.close()
451        out = open(export_file,"a")
452        csv_writer = csv.DictWriter(out,fields,)
453        csv_writer.writerows(l)
454        return export_file
455    ###)
456
457    security.declareProtected(ManagePortal, 'listMembers') ###(
458    def listMembers(self):
459        "list all members"
460        mtool = self.portal_membership
461        member = mtool.getAuthenticatedMember()
462        logger = logging.getLogger('WAeUPTool.listMembers')
463        if str(member) not in ('admin','joachim'):
464            logger.info('%s tried to list members' % (member))
465            return None
466        members = self.portal_directories.members
467        all = members.listEntryIdsAndTitles()
468        l = []
469        for user_id,name in all:
470            d = {}
471            d['user_id'] = user_id
472            d['name'] = name
473            d['pw'] = getattr(getattr(members,user_id),'password')
474            d['email'] = getattr(getattr(members,user_id),'email')
475            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477            l += d,
478        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480        logger.info('%s dumped member list to %s' % (member,export_file))
481        fields = l[0].keys()
482        headline = ','.join(fields)
483        out = open(export_file,"wb")
484        out.write(headline +'\n')
485        out.close()
486        out = open(export_file,"a")
487        csv_writer = csv.DictWriter(out,fields,)
488        csv_writer.writerows(l)
489    ###)
490
491    security.declareProtected(ManagePortal, 'listStudents') ###(
492    def listStudents(self):
493        "list all students"
494        mtool = self.portal_membership
495        member = mtool.getAuthenticatedMember()
496        logger = logging.getLogger('WAeUPTool.listStudents')
497        if str(member) not in ('admin','joachim'):
498            logger.info('%s tried to list students' % (member))
499            return None
500        students = self.portal_directories.students
501        all = students.listEntryIdsAndTitles()
502        l = []
503        for user_id,name in all:
504            d = {}
505            d['user_id'] = user_id
506            d['name'] = name
507            d['pw'] = getattr(getattr(students,user_id),'password')
508            d['email'] = getattr(getattr(students,user_id),'email')
509            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511            l += d,
512        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514        logger.info('%s dumped student list to %s' % (member,export_file))
515        fields = l[0].keys()
516        headline = ','.join(fields)
517        out = open(export_file,"wb")
518        out.write(headline +'\n')
519        out.close()
520        out = open(export_file,"a")
521        csv_writer = csv.DictWriter(out,fields,)
522        csv_writer.writerows(l)
523    ###)
524
525    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526    def removeDeletedDocIds(self, max=1000):
527        """
528        remove deleted docids from repository commit after max
529        """
530        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531        repository = getToolByName(self, 'portal_repository')
532        pxtool = getToolByName(self, 'portal_proxies')
533        logger.info('start')
534        pxtool_infos = pxtool.getRevisionsUsed()
535        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
536
537        nb_revs = 0
538        docids_d = {} # all docids
539        unused_docids_d = {} # all docids that are unused
540        ids_unused_revs_docids = [] # ids for revs of unused docids
541        unused_ids = [] # ids for unused revs
542        total = 0
543        idlist = repository.objectIds()
544        to_delete = 0
545        found = False
546        for id in idlist:
547            docid, rev = repository._splitId(id)
548            if docid is None:
549                logger.info("invalid doc_id %s" % docid)
550                continue
551            nb_revs += 1
552            if not pxtool_infos.has_key(docid):
553                found = True
554                to_delete += 1
555                unused_ids.append(id)
556            elif not pxtool_infos[docid].has_key(rev):
557                found = True
558                to_delete += 1
559                unused_ids.append(id)
560            if found and not to_delete % max:
561                found = False
562                #import pdb;pdb.set_trace()
563                repository.manage_delObjects(unused_ids)
564                transaction.commit()
565                logger.info('removed %d total %d unused docids ' % (max,to_delete))
566        else:
567            if unused_ids:
568                repository.manage_delObjects(unused_ids)
569                transaction.commit()
570        logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573###)
574
575    security.declareProtected(View,'getCredential') ###(
576    def getCredential(self,student_id):
577        student_entry = getattr(self.portal_directories.students,student_id,None)
578        if not self.isStaff():
579            mtool = self.portal_membership
580            member = mtool.getAuthenticatedMember()
581            logger = logging.getLogger('WAeUPTool.getCredential')
582            logger.info('%s tried to access password of %s' % (member,student_id))
583            return None
584        if student_entry is None:
585            return None
586        return getattr(student_entry,"password","not set")
587    ###)
588
589    security.declarePublic('checkPassword') ###(
590    def checkPassword(self,student_id,password):
591        student_entry = getattr(self.portal_directories.students,student_id,None)
592        if student_entry is None:
593            return False
594        return getattr(student_entry,"password","not set") == password
595    ###)
596
597    security.declarePublic('checkGenericPassword') ###(
598    def checkGenericPassword(self,member_id):
599        member_entry = getattr(self.portal_directories.members,member_id,None)
600        if member_entry is None:
601            return False
602        ltool = getToolByName(self, 'portal_layouts')
603        unsecure_words = ltool._getOb('members')['w__password'].check_words
604        password = getattr(member_entry,"password","not set")
605        is_unsecure = password in unsecure_words
606        if is_unsecure:
607            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609        return is_unsecure
610    ###)
611
612    security.declareProtected(ModifyPortalContent,'editPassword') ###(
613    def editPassword(self,student_id,password):
614        "edit a student password"
615        student_entry = getattr(self.portal_directories.students,student_id,None)
616        if student_entry is None:
617            return
618        setattr(student_entry,'password',password)
619    ###)
620
621    security.declareProtected(ModifyPortalContent,'doCommit') ###(
622    def doCommit(self,logger=None):
623        "commit some transactions"
624        transaction.commit()
625    ###)
626
627    security.declarePublic('loadStudentFoto') ###(
628    def loadStudentFoto(self,student,filename,folder):
629        "return a student passport picture"
630        #import pdb;pdb.set_trace()
631        picture ="%s/import/%s/%s" % (i_home,folder,filename)
632        student_id = student.getId()
633        images_dir = getImagesDir(student_id)
634        if not os.path.exists(images_dir):
635            os.mkdir(images_dir)
636        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637        for extension in ('.jpg','.JPG'):
638            fullname = "%(picture)s%(extension)s" % vars()
639            if os.path.exists(fullname):
640                copy2(fullname,image_name)
641                return "successfully copied passport picture"
642        return "passport picture not found: %s.jpg or .JPG" % picture
643    ###)
644
645    def old____loadStudentFoto(self,student,filename,folder): ###(
646        "return a student passport picture"
647        app = student.application
648        app_doc = app.getContent()
649        #clear = student.clearance
650        #clear_doc = clear.getContent()
651        #matric_no = clear_doc.matric_no.upper()
652        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654        #import pdb;pdb.set_trace()
655        if os.path.exists(picture1):
656            file = open(picture1)
657        elif os.path.exists(picture2):
658            file = open(picture2)
659        else:
660            return "passport picture not found %s" % picture1
661        reopened = False
662        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663            self.portal_workflow.doActionFor(app,'open')
664            reopened = True
665        outfile = file.read()
666        app_doc.manage_addFile('passport',
667                               file=outfile,
668                               title="%s.jpg" % filename)
669        if reopened:
670            self.portal_workflow.doActionFor(app,'close')
671        return "successfully loaded passport picture"
672    ###)
673
674    security.declareProtected(ModifyPortalContent,'createOne') ###(
675    def createOne(self,students_folder,student_brain,letter,commit=False):
676        sid = self.waeup_tool.generateStudentId(letter)
677        students_folder.invokeFactory('Student', sid)
678        student = getattr(students_folder,sid)
679        self.portal_workflow.doActionFor(student,'return')
680        student.manage_setLocalRoles(sid, ['Owner',])
681        matric_no = student_brain.matric_no
682        jamb_reg_no = student_brain.Entryregno
683        self.students_catalog.addRecord(id = sid,
684                                           matric_no = matric_no,
685                                           jamb_reg_no = jamb_reg_no,
686                                           sex = student_brain.Sex == "F",
687                                           name = "%s %s %s" % (student_brain.Firstname,
688                                                                student_brain.Middlename,
689                                                                student_brain.Lastname)
690                                        )
691        if commit:
692            transaction.commit()
693        return sid,jamb_reg_no
694    ###)
695
696    security.declareProtected(ModifyPortalContent,'addStudent') ###(
697    def addStudent(self,dict):
698        students_folder = self.portal_url.getPortalObject().campus.students
699        sid = self.waeup_tool.generateStudentId('?')
700        students_folder.invokeFactory('Student', sid)
701        student_obj = getattr(students_folder,sid)
702        f2t = self.field2types_student
703        #from pdb import set_trace; set_trace()
704        d = {}
705        #d['jamb_sex']  = 'M'
706        #if dict.get('sex'):
707        #    d['jamb_sex']  = 'F'
708
709        entry_session = dict.get('entry_session')
710        if entry_session == self.getSessionId()[0]:
711            wfaction = 'admit'
712            wft = 'wf_transition_admit'
713            password = None
714        else:
715            wfaction = 'return'
716            wft = 'wf_transition_return'
717            password = self.generatePassword()
718            self.makeStudentMember(sid,password)
719
720        for pt in f2t.keys():
721            student_obj.invokeFactory(pt,f2t[pt]['id'])
722            sub_obj = getattr(student_obj,f2t[pt]['id'])
723            sub_doc = sub_obj.getContent()
724            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725            d['Title'] = f2t[pt]['title']
726            for field in f2t[pt]['fields']:
727                d[field] = dict.get(field,'')
728            sub_doc.edit(mapping = d)
729            new_state = f2t[pt][wft]
730            if new_state != "remain":
731                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732        self.portal_workflow.doActionFor(student_obj,wfaction)
733        student_obj.manage_setLocalRoles(sid, ['Owner',])
734        return sid,password
735    ###)
736
737    security.declarePublic('getCertificateBrain') ###(
738    def getCertificateBrain(self,cert_id):
739        "do it"
740        res = ZCatalog.searchResults(self.portal_catalog_real,
741                                {'portal_type':"Certificate",
742                                      'id': cert_id})
743        if res:
744            return res[0]
745        return None
746    ###)
747
748    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749    def get_csv_filenames(self):
750        "do it"
751        files = [file for file in os.listdir("%s/import/" % (i_home))
752                 if file.endswith('.csv') and (file.find('imported') == -1 and
753                                               file.find('pending') == -1)]
754        return files
755    ###)
756
757    security.declarePublic('findStudentByMatricelNo') ###(
758    def findStudentByMatricelNo(self,matric_no):
759        "do it"
760        res = ZCatalog.searchResults(self.portal_catalog_real,
761                                {'portal_type':"StudentClearance",
762                                 'SearchableText': matric_no})
763        if res:
764            return res[0]
765        return None
766    ###)
767
768    security.declarePublic('makeStudentMember') ###(
769    def makeStudentMember(self,sid,password='uNsEt'):
770        """make the student a member"""
771        membership = self.portal_membership
772        membership.addMember(sid,
773                             password ,
774                             roles=('Member',
775                                     'Student',
776                                     ),
777                             domains='',
778                             properties = {'memberareaCreationFlag': False,
779                                           'homeless': True},)
780        member = membership.getMemberById(sid)
781        self.portal_registration.afterAdd(member, sid, password, None)
782        #self.manage_setLocalRoles(sid, ['Owner',])
783    ###)
784
785    security.declareProtected(View,'makeStudentData') ###(
786    def makeStudentData(self,student_id,email=None,phone_nr=None):
787        "create Datastructure for a returning Student"
788        #import pdb;pdb.set_trace()
789        logger = logging.getLogger('WAeUPTool.makeStudentData')
790        students_folder = self.portal_url.getPortalObject().campus.students
791        #res = self.students_catalog(id=student_id)
792        #if res:
793        #    st = res[0]
794        #res = self.returning_import(matric_no = st.matric_no)
795        res = self.returning_import(id = student_id)
796        if res:
797            student = res[0]
798        else:
799            logger.info('Id %s not found in returning_import' % student_id)
800            return
801        logger.info('%s creates data structure' % student_id)
802        s_results = self.results_import(matric_no = student.matric_no)
803        if s_results:
804            lnr = self.getLevelFromResultsCosCode(s_results)
805            level = "%d00" % lnr
806            verdict,eligible = self.getVerdict(s_results[0].Verdict)
807            #if eligible:
808            #    level = "%d00" % (lnr + 1)
809        else:
810            logger.info('matric_no %s not found in results_import' % student.matric_no)
811            level = ''
812            verdict = ''
813        #student should not be allowed to perform this transition
814        #wftool = self.portal_workflow
815        #wftool.doActionFor(student,'return')
816        certcode_org = student.Coursemajorcode
817        certcode = makeCertificateCode(certcode_org)
818        certificate_brain = self.getCertificateBrain(certcode)
819        if not certificate_brain:
820            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821            logger.info(em)
822        matric_no = student.matric_no
823        sid = student_id
824        student_obj = getattr(students_folder,sid)
825        if not getattr(student_obj,'application'):
826            student_obj.invokeFactory('StudentApplication','application')
827        application = student_obj.application
828        self.portal_workflow.doActionFor(application,'open',dest_container=application)
829        da = {'Title': 'Application Data'}
830        student_obj.invokeFactory('StudentPersonal','personal')
831        da['jamb_reg_no'] = student.Entryregno
832        em = self.getEntryMode(student.Entryregno)
833        da['entry_mode'] = em
834        personal = student_obj.personal
835        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836        dp = {'Title': 'Personal Data'}
837        student_obj.invokeFactory('StudentClearance','clearance')
838        clearance = student_obj.clearance
839        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840        dc = {'Title': 'Clearance/Eligibility Record'}
841        dc['matric_no'] = matric_no
842        state = student.State
843        lga = student.LGA
844        if state and lga:
845            lga =  state + ' / ' + lga
846        else:
847            lga = "None"
848        da['jamb_lga'] = dc['lga'] = lga
849        da['app_email'] = dp['email'] = email
850        da['app_mobile'] = dp['phone'] = phone_nr
851        dp['firstname'] = student.Firstname
852        dp['middlename'] = student.Middlename
853        dp['lastname'] = student.Lastname
854        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855        da['jamb_sex'] = student.Sex
856        dp['sex'] = student.Sex == 'F'
857        dp['perm_address'] = student.Permanent_Address
858        application.getContent().edit(mapping=da)
859        self.portal_workflow.doActionFor(application,'close',dest_container=application)
860        personal.getContent().edit(mapping=dp)
861        clearance.getContent().edit(mapping=dc)
862        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863        #
864        # Study Course
865        #
866        student_obj.invokeFactory('StudentStudyCourse','study_course')
867        studycourse = student_obj.study_course
868        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869        dsc = {}
870        dsc['study_course'] = certcode
871        dsc['current_level'] = level
872        dsc['current_verdict'] = verdict
873        dsc['current_mode'] = em
874        dsc['current_session'] = '05'
875        studycourse.getContent().edit(mapping=dsc)
876        #
877        # Level
878        #
879##        l = getattr(studycourse,level,None)
880##        if l is None:
881##            studycourse.invokeFactory('StudentStudyLevel', level)
882##            l = getattr(studycourse, level)
883##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
884##            l.getContent().edit(mapping={'Title': "Level %s" % level})
885###)
886
887    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
888    def admitOneStudent(self,brain,entry_session,pin_password):
889        "create Datastructure for an admitted Student"
890        #import pdb;pdb.set_trace()
891        if not hasattr(self,"_v_certificates"):
892            self._v_certificates = self.getCertificatesDict()
893        students_folder = self.portal_url.getPortalObject().campus.students
894        logger = logging.getLogger('WAeUPTool.admitOneStudent')
895        if brain.screening_type in ('cest','sandwich',):
896            reg_no = "%s%s/07" % (brain.course1,brain.serial)
897        else:
898            reg_no = brain.reg_no
899        res = self.students_catalog(jamb_reg_no = reg_no)
900        if res:
901            logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
902            return
903        if brain.status != "admitted":
904            logger.info('status of %s is %s' % (reg_no,brain.status))
905            return
906        pin_parts = brain.pin.split('-')
907        if pin_parts and len(pin_parts) != 3:
908            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
909            return
910        if brain.course_admitted not in self._v_certificates:
911            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
912            return
913        if brain.sex not in (True,False):
914            logger.info('sex of %s not available' % (reg_no))
915            return
916        student_id = self.generateStudentId('?')
917        students_folder.invokeFactory('Student', student_id)
918        student_object = getattr(students_folder,student_id)
919        if pin_password:
920            password = pin_parts[2]
921            self.makeStudentMember(student_id,password = password)
922        student_object.manage_setLocalRoles(student_id, ['Owner',])
923        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
924        #
925        # application
926        #
927        student_object.invokeFactory('StudentApplication','application')
928        application = student_object.application
929        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
930        da = {'Title': 'Application Data'}
931        da['jamb_reg_no'] = reg_no
932
933        sex = 'M'
934        if brain.sex:
935            sex = 'F'
936        da['jamb_sex'] = sex
937        da['jamb_age'] = brain.jamb_age
938        da['app_reg_pin'] = brain.pin
939        da['jamb_lga'] = brain.jamb_lga
940        da['jamb_state'] = brain.jamb_state
941        da['jamb_score'] = brain.aggregate
942        da['app_email'] = brain.email
943        da['app_mobile'] = brain.phone
944
945        da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode']
946
947        #if brain.entry_mode:
948        #    da['entry_mode'] = brain.entry_mode
949        #elif brain.screening_type == 'pume':
950        #    da['entry_mode'] = 'ume_ft'
951        #elif brain.screening_type == 'pde':
952        #    da['entry_mode'] = 'de_ft'
953        #elif brain.screening_type == 'pce':
954        #    da['entry_mode'] = 'pce'
955        #elif brain.screening_type == 'prence':
956        #    da['entry_mode'] = 'prence'
957        #else:
958        #    da['entry_mode'] = ''
959
960
961        da['entry_session'] = entry_session
962        da['jamb_lastname'] = brain.lastname
963        da['jamb_middlename'] = brain.middlenames   # different field names!
964        da['jamb_firstname'] = brain.firstname
965        da['screening_application_date'] = brain.application_date
966        da['date_of_birth'] = brain.date_of_birth
967        da['jamb_first_cos'] = brain.course1
968        da['jamb_second_cos'] = brain.course2
969        da['course3'] = brain.course3
970        da['screening_type'] = brain.screening_type
971        da['screening_score'] = brain.screening_score
972        da['screening_date'] = brain.screening_date
973        da['hq_type'] = brain.hq_type
974        da['hq_grade'] = brain.hq_grade
975        da['aos'] = brain.aos
976
977        application.getContent().edit(mapping=da)
978        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
979        #
980        # personal
981        #
982        student_object.invokeFactory('StudentPersonal','personal')
983        personal = student_object.personal
984        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
985        dp = {'Title': 'Personal Data'}
986        dp['sex'] = brain.sex
987        dp['email'] = brain.email
988        dp['phone'] = brain.phone
989        dp['lastname'] = brain.lastname
990        dp['middlename'] = brain.middlenames   # different field names!
991        dp['firstname'] = brain.firstname
992        personal.getContent().edit(mapping=dp)
993        #
994        # clearance
995        #
996        student_object.invokeFactory('StudentClearance','clearance')
997        clearance = student_object.clearance
998        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
999        dc = {'Title': 'Clearance/Eligibility Record'}
1000        dc['lga'] = brain.lga
1001        dc['birthday'] = brain.date_of_birth
1002        clearance.getContent().edit(mapping=dc)
1003        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1004        #
1005        # study Course
1006        #
1007        student_object.invokeFactory('StudentStudyCourse','study_course')
1008        studycourse = student_object.study_course
1009        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1010        dsc = {}
1011        dsc['study_course'] = brain.course_admitted
1012        dsc['current_verdict'] = ''
1013        dsc['current_mode'] = da['entry_mode']
1014        if da['entry_mode'].startswith('de'):
1015            dsc['current_level'] = '200'
1016        elif da['entry_mode'].startswith('pre'):
1017            dsc['current_level'] = '000'
1018        else:
1019            dsc['current_level'] = '100'
1020        dsc['current_session'] = entry_session
1021        studycourse.getContent().edit(mapping=dsc)
1022        #
1023        # payments folder
1024        student_object.invokeFactory('PaymentsFolder','payments')
1025        payments = getattr(student_object,'payments')
1026        dpay = {}
1027        dpay['Title'] = 'Payments'
1028        payments.getContent().edit(mapping=dpay)
1029        self.portal_workflow.doActionFor(payments,'open')
1030        #
1031        # passport foto
1032        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1033                                                              brain.screening_type,
1034                                                              brain.reg_no)
1035        images_dir = getImagesDir(student_id)
1036        #images_dir = os.path.join("%s" % images_base,student_id)
1037        letter_dir,student_dir = os.path.split(images_dir)
1038        if not os.path.exists(letter_dir):
1039            os.mkdir(letter_dir)
1040        if not os.path.exists(images_dir):
1041            os.mkdir(images_dir)
1042        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1043        if os.path.exists(app_picture):
1044            copy2(app_picture,image_name)
1045        else:
1046            logger.info('passport of %s/%s not found: %s' % (student_id,
1047                                                             brain.reg_no,
1048                                                             app_picture))
1049
1050        return student_id
1051    ###)
1052
1053    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1054    def makeStudentLevel(self,student_id):
1055        "create the StudyLevel for a returning Student"
1056        #import pdb;pdb.set_trace()
1057        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1058        students_folder = self.portal_url.getPortalObject().campus.students
1059        res = self.students_catalog(id=student_id)
1060        if res:
1061            st = res[0]
1062        course = st.course
1063        matric_no = st.matric_no
1064        level = st.level
1065        res = self.results_import(matric_no = matric_no)
1066        if res:
1067            results = res
1068        logger.info('%s creating Level %s' % (student_id,level))
1069        #
1070        # Level
1071        #
1072        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1073        studycourse = getattr(student_obj,"study_course",None)
1074        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1075        l = getattr(studycourse,level,None)
1076        if l is None:
1077            studycourse.invokeFactory('StudentStudyLevel', level)
1078            l = getattr(studycourse, level)
1079            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1080            l.getContent().edit(mapping={'Title': "Level %s" % level})
1081        ###)
1082
1083    security.declarePublic('getHallInfo') ###(
1084    def getHallInfo(self,bed):
1085        """return Hall Info"""
1086        info = {}
1087        hall,block,room,letter = bed.split('_')
1088        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1089        if res and len(res) == 1:
1090            hall_brain = res[0]
1091            hall_doc = hall_brain.getObject().getContent()
1092        else:
1093            return info
1094        info['hall_title'] = hall_brain.Title
1095        info['maintenance_code'] = hall_doc.maintenance_code
1096        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1097        batch_doc = None
1098        for brain in res:
1099            if brain.id.startswith(info['maintenance_code']):
1100                batch_doc = brain.getObject().getContent()
1101                break
1102        if batch_doc is None:
1103            info['maintenance_fee'] = ''
1104        else:
1105            info['maintenance_fee'] = batch_doc.cost
1106        return info
1107    ###)
1108
1109    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1110    def removePictureFolder(self,student_id):
1111        """remove picture_folder by renaming it"""
1112        path = 'images'
1113        picture_path = os.path.join(i_home,path,student_id)
1114        if not os.path.exists(picture_path):
1115            return False
1116        os.rename(picture_path,picture_path + "_removed")
1117        return True
1118    ###)
1119
1120    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1121    def restorePictureFolder(self,student_id):
1122        """restore picture_folder by renaming it"""
1123        path = 'images'
1124        picture_path = os.path.join(i_home,path,student_id)
1125        if not os.path.exists(picture_path + "_removed"):
1126            return False
1127        os.rename(picture_path + "_removed",picture_path)
1128        return True
1129    ###)
1130
1131    security.declarePublic('picturesExist') ###(
1132    def picturesExist(self, ids,student_id=None):
1133        """check if pictures exist in the filesystem"""
1134        if student_id is None:
1135            student_id = self.getStudentId()
1136        if student_id is None:
1137            return False
1138        picture_path = getImagesDir(student_id)
1139        #picture_path = os.path.join(images_base,student_id)
1140        if not os.path.exists(picture_path):
1141            return False
1142        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1143        return set(ids).issubset(set(pictures))
1144    ###)
1145
1146    security.declarePublic('picturesList') ###(
1147    def picturesList(self):
1148        """check if pictures exist in the filesystem"""
1149        path = 'images'
1150        student_id = self.getStudentId()
1151        #picture_path = os.path.join(i_home,path,student_id)
1152        picture_path = getImagesDir(student_id)
1153        if not os.path.exists(picture_path):
1154            return []
1155        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1156    ###)
1157
1158    security.declarePublic('showFsPicture') ###(
1159    def showFsPicture(self,path):
1160        """return a picture from the filesystem"""
1161        #picture_path = os.path.join(i_home,path)
1162        picture_path = os.path.join(images_base,path)
1163        response = self.REQUEST.RESPONSE
1164        #import pdb;pdb.set_trace()
1165        registry = getToolByName(self, 'mimetypes_registry')
1166        mimetype = str(registry.lookupExtension(path.lower()) or
1167                    registry.lookupExtension('file.bin'))
1168        if os.path.exists(picture_path):
1169            response.setHeader('Content-type',mimetype)
1170            return open(picture_path).read()
1171        picture_path = os.path.join(i_home,'import',path)
1172        if os.path.exists(picture_path):
1173            return open(picture_path).read()
1174    ###)
1175
1176    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1177    def deleteAllCourses(self,department="All"):
1178        ''' delete the courses'''
1179        pm = self.portal_membership
1180        member = pm.getAuthenticatedMember()
1181
1182        if str(member) not in ("henrik","joachim"):
1183            return "not possible"
1184        if department == "All":
1185            res = self.portal_catalog({'meta_type': 'Department'})
1186        if len(res) < 1:
1187            return "No Departments found"
1188
1189        deleted = []
1190        for dep in res:
1191            cf = dep.getObject().courses
1192            if cf:
1193                cf.manage_delObjects(ids=cf.objectIds())
1194                deleted.append("deleted Courses in %s" % dep.getId)
1195        return "\r".join(deleted)
1196    ###)
1197
1198    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1199    def getLogfileLines(self,filename="event.log",numlines=20):
1200        """Get last NUMLINES lines of logfile FILENAME.
1201
1202        Return last lines' of a file in the instances logfile directory as
1203        a list. The number of returned lines equals `numlines' or less. If
1204        less than `numlines' lines are available, the whole file ist
1205        returned. If the file can not be opened or some other error
1206        occurs, empty list is returend.
1207        """
1208        result = []
1209        lines_hit = 0
1210
1211        # We only handle files in instances' log directory...
1212        logpath = os.path.join(i_home, "log")
1213        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1214        if not filename.startswith( logpath ):
1215            # Attempt to access file outside log-dir...
1216            return []
1217
1218        try:
1219            fd = file( filename, "rb" )
1220        except IOError:
1221            return []
1222        if not fd:
1223            return []
1224
1225        if os.linesep == None:
1226            linesep = '\n'
1227        else:
1228            linesep = os.linesep
1229
1230        # Try to find 'numlines' times a lineseparator, searching from end
1231        # and moving to the beginning of file...
1232        fd.seek( 0, 2) # Move to end of file...
1233        while lines_hit < numlines:
1234            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1235                                          # one step forward...
1236                lines_hit += 1
1237            try:
1238                fd.seek( -2, 1) # Go two bytes back from current pos...
1239            except IOError:
1240                # We cannot go back two bytes. Maybe the file is too small...
1241                break
1242        fd.seek(2,1)
1243
1244        # Read all lines from current position...
1245        result = fd.readlines()
1246        # Remove line endings...
1247        result = [x.strip() for x in result]
1248        fd.close()
1249        return result
1250    ###)
1251
1252    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1253    def getCallbacksFromLog(self,filename):
1254        """fix Online Payment Transactions from Z2.log entries"""
1255        import transaction
1256        import random
1257        from cgi import parse_qs
1258        from urlparse import urlparse
1259        #from pdb import set_trace
1260        wftool = self.portal_workflow
1261        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1262        students_folder = self.portal_url.getPortalObject().campus.students
1263        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1264        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1265        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1266        data = re.compile(s)
1267        start = True
1268        tr_count = 1
1269        total = 0
1270        #name = 'pume_results'
1271        #name = 'epaymentsuccessful_z2log2'
1272        name = filename
1273        no_import = []
1274        imported = []
1275        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1276        try:
1277            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1278        except:
1279            logger.error('Error reading %s' % name)
1280            return
1281        tas = []
1282        for line in transactions:
1283            dict = {}
1284            items = data.search(line)
1285            dict['idict'] = idict = items.groupdict()
1286            #print idict
1287            #from pdb import set_trace;set_trace()
1288            urlparsed = urlparse(idict['get'][4:])
1289            #print urlparsed
1290            path = urlparsed[2].split('/')
1291            dict['student_id'] = student_id = path[8]
1292            dict['payment_id'] = payment_id = path[10]
1293            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1294            tas.append(dict)
1295            tr_count += 1
1296        return tas
1297    ###)
1298
1299    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1300    def importOnlinePaymentTransactions(self):
1301        """load Online Payment Transactions from CSV values"""
1302        import transaction
1303        import random
1304        #from pdb import set_trace
1305        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1306        opt = self.online_payments_import
1307        students_folder = self.portal_url.getPortalObject().campus.students
1308        start = True
1309        tr_count = 1
1310        total = 0
1311        #name = 'pume_results'
1312        name = 'OnlineTransactions'
1313        no_import = []
1314        imported = []
1315        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1316        try:
1317            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1318        except:
1319            logger.error('Error reading %s.csv' % name)
1320            return
1321        for pay_transaction in transactions:
1322            if start:
1323                start = False
1324                logger.info('Start loading from %s.csv' % name)
1325                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1326                no_import.append('%s,"Error"' % s)
1327                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1328                format_error = format + ',"%(Error)s"'
1329            data = {}
1330
1331            # format of the first file sent by Tayo
1332            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1333            #data['student_id'] = student_id = pay_transaction['Payer ID']
1334            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1335            #data['response_code'] = response_code = pay_transaction['Resp Code']
1336            #data['amount'] = amount = pay_transaction['Amount']
1337
1338            # format of the second file sent by Tayo
1339            #data['datetime'] = date = 0
1340            #data['student_id'] = student_id = pay_transaction['Payer ID']
1341            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1342            #data['response_code'] = response_code = '00'
1343            #data['amount'] = amount = pay_transaction['Amount']
1344
1345            # format of the third file sent by Kehinde
1346            data['datetime'] = date = 0
1347            data['student_id'] = student_id = pay_transaction['customer_id']
1348            data['order_id'] = order_id = pay_transaction['merchant_reference']
1349            data['response_code'] = response_code = '00'
1350            data['amount'] = amount = pay_transaction['Amount']
1351
1352            dup = False
1353            if response_code == "12":
1354                continue
1355            try:
1356                opt.addRecord(**data)
1357            except ValueError:
1358                dup = True
1359            #from pdb import set_trace;set_trace()
1360            if dup:
1361                if response_code == "00":
1362                    try:
1363                        opt.modifyRecord(**data)
1364                    except:
1365                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1366                        continue
1367                else:
1368                    pay_transaction['Error'] = "Duplicate order_id"
1369                    no_import.append( format_error % pay_transaction)
1370                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1371                    continue
1372            tr_count += 1
1373            if tr_count > 1000:
1374                if len(no_import) > 0:
1375                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1376                             '\n'.join(no_import) + '\n')
1377                    no_import = []
1378                em = '%d transactions committed\n' % (tr_count)
1379                transaction.commit()
1380                regs = []
1381                logger.info(em)
1382                total += tr_count
1383                tr_count = 0
1384        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1385                                                '\n'.join(no_import))
1386        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1387    ###)
1388
1389    security.declareProtected(ModifyPortalContent,"importData")###(
1390    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1391        """load data from CSV values"""
1392        import transaction
1393        import random
1394        students_folder = self.portal_url.getPortalObject().campus.students
1395        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1396        pending_only = False
1397        pend_str = '--'
1398        elapse = time.time()
1399        #
1400        # preparations
1401        #
1402        if filename == pend_str:
1403            pending_only = True
1404        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1405        importer = eval("%sImport" % importer_name)(self)
1406        logger = importer.logger
1407        if importer.init_errors:
1408            logger.info(importer.init_errors)
1409            return importer.init_errors
1410        member = importer.member
1411        #current = importer.current
1412        import_date = importer.import_date
1413        #
1414        # not_imported
1415        #
1416        info = importer.info
1417        data_keys = importer.data_keys
1418        csv_keys = importer.csv_keys
1419        #csv_keys.extend(info.keys())
1420        headline_mapping = dict((k,k) for k in csv_keys)
1421        #
1422        # pending
1423        #
1424        pending_path = importer.pending_path
1425        pending_tmp = importer.pending_tmp
1426        pending_backup = importer.pending_backup
1427        pending_fn = importer.pending_fn
1428        imported_path = importer.imported_path
1429        imported_fn = importer.imported_fn
1430        commit_after = importer.commit_after
1431        pending = []
1432        pending_digests = []
1433        #total_added_to_pending = 0
1434        if not pending_only:
1435            pending,pending_digests = importer.makeIdLists()
1436            pending_at_start = len(pending)
1437        datafile = open(pending_tmp,"w")
1438        pending_csv_writer = csv.DictWriter(datafile,
1439                                                    csv_keys,
1440                                                    extrasaction='ignore')
1441        pending_csv_writer.writerow(headline_mapping)
1442        datafile.close()
1443        #
1444        # imported
1445        #
1446        if not os.path.exists(imported_path):
1447            datafile = open(imported_path,"w")
1448            imported_csv_writer = csv.DictWriter(datafile,
1449                                                 csv_keys,
1450                                                 extrasaction='ignore')
1451            imported_csv_writer.writerow(headline_mapping)
1452            datafile.close()
1453        start = True
1454        tr_count = 0
1455        total = 0
1456        total_added_to_imported = 0
1457        total_pending = 0
1458        import_source_done = ""
1459        if pending_only:
1460            import_source_path = pending_path
1461        else:
1462            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1463            import_source_done = "%s/import/%s.done" % (i_home,filename)
1464        if not os.path.exists(import_source_path):
1465            fn = os.path.split(import_source_path)[1]
1466            em = 'no import file %(fn)s' % vars()
1467            return em
1468        import_source_fn = os.path.split(import_source_path)[1]
1469        if not pending_only:
1470            info['imported_from'] = import_source_fn
1471        headline = csv.reader(open(import_source_path,"rb")).next()
1472        if "import_mode" not in headline:
1473            msg = 'import_mode must be in heading'
1474            return msg
1475        invalid_keys = importer.checkHeadline(headline)
1476        if invalid_keys:
1477            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1478
1479        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1480                                                         or k.strip() in info.keys())]
1481        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1482        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1483        # if diff2schema and diff2schema != set(['id',]):
1484        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1485        #     return msg
1486        #
1487        # start importing
1488        #
1489        try:
1490            reader = csv.DictReader(open(import_source_path,"rb"))
1491        except:
1492            msg = 'Error reading %s.csv' % filename
1493            logger.error(msg)
1494            return msg
1495        items = [item for item in reader]
1496        total_to_import = len(items)
1497        tti_float = float(total_to_import)
1498        if pending_only:
1499            pending_at_start = total_to_import
1500        count = 0
1501        imported = []
1502        old_commit_count = -1
1503        error_count = imported_count = 0
1504        already_in = 0
1505        for record in items:
1506            item = {}
1507            empty_value_keys = []
1508            for k,v in record.items():
1509                if k is None:
1510                    continue
1511                if v:
1512                    if v == EMPTY:
1513                        empty_value_keys += k,
1514                        v = ''
1515                    item[k.strip()] = v.strip()
1516            count += 1
1517            if start:
1518                start = False
1519                adapters = [MappingStorageAdapter(importer.schema, item)]
1520                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1521            dm = DataModel(item, adapters,context=self)
1522            ds = DataStructure(data=item,datamodel=dm)
1523            error_string = ""
1524            total += 1
1525            import_mode = item.get('import_mode','')
1526            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1527            if import_method is None:
1528                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1529            elif (import_mode in importer.required_modes and
1530                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1531                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1532                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1533            else:
1534                for k in import_keys:
1535                    if k not in item.keys() or k not in importer.validators.keys():
1536                        continue
1537                    if not importer.validators[k](ds,mode=import_mode):
1538                        error_string += ' ++ '
1539                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1540                                                                           ds.getErrorMapping(k)))
1541            if error_string:
1542                error = error_string
1543                id = ''
1544                mapping = item
1545            else:
1546                temp_item = item.copy()
1547                temp_item.update(dm)
1548                results = import_method(temp_item)
1549                id = results[0]
1550                error = results[1]
1551                mapping = results[2]
1552            #
1553            # restore some values
1554            #
1555            for k in empty_value_keys:
1556                mapping[k] = EMPTY
1557            if mapping.has_key('sex'):
1558                #import pdb;pdb.set_trace()
1559                if mapping['sex'] == True:
1560                    mapping['sex'] = 'F'
1561                elif mapping['sex'] == False:
1562                    mapping['sex'] = 'M'
1563            if pending_only:
1564                info['imported_from'] = item['imported_from']
1565            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1566            info['error'] = error
1567            info['import_record_no'] = count + 1
1568            mapping.update(info)
1569            log_list = []
1570            if error:
1571                error_count += 1
1572                digest = makeDigest(mapping,data_keys)
1573                if digest not in pending_digests:
1574                    pending_digests += digest,
1575                    pending.append(mapping)
1576                    if not pending_only:
1577                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1578                else:
1579                    already_in += 1
1580                    pass
1581            else:
1582                imported_count += 1
1583                imported += mapping,
1584                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1585            if log_list:
1586                time_till_now = time.time() - elapse
1587                percent_finished = (error_count + imported_count)/tti_float*100
1588                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1589                logger.info(' '.join(log_list))
1590            finished = count > total_to_import - 1
1591            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1592            if must_commit:
1593                old_commit_count = imported_count
1594
1595            if must_commit or finished:
1596                if len(imported):
1597                    transaction.commit()
1598                    datafile = open(imported_path,"a")
1599                    writer = csv.DictWriter(datafile,
1600                                            csv_keys,
1601                                            extrasaction='ignore')
1602                    writer.writerows(imported)
1603                    datafile.close()
1604                    total_added_to_imported += len(imported)
1605                    imported = []
1606                if len(pending) > 0:
1607                    datafile = open(pending_tmp,"a")
1608                    writer = csv.DictWriter(datafile,
1609                                            csv_keys,
1610                                            extrasaction='ignore')
1611                    writer.writerows(pending)
1612                    datafile.close()
1613                    total_pending += len(pending)
1614                    #total_added_to_pending += len(pending)
1615                    pending = []
1616                if not finished:
1617                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1618                    logger.info(msg)
1619        elapse = time.time() - elapse
1620        if os.path.exists(pending_path):
1621            copy2(pending_path,pending_backup)
1622        copy2(pending_tmp,pending_path)
1623        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1624        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1625        if pending_only:
1626            removed_pending = pending_at_start - total_pending
1627            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1628        else:
1629            added_pending = total_pending - pending_at_start
1630            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1631        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1632        logger.info(msg)
1633        if import_source_done:
1634            copy(import_source_path,import_source_done)
1635            os.remove(import_source_path)
1636            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1637            if upload is not None:
1638                upload_doc = upload.getContent()
1639                mapping = {}
1640                #mapping['import_date'] = DateTime.DateTime()
1641                mapping['import_date'] = import_date
1642                mapping['imported_by'] = importer.imported_by
1643                mapping['import_message'] = msg
1644                upload_doc.edit(mapping = mapping)
1645        os.remove(pending_tmp)
1646        return msg
1647    ###)
1648
1649    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1650    def moveImagesToFS(self,student_id="O738726"):
1651        "move the images to the filesystem"
1652        images_dir = getImagesDir(student_id)
1653        #images_dir = os.path.join("%s" % images_base,student_id)
1654        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1655        stool = getToolByName(self, 'portal_schemas')
1656        schemas = ['student_application',
1657                   'student_clearance',
1658                   ]
1659        created = False
1660        for schema_id in schemas:
1661            schema = stool._getOb(schema_id)
1662            object = getattr(student_folder,schema_id[len('student_'):],None)
1663            if object is None:
1664                continue
1665            doc = object.getContent()
1666            for key in schema.keys():
1667                if schema[key].meta_type != "CPS Image Field":
1668                    continue
1669                #import pdb;pdb.set_trace()
1670                image = getattr(doc,key,None)
1671                if not image or not hasattr(image,"data"):
1672                    continue
1673                if not created:
1674                    if not os.path.exists(images_dir):
1675                        os.mkdir(images_dir)
1676                    created = True
1677                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1678                open(filename,"wb").write(str(image.data))
1679    ###)
1680
1681    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1682    def movePassportToFS(self,student_id="O738726"):
1683        "move the passports to the filesystem"
1684        images_dir = os.path.join("%s" % i_home,'passports')
1685        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1686        stool = getToolByName(self, 'portal_schemas')
1687        schemas = ['student_application',
1688                   #'student_clearance',
1689                   ]
1690        created = False
1691        for schema_id in schemas:
1692            schema = stool._getOb(schema_id)
1693            object = getattr(student_folder,schema_id[len('student_'):],None)
1694            if object is None:
1695                continue
1696            doc = object.getContent()
1697            for key in schema.keys():
1698                if schema[key].meta_type != "CPS Image Field":
1699                    continue
1700                #import pdb;pdb.set_trace()
1701                image = getattr(doc,key)
1702                if not hasattr(image,"data"):
1703                    continue
1704                if not created:
1705                    if not os.path.exists(images_dir):
1706                        os.mkdir(images_dir)
1707                    created = True
1708                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1709                open(filename,"wb").write(str(image.data))
1710    ###)
1711
1712InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.