source: WAeUP_SRP/base/WAeUPTool.py @ 3308

Last change on this file since 3308 was 3308, checked in by Henrik Bettermann, 17 years ago

set correct import_date and imported_by
  • Property svn:keywords set to Id
File size: 69.4 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3308 2008-03-10 07:13:11Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108
109###)
110
111    security.declareProtected(View,'updateRoleMappingsFor') ###(
112    def updateRoleMappingsFor(self,wf_definition,ob):
113        "do so for public"
114        wf_def = getattr(self.portal_workflow,wf_definition)
115        wf_def.updateRoleMappingsFor(ob)
116    ###)
117
118    security.declareProtected(View,'getStatesLgas') ###(
119    def getStatesLgas(self):
120        """return lga info"""
121        voc = getattr(self.portal_vocabularies,'local_gov_areas')
122        states = []
123        lgas  = []
124        d = {}
125        wd = {}
126        for k,v in voc.items():
127            parts = v.split(' / ')
128            if len(parts) == 1:
129                state = parts[0].lower()
130                lga = ""
131            elif len(parts) == 2:
132                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134            else:
135                continue
136            if state not in states:
137                states.append(state)
138            if lga not in lgas:
139                lgas.append(lga)
140            words = re.split('[^a-zA-Z0-9/]',k)
141            words.sort()
142            wd[k] = words
143            d[k] = v
144        mapping = {}
145        mapping['word_dict'] = wd
146        mapping['lga_dict'] = d
147        mapping['states'] = states
148        mapping['lgas'] = lgas
149        return mapping
150    ###)
151
152    security.declareProtected(View,'findLga') ###(
153    def findLga(self,words,words_dict):
154        words = re.split('[^a-zA-Z0-9/]',words)
155        lga_words = []
156        for word in words:
157            if word:
158                lga_words += word.strip().lower(),
159        lga_words.sort()
160        state_lga = ''
161        while not state_lga:
162            for k,l in words_dict.items():
163                if lga_words == l:
164                    state_lga = k
165                    break
166            break
167        return state_lga
168    ###)
169    security.declareProtected(View,'getAccessInfo') ###(
170    def getAccessInfo(self,context):
171        "return a dict with access_info"
172        logger = logging.getLogger('WAeUPTool.getAccessInfo')
173        mtool = self.portal_membership
174        member = mtool.getAuthenticatedMember()
175        member_id = str(member)
176        info = {}
177        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179        is_staff = info['is_staff'] = not is_anonymous and not is_student
180        roles = member.getRolesInContext(context)
181        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182                                                                            "SectionManager" in roles or
183                                                                            "Manager" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif is_anonymous or member_id != requested_id:
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378
379###)
380
381    def generateStudentId(self,letter): ###(
382        import random
383        r = random
384        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385        if letter == '?':
386            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387        sid = "%c%d" % (letter,r.randint(99999,1000000))
388        students = self.portal_url.getPortalObject().campus.students
389##        while hasattr(students, sid):
390##            sid = "%c%d" % (letter,r.randint(99999,1000000))
391        while self.students_catalog(id = sid):
392            sid = "%c%d" % (letter,r.randint(99999,1000000))
393        return sid
394    ###)
395
396    def generatePassword(self,s=None): ###(
397        import random
398        r = random
399        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400        if s is None:
401            s = 'abcdefghklmnpqrstuvwxy23456789'
402        pw = ''
403        while len(pw) < 6:
404            pw += r.choice(s)
405        return pw
406    ###)
407
408    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409    def dumpSchoolfeePayments(self):
410        "dump paid schoolfees"
411        mtool = self.portal_membership
412        member = mtool.getAuthenticatedMember()
413        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414        aq_student = self.students_catalog.evalAdvancedQuery
415        query = In('review_state',('schoolfee_paid',
416                                   'courses_registered',
417                                   'courses_validated',
418                                   ))
419        res = aq_student(query)
420        #import pdb;pdb.set_trace()
421        l = []
422        logger.info("start for %d" % len(res))
423        count = 1
424        log_after = 100
425        for student in res:
426            if not count % log_after:
427                logger.info("processed %d total %d" % (log_after,count))
428            count += 1
429            fee_dict =self.getSchoolFee(student)
430            fulltime = student.mode.endswith('_ft')
431            d = {}
432            d['student_id'] = student.id
433            d['name'] = student.name
434            d['amount'] = fee_dict.get(new_returning)
435            l += d,
436        csv_name = self.dumpListToCSV(l,'payments')
437        logger.info('%s dumped payments to %s' % (member,export_file))
438    ###)
439
440    security.declarePublic('dumpListToCSV') ###(
441    def dumpListToCSV(self,l,filename,fields=None):
442        """dump a list of dicts to a CSV file"""
443        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445        if fields is None:
446            fields = l[0].keys()
447        headline = ','.join(fields)
448        out = open(export_file,"wb")
449        out.write(headline +'\n')
450        out.close()
451        out = open(export_file,"a")
452        csv_writer = csv.DictWriter(out,fields,)
453        csv_writer.writerows(l)
454        return export_file
455    ###)
456
457    security.declareProtected(ManagePortal, 'listMembers') ###(
458    def listMembers(self):
459        "list all members"
460        mtool = self.portal_membership
461        member = mtool.getAuthenticatedMember()
462        logger = logging.getLogger('WAeUPTool.listMembers')
463        if str(member) not in ('admin','joachim'):
464            logger.info('%s tried to list members' % (member))
465            return None
466        members = self.portal_directories.members
467        all = members.listEntryIdsAndTitles()
468        l = []
469        for user_id,name in all:
470            d = {}
471            d['user_id'] = user_id
472            d['name'] = name
473            d['pw'] = getattr(getattr(members,user_id),'password')
474            d['email'] = getattr(getattr(members,user_id),'email')
475            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477            l += d,
478        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480        logger.info('%s dumped member list to %s' % (member,export_file))
481        fields = l[0].keys()
482        headline = ','.join(fields)
483        out = open(export_file,"wb")
484        out.write(headline +'\n')
485        out.close()
486        out = open(export_file,"a")
487        csv_writer = csv.DictWriter(out,fields,)
488        csv_writer.writerows(l)
489    ###)
490
491    security.declareProtected(ManagePortal, 'listStudents') ###(
492    def listStudents(self):
493        "list all students"
494        mtool = self.portal_membership
495        member = mtool.getAuthenticatedMember()
496        logger = logging.getLogger('WAeUPTool.listStudents')
497        if str(member) not in ('admin','joachim'):
498            logger.info('%s tried to list students' % (member))
499            return None
500        students = self.portal_directories.students
501        all = students.listEntryIdsAndTitles()
502        l = []
503        for user_id,name in all:
504            d = {}
505            d['user_id'] = user_id
506            d['name'] = name
507            d['pw'] = getattr(getattr(students,user_id),'password')
508            d['email'] = getattr(getattr(students,user_id),'email')
509            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511            l += d,
512        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514        logger.info('%s dumped student list to %s' % (member,export_file))
515        fields = l[0].keys()
516        headline = ','.join(fields)
517        out = open(export_file,"wb")
518        out.write(headline +'\n')
519        out.close()
520        out = open(export_file,"a")
521        csv_writer = csv.DictWriter(out,fields,)
522        csv_writer.writerows(l)
523    ###)
524
525    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526    def removeDeletedDocIds(self, max=1000):
527        """
528        remove deleted docids from repository commit after max
529        """
530        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531        repository = getToolByName(self, 'portal_repository')
532        pxtool = getToolByName(self, 'portal_proxies')
533        logger.info('start')
534        pxtool_infos = pxtool.getRevisionsUsed()
535        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
536
537        nb_revs = 0
538        docids_d = {} # all docids
539        unused_docids_d = {} # all docids that are unused
540        ids_unused_revs_docids = [] # ids for revs of unused docids
541        unused_ids = [] # ids for unused revs
542        total = 0
543        idlist = repository.objectIds()
544        to_delete = 0
545        found = False
546        for id in idlist:
547            docid, rev = repository._splitId(id)
548            if docid is None:
549                logger.info("invalid doc_id %s" % docid)
550                continue
551            nb_revs += 1
552            if not pxtool_infos.has_key(docid):
553                found = True
554                to_delete += 1
555                unused_ids.append(id)
556            elif not pxtool_infos[docid].has_key(rev):
557                found = True
558                to_delete += 1
559                unused_ids.append(id)
560            if found and not to_delete % max:
561                found = False
562                #import pdb;pdb.set_trace()
563                repository.manage_delObjects(unused_ids)
564                transaction.commit()
565                logger.info('removed %d total %d unused docids ' % (max,to_delete))
566        else:
567            if unused_ids:
568                repository.manage_delObjects(unused_ids)
569                transaction.commit()
570        logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573###)
574
575    security.declareProtected(View,'getCredential') ###(
576    def getCredential(self,student_id):
577        student_entry = getattr(self.portal_directories.students,student_id,None)
578        if not self.isStaff():
579            mtool = self.portal_membership
580            member = mtool.getAuthenticatedMember()
581            logger = logging.getLogger('WAeUPTool.getCredential')
582            logger.info('%s tried to access password of %s' % (member,student_id))
583            return None
584        if student_entry is None:
585            return None
586        return getattr(student_entry,"password","not set")
587    ###)
588
589    security.declarePublic('checkPassword') ###(
590    def checkPassword(self,student_id,password):
591        student_entry = getattr(self.portal_directories.students,student_id,None)
592        if student_entry is None:
593            return False
594        return getattr(student_entry,"password","not set") == password
595    ###)
596
597    security.declarePublic('checkGenericPassword') ###(
598    def checkGenericPassword(self,member_id):
599        member_entry = getattr(self.portal_directories.members,member_id,None)
600        if member_entry is None:
601            return False
602        ltool = getToolByName(self, 'portal_layouts')
603        unsecure_words = ltool._getOb('members')['w__password'].check_words
604        password = getattr(member_entry,"password","not set")
605        is_unsecure = password in unsecure_words
606        if is_unsecure:
607            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609        return is_unsecure
610    ###)
611
612    security.declareProtected(ModifyPortalContent,'editPassword') ###(
613    def editPassword(self,student_id,password):
614        "edit a student password"
615        student_entry = getattr(self.portal_directories.students,student_id,None)
616        if student_entry is None:
617            return
618        setattr(student_entry,'password',password)
619    ###)
620
621    security.declareProtected(ModifyPortalContent,'doCommit') ###(
622    def doCommit(self,logger=None):
623        "commit some transactions"
624        transaction.commit()
625    ###)
626
627    security.declarePublic('loadStudentFoto') ###(
628    def loadStudentFoto(self,student,filename,folder):
629        "return a student passport picture"
630        #import pdb;pdb.set_trace()
631        picture ="%s/import/%s/%s" % (i_home,folder,filename)
632        student_id = student.getId()
633        images_dir = getImagesDir(student_id)
634        if not os.path.exists(images_dir):
635            os.mkdir(images_dir)
636        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637        for extension in ('.jpg','.JPG'):
638            fullname = "%(picture)s%(extension)s" % vars()
639            if os.path.exists(fullname):
640                copy2(fullname,image_name)
641                return "successfully copied passport picture"
642        return "passport picture not found: %s.jpg or .JPG" % picture
643    ###)
644
645    def old____loadStudentFoto(self,student,filename,folder): ###(
646        "return a student passport picture"
647        app = student.application
648        app_doc = app.getContent()
649        #clear = student.clearance
650        #clear_doc = clear.getContent()
651        #matric_no = clear_doc.matric_no.upper()
652        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654        #import pdb;pdb.set_trace()
655        if os.path.exists(picture1):
656            file = open(picture1)
657        elif os.path.exists(picture2):
658            file = open(picture2)
659        else:
660            return "passport picture not found %s" % picture1
661        reopened = False
662        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663            self.portal_workflow.doActionFor(app,'open')
664            reopened = True
665        outfile = file.read()
666        app_doc.manage_addFile('passport',
667                               file=outfile,
668                               title="%s.jpg" % filename)
669        if reopened:
670            self.portal_workflow.doActionFor(app,'close')
671        return "successfully loaded passport picture"
672    ###)
673
674    security.declareProtected(ModifyPortalContent,'createOne') ###(
675    def createOne(self,students_folder,student_brain,letter,commit=False):
676        sid = self.waeup_tool.generateStudentId(letter)
677        students_folder.invokeFactory('Student', sid)
678        student = getattr(students_folder,sid)
679        self.portal_workflow.doActionFor(student,'return')
680        student.manage_setLocalRoles(sid, ['Owner',])
681        matric_no = student_brain.matric_no
682        jamb_reg_no = student_brain.Entryregno
683        self.students_catalog.addRecord(id = sid,
684                                           matric_no = matric_no,
685                                           jamb_reg_no = jamb_reg_no,
686                                           sex = student_brain.Sex == "F",
687                                           name = "%s %s %s" % (student_brain.Firstname,
688                                                                student_brain.Middlename,
689                                                                student_brain.Lastname)
690                                        )
691        if commit:
692            transaction.commit()
693        return sid,jamb_reg_no
694    ###)
695
696    security.declareProtected(ModifyPortalContent,'addStudent') ###(
697    def addStudent(self,dict):
698        students_folder = self.portal_url.getPortalObject().campus.students
699        sid = self.waeup_tool.generateStudentId('?')
700        students_folder.invokeFactory('Student', sid)
701        student_obj = getattr(students_folder,sid)
702        f2t = self.field2types_student
703        #from pdb import set_trace; set_trace()
704        d = {}
705        #d['jamb_sex']  = 'M'
706        #if dict.get('sex'):
707        #    d['jamb_sex']  = 'F'
708
709        entry_session = dict.get('entry_session')
710        if entry_session == self.getSessionId()[0]:
711            wfaction = 'admit'
712            wft = 'wf_transition_admit'
713            password = None
714        else:
715            wfaction = 'return'
716            wft = 'wf_transition_return'
717            password = self.generatePassword()
718            self.makeStudentMember(sid,password)
719
720        for pt in f2t.keys():
721            student_obj.invokeFactory(pt,f2t[pt]['id'])
722            sub_obj = getattr(student_obj,f2t[pt]['id'])
723            sub_doc = sub_obj.getContent()
724            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725            d['Title'] = f2t[pt]['title']
726            for field in f2t[pt]['fields']:
727                d[field] = dict.get(field,'')
728            sub_doc.edit(mapping = d)
729            new_state = f2t[pt][wft]
730            if new_state != "remain":
731                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732        self.portal_workflow.doActionFor(student_obj,wfaction)
733        student_obj.manage_setLocalRoles(sid, ['Owner',])
734        return sid,password
735    ###)
736
737    security.declarePublic('getCertificateBrain') ###(
738    def getCertificateBrain(self,cert_id):
739        "do it"
740        res = ZCatalog.searchResults(self.portal_catalog_real,
741                                {'portal_type':"Certificate",
742                                      'id': cert_id})
743        if res:
744            return res[0]
745        return None
746    ###)
747
748    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749    def get_csv_filenames(self):
750        "do it"
751        files = [file for file in os.listdir("%s/import/" % (i_home))
752                 if file.endswith('.csv') and (file.find('imported') == -1 and
753                                               file.find('pending') == -1)]
754        return files
755    ###)
756
757    security.declarePublic('findStudentByMatricelNo') ###(
758    def findStudentByMatricelNo(self,matric_no):
759        "do it"
760        res = ZCatalog.searchResults(self.portal_catalog_real,
761                                {'portal_type':"StudentClearance",
762                                 'SearchableText': matric_no})
763        if res:
764            return res[0]
765        return None
766    ###)
767
768    security.declarePublic('makeStudentMember') ###(
769    def makeStudentMember(self,sid,password='uNsEt'):
770        """make the student a member"""
771        membership = self.portal_membership
772        membership.addMember(sid,
773                             password ,
774                             roles=('Member',
775                                     'Student',
776                                     ),
777                             domains='',
778                             properties = {'memberareaCreationFlag': False,
779                                           'homeless': True},)
780        member = membership.getMemberById(sid)
781        self.portal_registration.afterAdd(member, sid, password, None)
782        #self.manage_setLocalRoles(sid, ['Owner',])
783    ###)
784
785    security.declareProtected(View,'makeStudentData') ###(
786    def makeStudentData(self,student_id,email=None,phone_nr=None):
787        "create Datastructure for a returning Student"
788        #import pdb;pdb.set_trace()
789        logger = logging.getLogger('WAeUPTool.makeStudentData')
790        students_folder = self.portal_url.getPortalObject().campus.students
791        #res = self.students_catalog(id=student_id)
792        #if res:
793        #    st = res[0]
794        #res = self.returning_import(matric_no = st.matric_no)
795        res = self.returning_import(id = student_id)
796        if res:
797            student = res[0]
798        else:
799            logger.info('Id %s not found in returning_import' % student_id)
800            return
801        logger.info('%s creates data structure' % student_id)
802        s_results = self.results_import(matric_no = student.matric_no)
803        if s_results:
804            lnr = self.getLevelFromResultsCosCode(s_results)
805            level = "%d00" % lnr
806            verdict,eligible = self.getVerdict(s_results[0].Verdict)
807            #if eligible:
808            #    level = "%d00" % (lnr + 1)
809        else:
810            logger.info('matric_no %s not found in results_import' % student.matric_no)
811            level = ''
812            verdict = ''
813        #student should not be allowed to perform this transition
814        #wftool = self.portal_workflow
815        #wftool.doActionFor(student,'return')
816        certcode_org = student.Coursemajorcode
817        certcode = makeCertificateCode(certcode_org)
818        certificate_brain = self.getCertificateBrain(certcode)
819        if not certificate_brain:
820            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821            logger.info(em)
822        matric_no = student.matric_no
823        sid = student_id
824        student_obj = getattr(students_folder,sid)
825        if not getattr(student_obj,'application'):
826            student_obj.invokeFactory('StudentApplication','application')
827        application = student_obj.application
828        self.portal_workflow.doActionFor(application,'open',dest_container=application)
829        da = {'Title': 'Application Data'}
830        student_obj.invokeFactory('StudentPersonal','personal')
831        da['jamb_reg_no'] = student.Entryregno
832        em = self.getEntryMode(student.Entryregno)
833        da['entry_mode'] = em
834        personal = student_obj.personal
835        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836        dp = {'Title': 'Personal Data'}
837        student_obj.invokeFactory('StudentClearance','clearance')
838        clearance = student_obj.clearance
839        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840        dc = {'Title': 'Clearance/Eligibility Record'}
841        dc['matric_no'] = matric_no
842        state = student.State
843        lga = student.LGA
844        if state and lga:
845            lga =  state + ' / ' + lga
846        else:
847            lga = "None"
848        da['jamb_lga'] = dc['lga'] = lga
849        da['app_email'] = dp['email'] = email
850        da['app_mobile'] = dp['phone'] = phone_nr
851        dp['firstname'] = student.Firstname
852        dp['middlename'] = student.Middlename
853        dp['lastname'] = student.Lastname
854        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855        da['jamb_sex'] = student.Sex
856        dp['sex'] = student.Sex == 'F'
857        dp['perm_address'] = student.Permanent_Address
858        application.getContent().edit(mapping=da)
859        self.portal_workflow.doActionFor(application,'close',dest_container=application)
860        personal.getContent().edit(mapping=dp)
861        clearance.getContent().edit(mapping=dc)
862        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863        #
864        # Study Course
865        #
866        student_obj.invokeFactory('StudentStudyCourse','study_course')
867        studycourse = student_obj.study_course
868        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869        dsc = {}
870        dsc['study_course'] = certcode
871        dsc['current_level'] = level
872        dsc['current_verdict'] = verdict
873        dsc['current_mode'] = em
874        dsc['current_session'] = '05'
875        studycourse.getContent().edit(mapping=dsc)
876        #
877        # Level
878        #
879##        l = getattr(studycourse,level,None)
880##        if l is None:
881##            studycourse.invokeFactory('StudentStudyLevel', level)
882##            l = getattr(studycourse, level)
883##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
884##            l.getContent().edit(mapping={'Title': "Level %s" % level})
885###)
886
887    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
888    def admitOneStudent(self,brain,entry_session,pin_password):
889        "create Datastructure for an admitted Student"
890        #import pdb;pdb.set_trace()
891        if not hasattr(self,"_v_certificates"):
892            self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()]
893        students_folder = self.portal_url.getPortalObject().campus.students
894        logger = logging.getLogger('WAeUPTool.admitOneStudent')
895        if brain.screening_type in ('cest','sandwich',):
896            reg_no = "%s%s/07" % (brain.course1,brain.serial)
897        else:
898            reg_no = brain.reg_no
899        res = self.students_catalog(jamb_reg_no = reg_no)
900        if res:
901            logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
902            return
903        if brain.status != "admitted":
904            logger.info('status of %s is %s' % (reg_no,brain.status))
905            return
906        pin_parts = brain.pin.split('-')
907        if pin_parts and len(pin_parts) != 3:
908            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
909            return
910        if brain.course_admitted not in self._v_certificates:
911            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
912            return
913        if brain.sex not in (True,False):
914            logger.info('sex of %s not available' % (reg_no))
915            return
916        student_id = self.generateStudentId('?')
917        students_folder.invokeFactory('Student', student_id)
918        student_object = getattr(students_folder,student_id)
919        if pin_password:
920            password = pin_parts[2]
921            self.makeStudentMember(student_id,password = password)
922        student_object.manage_setLocalRoles(student_id, ['Owner',])
923        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
924        #
925        # application
926        #
927        student_object.invokeFactory('StudentApplication','application')
928        application = student_object.application
929        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
930        da = {'Title': 'Application Data'}
931        da['jamb_reg_no'] = reg_no
932
933        sex = 'M'
934        if brain.sex:
935            sex = 'F'
936        da['jamb_sex'] = sex
937        da['jamb_age'] = brain.jamb_age
938        #da['app_ac_pin'] = brain.pin
939        da['jamb_lga'] = brain.jamb_lga
940        da['jamb_state'] = brain.jamb_state
941        da['jamb_score'] = brain.aggregate
942        da['app_email'] = brain.email
943        da['app_mobile'] = brain.phone
944        if brain.entry_mode:
945            da['entry_mode'] = brain.entry_mode
946        elif brain.screening_type == 'pume':
947            da['entry_mode'] = 'ume_ft'
948        elif brain.screening_type == 'pde':
949            da['entry_mode'] = 'de_ft'
950        elif brain.screening_type == 'pce':
951            da['entry_mode'] = 'pce'
952        elif brain.screening_type == 'prence':
953            da['entry_mode'] = 'prence'
954        else:
955            da['entry_mode'] = ''
956        da['entry_session'] = entry_session
957        da['jamb_lastname'] = brain.lastname
958        da['jamb_middlename'] = brain.middlenames   # different field names!
959        da['jamb_firstname'] = brain.firstname
960        da['screening_application_date'] = brain.application_date
961        da['date_of_birth'] = brain.date_of_birth
962        da['jamb_first_cos'] = brain.course1
963        da['jamb_second_cos'] = brain.course2
964        da['course3'] = brain.course3
965        da['screening_type'] = brain.screening_type
966        da['screening_score'] = brain.screening_score
967        da['screening_date'] = brain.screening_date
968        da['hq_type'] = brain.hq_type
969        da['hq_grade'] = brain.hq_grade
970        da['aos'] = brain.aos
971
972        application.getContent().edit(mapping=da)
973        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
974        #
975        # personal
976        #
977        student_object.invokeFactory('StudentPersonal','personal')
978        personal = student_object.personal
979        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
980        dp = {'Title': 'Personal Data'}
981        dp['sex'] = brain.sex
982        dp['email'] = brain.email
983        dp['phone'] = brain.phone
984        dp['lastname'] = brain.lastname
985        dp['middlename'] = brain.middlenames   # different field names!
986        dp['firstname'] = brain.firstname
987        personal.getContent().edit(mapping=dp)
988        #
989        # clearance
990        #
991        student_object.invokeFactory('StudentClearance','clearance')
992        clearance = student_object.clearance
993        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
994        dc = {'Title': 'Clearance/Eligibility Record'}
995        dc['lga'] = brain.lga
996        dc['birthday'] = brain.date_of_birth
997        clearance.getContent().edit(mapping=dc)
998        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
999        #
1000        # study Course
1001        #
1002        student_object.invokeFactory('StudentStudyCourse','study_course')
1003        studycourse = student_object.study_course
1004        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1005        dsc = {}
1006        dsc['study_course'] = brain.course_admitted
1007        dsc['current_verdict'] = ''
1008        dsc['current_mode'] = da['entry_mode']
1009        if da['entry_mode'].startswith('de'):
1010            dsc['current_level'] = '200'
1011        elif da['entry_mode'].startswith('pre'):
1012            dsc['current_level'] = '000'
1013        else:
1014            dsc['current_level'] = '100'
1015        dsc['current_session'] = entry_session
1016        studycourse.getContent().edit(mapping=dsc)
1017        #
1018        # payments folder
1019        student_object.invokeFactory('PaymentsFolder','payments')
1020        payments = getattr(student_object,'payments')
1021        dpay = {}
1022        dpay['Title'] = 'Payments'
1023        payments.getContent().edit(mapping=dpay)
1024        self.portal_workflow.doActionFor(payments,'open')
1025        #
1026        # passport foto
1027        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1028                                                              brain.screening_type,
1029                                                              brain.reg_no)
1030        images_dir = getImagesDir(student_id)
1031        #images_dir = os.path.join("%s" % images_base,student_id)
1032        letter_dir,student_dir = os.path.split(images_dir)
1033        if not os.path.exists(letter_dir):
1034            os.mkdir(letter_dir)
1035        if not os.path.exists(images_dir):
1036            os.mkdir(images_dir)
1037        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1038        if os.path.exists(app_picture):
1039            copy2(app_picture,image_name)
1040        else:
1041            logger.info('passport of %s/%s not found: %s' % (student_id,
1042                                                             brain.reg_no,
1043                                                             app_picture))
1044
1045        return student_id
1046    ###)
1047
1048    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1049    def makeStudentLevel(self,student_id):
1050        "create the StudyLevel for a returning Student"
1051        #import pdb;pdb.set_trace()
1052        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1053        students_folder = self.portal_url.getPortalObject().campus.students
1054        res = self.students_catalog(id=student_id)
1055        if res:
1056            st = res[0]
1057        course = st.course
1058        matric_no = st.matric_no
1059        level = st.level
1060        res = self.results_import(matric_no = matric_no)
1061        if res:
1062            results = res
1063        logger.info('%s creating Level %s' % (student_id,level))
1064        #
1065        # Level
1066        #
1067        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1068        studycourse = getattr(student_obj,"study_course",None)
1069        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1070        l = getattr(studycourse,level,None)
1071        if l is None:
1072            studycourse.invokeFactory('StudentStudyLevel', level)
1073            l = getattr(studycourse, level)
1074            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1075            l.getContent().edit(mapping={'Title': "Level %s" % level})
1076        ###)
1077
1078    security.declarePublic('getHallInfo') ###(
1079    def getHallInfo(self,bed):
1080        """return Hall Info"""
1081        info = {}
1082        hall,block,room,letter = bed.split('_')
1083        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1084        if res and len(res) == 1:
1085            hall_brain = res[0]
1086            hall_doc = hall_brain.getObject().getContent()
1087        else:
1088            return info
1089        info['hall_title'] = hall_brain.Title
1090        info['maintenance_code'] = hall_doc.maintenance_code
1091        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1092        batch_doc = None
1093        for brain in res:
1094            if brain.id.startswith(info['maintenance_code']):
1095                batch_doc = brain.getObject().getContent()
1096                break
1097        if batch_doc is None:
1098            info['maintenance_fee'] = ''
1099        else:
1100            info['maintenance_fee'] = batch_doc.cost
1101        return info
1102    ###)
1103
1104    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1105    def removePictureFolder(self,student_id):
1106        """remove picture_folder by renaming it"""
1107        path = 'images'
1108        picture_path = os.path.join(i_home,path,student_id)
1109        if not os.path.exists(picture_path):
1110            return False
1111        os.rename(picture_path,picture_path + "_removed")
1112        return True
1113    ###)
1114
1115    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1116    def restorePictureFolder(self,student_id):
1117        """restore picture_folder by renaming it"""
1118        path = 'images'
1119        picture_path = os.path.join(i_home,path,student_id)
1120        if not os.path.exists(picture_path + "_removed"):
1121            return False
1122        os.rename(picture_path + "_removed",picture_path)
1123        return True
1124    ###)
1125
1126    security.declarePublic('picturesExist') ###(
1127    def picturesExist(self, ids,student_id=None):
1128        """check if pictures exist in the filesystem"""
1129        if student_id is None:
1130            student_id = self.getStudentId()
1131        if student_id is None:
1132            return False
1133        picture_path = getImagesDir(student_id)
1134        #picture_path = os.path.join(images_base,student_id)
1135        if not os.path.exists(picture_path):
1136            return False
1137        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1138        return set(ids).issubset(set(pictures))
1139    ###)
1140
1141    security.declarePublic('picturesList') ###(
1142    def picturesList(self):
1143        """check if pictures exist in the filesystem"""
1144        path = 'images'
1145        student_id = self.getStudentId()
1146        #picture_path = os.path.join(i_home,path,student_id)
1147        picture_path = getImagesDir(student_id)
1148        if not os.path.exists(picture_path):
1149            return []
1150        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1151    ###)
1152
1153    security.declarePublic('showFsPicture') ###(
1154    def showFsPicture(self,path):
1155        """return a picture from the filesystem"""
1156        #picture_path = os.path.join(i_home,path)
1157        picture_path = os.path.join(images_base,path)
1158        response = self.REQUEST.RESPONSE
1159        #import pdb;pdb.set_trace()
1160        registry = getToolByName(self, 'mimetypes_registry')
1161        mimetype = str(registry.lookupExtension(path.lower()) or
1162                    registry.lookupExtension('file.bin'))
1163        if os.path.exists(picture_path):
1164            response.setHeader('Content-type',mimetype)
1165            return open(picture_path).read()
1166        picture_path = os.path.join(i_home,'import',path)
1167        if os.path.exists(picture_path):
1168            return open(picture_path).read()
1169    ###)
1170
1171    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1172    def deleteAllCourses(self,department="All"):
1173        ''' delete the courses'''
1174        pm = self.portal_membership
1175        member = pm.getAuthenticatedMember()
1176
1177        if str(member) not in ("henrik","joachim"):
1178            return "not possible"
1179        if department == "All":
1180            res = self.portal_catalog({'meta_type': 'Department'})
1181        if len(res) < 1:
1182            return "No Departments found"
1183
1184        deleted = []
1185        for dep in res:
1186            cf = dep.getObject().courses
1187            if cf:
1188                cf.manage_delObjects(ids=cf.objectIds())
1189                deleted.append("deleted Courses in %s" % dep.getId)
1190        return "\r".join(deleted)
1191    ###)
1192
1193    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1194    def getLogfileLines(self,filename="event.log",numlines=20):
1195        """Get last NUMLINES lines of logfile FILENAME.
1196
1197        Return last lines' of a file in the instances logfile directory as
1198        a list. The number of returned lines equals `numlines' or less. If
1199        less than `numlines' lines are available, the whole file ist
1200        returned. If the file can not be opened or some other error
1201        occurs, empty list is returend.
1202        """
1203        result = []
1204        lines_hit = 0
1205
1206        # We only handle files in instances' log directory...
1207        logpath = os.path.join(i_home, "log")
1208        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1209        if not filename.startswith( logpath ):
1210            # Attempt to access file outside log-dir...
1211            return []
1212
1213        try:
1214            fd = file( filename, "rb" )
1215        except IOError:
1216            return []
1217        if not fd:
1218            return []
1219
1220        if os.linesep == None:
1221            linesep = '\n'
1222        else:
1223            linesep = os.linesep
1224
1225        # Try to find 'numlines' times a lineseparator, searching from end
1226        # and moving to the beginning of file...
1227        fd.seek( 0, 2) # Move to end of file...
1228        while lines_hit < numlines:
1229            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1230                                          # one step forward...
1231                lines_hit += 1
1232            try:
1233                fd.seek( -2, 1) # Go two bytes back from current pos...
1234            except IOError:
1235                # We cannot go back two bytes. Maybe the file is too small...
1236                break
1237        fd.seek(2,1)
1238
1239        # Read all lines from current position...
1240        result = fd.readlines()
1241        # Remove line endings...
1242        result = [x.strip() for x in result]
1243        fd.close()
1244        return result
1245    ###)
1246
1247    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1248    def getCallbacksFromLog(self,filename):
1249        """fix Online Payment Transactions from Z2.log entries"""
1250        import transaction
1251        import random
1252        from cgi import parse_qs
1253        from urlparse import urlparse
1254        #from pdb import set_trace
1255        wftool = self.portal_workflow
1256        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1257        students_folder = self.portal_url.getPortalObject().campus.students
1258        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1259        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1260        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1261        data = re.compile(s)
1262        start = True
1263        tr_count = 1
1264        total = 0
1265        #name = 'pume_results'
1266        #name = 'epaymentsuccessful_z2log2'
1267        name = filename
1268        no_import = []
1269        imported = []
1270        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1271        try:
1272            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1273        except:
1274            logger.error('Error reading %s' % name)
1275            return
1276        tas = []
1277        for line in transactions:
1278            dict = {}
1279            items = data.search(line)
1280            dict['idict'] = idict = items.groupdict()
1281            #print idict
1282            #from pdb import set_trace;set_trace()
1283            urlparsed = urlparse(idict['get'][4:])
1284            #print urlparsed
1285            path = urlparsed[2].split('/')
1286            dict['student_id'] = student_id = path[8]
1287            dict['payment_id'] = payment_id = path[10]
1288            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1289            tas.append(dict)
1290            tr_count += 1
1291        return tas
1292    ###)
1293
1294    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1295    def importOnlinePaymentTransactions(self):
1296        """load Online Payment Transactions from CSV values"""
1297        import transaction
1298        import random
1299        #from pdb import set_trace
1300        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1301        opt = self.online_payments_import
1302        students_folder = self.portal_url.getPortalObject().campus.students
1303        start = True
1304        tr_count = 1
1305        total = 0
1306        #name = 'pume_results'
1307        name = 'OnlineTransactions'
1308        no_import = []
1309        imported = []
1310        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1311        try:
1312            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1313        except:
1314            logger.error('Error reading %s.csv' % name)
1315            return
1316        for pay_transaction in transactions:
1317            if start:
1318                start = False
1319                logger.info('Start loading from %s.csv' % name)
1320                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1321                no_import.append('%s,"Error"' % s)
1322                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1323                format_error = format + ',"%(Error)s"'
1324            data = {}
1325
1326            # format of the first file sent by Tayo
1327            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1328            #data['student_id'] = student_id = pay_transaction['Payer ID']
1329            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1330            #data['response_code'] = response_code = pay_transaction['Resp Code']
1331            #data['amount'] = amount = pay_transaction['Amount']
1332
1333            # format of the second file sent by Tayo
1334            #data['datetime'] = date = 0
1335            #data['student_id'] = student_id = pay_transaction['Payer ID']
1336            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1337            #data['response_code'] = response_code = '00'
1338            #data['amount'] = amount = pay_transaction['Amount']
1339
1340            # format of the third file sent by Kehinde
1341            data['datetime'] = date = 0
1342            data['student_id'] = student_id = pay_transaction['customer_id']
1343            data['order_id'] = order_id = pay_transaction['merchant_reference']
1344            data['response_code'] = response_code = '00'
1345            data['amount'] = amount = pay_transaction['Amount']
1346
1347            dup = False
1348            if response_code == "12":
1349                continue
1350            try:
1351                opt.addRecord(**data)
1352            except ValueError:
1353                dup = True
1354            #from pdb import set_trace;set_trace()
1355            if dup:
1356                if response_code == "00":
1357                    try:
1358                        opt.modifyRecord(**data)
1359                    except:
1360                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1361                        continue
1362                else:
1363                    pay_transaction['Error'] = "Duplicate order_id"
1364                    no_import.append( format_error % pay_transaction)
1365                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1366                    continue
1367            tr_count += 1
1368            if tr_count > 1000:
1369                if len(no_import) > 0:
1370                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1371                             '\n'.join(no_import) + '\n')
1372                    no_import = []
1373                em = '%d transactions committed\n' % (tr_count)
1374                transaction.commit()
1375                regs = []
1376                logger.info(em)
1377                total += tr_count
1378                tr_count = 0
1379        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1380                                                '\n'.join(no_import))
1381        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1382    ###)
1383
1384    security.declareProtected(ModifyPortalContent,"importData")###(
1385    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1386        """load data from CSV values"""
1387        import transaction
1388        import random
1389        students_folder = self.portal_url.getPortalObject().campus.students
1390        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1391        pending_only = False
1392        pend_str = '--'
1393        elapse = time.time()
1394        #
1395        # preparations
1396        #
1397        if filename == pend_str:
1398            pending_only = True
1399        # mode = "create"
1400        # if edit:
1401        #     if filename.endswith('_toDelete'):
1402        #         mode = "delete"
1403        #     else:
1404        #         mode = "edit"
1405        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1406        importer = eval("%sImport" % importer_name)(self)
1407        logger = importer.logger
1408        if importer.init_errors:
1409            logger.info(importer.init_errors)
1410            return importer.init_errors
1411        member = importer.member
1412        current = importer.current
1413        import_date = importer.import_date
1414        imported_by = importer.imported_by
1415        #
1416        # not_imported
1417        #
1418        info = importer.info
1419        data_keys = importer.data_keys
1420        csv_keys = importer.csv_keys
1421        #csv_keys.extend(info.keys())
1422        headline_mapping = dict((k,k) for k in csv_keys)
1423        #
1424        # pending
1425        #
1426        pending_path = importer.pending_path
1427        pending_tmp = importer.pending_tmp
1428        pending_backup = importer.pending_backup
1429        pending_fn = importer.pending_fn
1430        imported_path = importer.imported_path
1431        imported_fn = importer.imported_fn
1432        commit_after = importer.commit_after
1433        pending = []
1434        pending_digests = []
1435        #total_added_to_pending = 0
1436        if not pending_only:
1437            pending,pending_digests = importer.makeIdLists()
1438            pending_at_start = len(pending)
1439        datafile = open(pending_tmp,"w")
1440        pending_csv_writer = csv.DictWriter(datafile,
1441                                                    csv_keys,
1442                                                    extrasaction='ignore')
1443        pending_csv_writer.writerow(headline_mapping)
1444        datafile.close()
1445        #
1446        # imported
1447        #
1448        if not os.path.exists(imported_path):
1449            datafile = open(imported_path,"w")
1450            imported_csv_writer = csv.DictWriter(datafile,
1451                                                 csv_keys,
1452                                                 extrasaction='ignore')
1453            imported_csv_writer.writerow(headline_mapping)
1454            datafile.close()
1455        start = True
1456        tr_count = 0
1457        total = 0
1458        total_added_to_imported = 0
1459        total_pending = 0
1460        import_source_done = ""
1461        if pending_only:
1462            import_source_path = pending_path
1463        else:
1464            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1465            import_source_done = "%s/import/%s.done" % (i_home,filename)
1466        if not os.path.exists(import_source_path):
1467            fn = os.path.split(import_source_path)[1]
1468            em = 'no import file %(fn)s' % vars()
1469            return em
1470        import_source_fn = os.path.split(import_source_path)[1]
1471        if not pending_only:
1472            info['imported_from'] = import_source_fn
1473        headline = csv.reader(open(import_source_path,"rb")).next()
1474        if "import_mode" not in headline:
1475            msg = 'import_mode must be in heading'
1476            return msg
1477        invalid_keys = importer.checkHeadline(headline)
1478        if invalid_keys:
1479            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1480
1481        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1482                                                         or k.strip() in info.keys())]
1483        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1484        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1485        # if diff2schema and diff2schema != set(['id',]):
1486        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1487        #     return msg
1488        #
1489        # start importing
1490        #
1491        try:
1492            reader = csv.DictReader(open(import_source_path,"rb"))
1493        except:
1494            msg = 'Error reading %s.csv' % filename
1495            logger.error(msg)
1496            return msg
1497        items = [item for item in reader]
1498        total_to_import = len(items)
1499        tti_float = float(total_to_import)
1500        if pending_only:
1501            pending_at_start = total_to_import
1502        count = 0
1503        imported = []
1504        old_commit_count = -1
1505        error_count = imported_count = 0
1506        already_in = 0
1507        for record in items:
1508            item = {}
1509            empty_value_keys = []
1510            for k,v in record.items():
1511                if v:
1512                    if v == EMPTY:
1513                        empty_value_keys += k,
1514                        v = ''
1515                    item[k.strip()] = v.strip()
1516            count += 1
1517            if start:
1518                start = False
1519                adapters = [MappingStorageAdapter(importer.schema, item)]
1520                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1521            dm = DataModel(item, adapters,context=self)
1522            ds = DataStructure(data=item,datamodel=dm)
1523            error_string = ""
1524            total += 1
1525            import_mode = item.get('import_mode','')
1526            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1527            if import_method is None:
1528                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1529            elif (import_mode in importer.required_modes and
1530                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1531                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1532                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1533            else:
1534                for k in import_keys:
1535                    if k not in item.keys() or k not in importer.validators.keys():
1536                        continue
1537                    if not importer.validators[k](ds,mode=import_mode):
1538                        error_string += ' ++ '
1539                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1540                                                                           ds.getErrorMapping(k)))
1541            if error_string:
1542                error = error_string
1543                id = ''
1544                mapping = item
1545            else:
1546                temp_item = item.copy()
1547                temp_item.update(dm)
1548                results = import_method(temp_item)
1549                id = results[0]
1550                error = results[1]
1551                mapping = results[2]
1552            #
1553            # restore some values
1554            #
1555            for k in empty_value_keys:
1556                mapping[k] = EMPTY
1557            if mapping.has_key('sex'):
1558                #import pdb;pdb.set_trace()
1559                if mapping['sex'] == True:
1560                    mapping['sex'] = 'F'
1561                elif mapping['sex'] == False:
1562                    mapping['sex'] = 'M'
1563            if pending_only:
1564                info['imported_from'] = item['imported_from']
1565            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1566            info['error'] = error
1567            mapping.update(info)
1568            log_list = []
1569            if error:
1570                error_count += 1
1571                digest = makeDigest(mapping,data_keys)
1572                if digest not in pending_digests:
1573                    pending_digests += digest,
1574                    pending.append(mapping)
1575                    if not pending_only:
1576                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1577                else:
1578                    already_in += 1
1579                    pass
1580            else:
1581                imported_count += 1
1582                imported += mapping,
1583                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1584            if log_list:
1585                time_till_now = time.time() - elapse
1586                percent_finished = (error_count + imported_count)/tti_float*100
1587                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1588                logger.info(' '.join(log_list))
1589            finished = count > total_to_import - 1
1590            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1591            if must_commit:
1592                old_commit_count = imported_count
1593
1594            if must_commit or finished:
1595                if len(imported):
1596                    transaction.commit()
1597                    datafile = open(imported_path,"a")
1598                    writer = csv.DictWriter(datafile,
1599                                            csv_keys,
1600                                            extrasaction='ignore')
1601                    writer.writerows(imported)
1602                    datafile.close()
1603                    total_added_to_imported += len(imported)
1604                    imported = []
1605                if len(pending) > 0:
1606                    datafile = open(pending_tmp,"a")
1607                    writer = csv.DictWriter(datafile,
1608                                            csv_keys,
1609                                            extrasaction='ignore')
1610                    writer.writerows(pending)
1611                    datafile.close()
1612                    total_pending += len(pending)
1613                    #total_added_to_pending += len(pending)
1614                    pending = []
1615                if not finished:
1616                    msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars()
1617                    logger.info(msg)
1618        elapse = time.time() - elapse
1619        if os.path.exists(pending_path):
1620            copy2(pending_path,pending_backup)
1621        copy2(pending_tmp,pending_path)
1622        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1623        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1624        if pending_only:
1625            removed_pending = pending_at_start - total_pending
1626            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1627        else:
1628            added_pending = total_pending - pending_at_start
1629            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1630        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1631        logger.info(msg)
1632        if import_source_done:
1633            copy(import_source_path,import_source_done)
1634            os.remove(import_source_path)
1635            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1636            if upload is not None:
1637                upload_doc = upload.getContent()
1638                mapping = {}
1639                #mapping['import_date'] = DateTime.DateTime()
1640                mapping['import_date'] = import_date
1641                mapping['imported_by'] = imported_by
1642                mapping['import_message'] = msg
1643                upload_doc.edit(mapping = mapping)
1644        os.remove(pending_tmp)
1645        return msg
1646    ###)
1647
1648    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1649    def moveImagesToFS(self,student_id="O738726"):
1650        "move the images to the filesystem"
1651        images_dir = getImagesDir(student_id)
1652        #images_dir = os.path.join("%s" % images_base,student_id)
1653        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1654        stool = getToolByName(self, 'portal_schemas')
1655        schemas = ['student_application',
1656                   'student_clearance',
1657                   ]
1658        created = False
1659        for schema_id in schemas:
1660            schema = stool._getOb(schema_id)
1661            object = getattr(student_folder,schema_id[len('student_'):],None)
1662            if object is None:
1663                continue
1664            doc = object.getContent()
1665            for key in schema.keys():
1666                if schema[key].meta_type != "CPS Image Field":
1667                    continue
1668                #import pdb;pdb.set_trace()
1669                image = getattr(doc,key,None)
1670                if not image or not hasattr(image,"data"):
1671                    continue
1672                if not created:
1673                    if not os.path.exists(images_dir):
1674                        os.mkdir(images_dir)
1675                    created = True
1676                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1677                open(filename,"wb").write(str(image.data))
1678    ###)
1679
1680    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1681    def movePassportToFS(self,student_id="O738726"):
1682        "move the passports to the filesystem"
1683        images_dir = os.path.join("%s" % i_home,'passports')
1684        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1685        stool = getToolByName(self, 'portal_schemas')
1686        schemas = ['student_application',
1687                   #'student_clearance',
1688                   ]
1689        created = False
1690        for schema_id in schemas:
1691            schema = stool._getOb(schema_id)
1692            object = getattr(student_folder,schema_id[len('student_'):],None)
1693            if object is None:
1694                continue
1695            doc = object.getContent()
1696            for key in schema.keys():
1697                if schema[key].meta_type != "CPS Image Field":
1698                    continue
1699                #import pdb;pdb.set_trace()
1700                image = getattr(doc,key)
1701                if not hasattr(image,"data"):
1702                    continue
1703                if not created:
1704                    if not os.path.exists(images_dir):
1705                        os.mkdir(images_dir)
1706                    created = True
1707                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1708                open(filename,"wb").write(str(image.data))
1709    ###)
1710
1711InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.