source: WAeUP_SRP/base/WAeUPTool.py @ 3339

Last change on this file since 3339 was 3339, checked in by Henrik Bettermann, 17 years ago

resolve ticket #507

Now we need a fix which replaces current_mode of all students by the certificate study_mode value.
  • Property svn:keywords set to Id
File size: 69.5 KB
Line 
1# -*- mode: python; mode: fold; -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 3339 2008-03-16 08:38:39Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29from zExceptions import BadRequest
30
31from Products.CMFCore.utils import getToolByName
32from Products.CPSSchemas.DataStructure import DataStructure
33from Products.CPSSchemas.DataModel import DataModel
34from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
35from Products.CMFCore.ActionProviderBase import ActionProviderBase
36from Products.CMFCore.permissions import View
37from Products.ZCatalog.ZCatalog import ZCatalog
38from Products.CMFCore.permissions import ModifyPortalContent
39from Products.CMFCore.permissions import ManagePortal
40from Products.CMFCore.utils import UniqueObject
41from Products.CMFCore.URLTool import URLTool
42from Products.CMFCore.utils import getToolByName
43from Students import makeCertificateCode
44from Globals import package_home,INSTANCE_HOME
45from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport
46from WAeUPImport import CourseImport,CourseResultImport
47from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport
48from utils import makeDigest
49import DateTime,time
50import logging
51import transaction
52import csv,re,os,sys
53import md5
54from shutil import copy2,copy
55from Products.AdvancedQuery import Eq, Between, Le,In
56
57p_home = package_home(globals())
58i_home = INSTANCE_HOME
59images_base = os.path.join(i_home,"images")
60EMPTY = 'XXX'
61
62def getImagesDir(student_id):
63    return os.path.join("%s" % images_base,student_id[0],student_id)
64
65def getObject(object,name):
66    if object.hasObject(name):
67        return getattr(object,name)
68    return None
69
70class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
71    """WAeUP tool"""
72
73    id = 'waeup_tool'
74    meta_type = 'WAeUP Tool'
75    _actions = ()
76    security = ClassSecurityInfo()
77    security.declareObjectProtected(View)
78    manage_options = ( ActionProviderBase.manage_options
79                     + SimpleItem.manage_options
80                     )
81
82    security.declareProtected(View,'re_split') ###(
83    def re_split(self,split_string,string):
84        return re.split(split_string,string)
85    ###)
86
87    security.declareProtected(View,'difference') ###(
88    def difference(self,l1,l2):
89        return set(l1).difference(set(l2))
90    ###)
91
92    def rwrite(self,s): ###(
93        response = self.REQUEST.RESPONSE
94        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
95        response.write("%s<br />\r\n" % s)
96    ###)
97
98    def addtodict(self,d,key,item): ###(
99        d[key].append(item)
100        return d[key]
101    ###)
102
103    def sleep(self,secs): ###(
104        "sleep"
105        import time
106        time.sleep(secs)
107        return
108
109###)
110
111    security.declareProtected(View,'updateRoleMappingsFor') ###(
112    def updateRoleMappingsFor(self,wf_definition,ob):
113        "do so for public"
114        wf_def = getattr(self.portal_workflow,wf_definition)
115        wf_def.updateRoleMappingsFor(ob)
116    ###)
117
118    security.declareProtected(View,'getStatesLgas') ###(
119    def getStatesLgas(self):
120        """return lga info"""
121        voc = getattr(self.portal_vocabularies,'local_gov_areas')
122        states = []
123        lgas  = []
124        d = {}
125        wd = {}
126        for k,v in voc.items():
127            parts = v.split(' / ')
128            if len(parts) == 1:
129                state = parts[0].lower()
130                lga = ""
131            elif len(parts) == 2:
132                state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower()))
133                lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower()))
134            else:
135                continue
136            if state not in states:
137                states.append(state)
138            if lga not in lgas:
139                lgas.append(lga)
140            words = re.split('[^a-zA-Z0-9/]',k)
141            words.sort()
142            wd[k] = words
143            d[k] = v
144        mapping = {}
145        mapping['word_dict'] = wd
146        mapping['lga_dict'] = d
147        mapping['states'] = states
148        mapping['lgas'] = lgas
149        return mapping
150    ###)
151
152    security.declareProtected(View,'findLga') ###(
153    def findLga(self,words,words_dict):
154        words = re.split('[^a-zA-Z0-9/]',words)
155        lga_words = []
156        for word in words:
157            if word:
158                lga_words += word.strip().lower(),
159        lga_words.sort()
160        state_lga = ''
161        while not state_lga:
162            for k,l in words_dict.items():
163                if lga_words == l:
164                    state_lga = k
165                    break
166            break
167        return state_lga
168    ###)
169    security.declareProtected(View,'getAccessInfo') ###(
170    def getAccessInfo(self,context):
171        "return a dict with access_info"
172        logger = logging.getLogger('WAeUPTool.getAccessInfo')
173        mtool = self.portal_membership
174        member = mtool.getAuthenticatedMember()
175        member_id = str(member)
176        info = {}
177        is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser()
178        is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57
179        is_staff = info['is_staff'] = not is_anonymous and not is_student
180        roles = member.getRolesInContext(context)
181        is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or
182                                                                            "SectionManager" in roles or
183                                                                            "Manager" in roles)
184        is_allowed = info['is_allowed'] = not is_anonymous
185        requested_id = context.getStudentId()
186        student_id  = None
187        if is_allowed:
188            if not is_student and requested_id:
189                student_id  = requested_id
190            elif is_anonymous or member_id != requested_id:
191                logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id))
192            else:
193                student_id = member_id
194        info['student_id'] = student_id
195        return info
196    ###)
197
198    security.declareProtected(ModifyPortalContent,'openLog') ###(
199    def openLog(self,name):
200        """open a log file"""
201        version = 1
202        path = "%s/log/%s_%d.log" % (i_home,name,version)
203        while os.path.exists(path):
204            version += 1
205            path = "%s/log/%s_%d.log" % (i_home,name,version)
206        log = open(path,"w")
207        return log
208    ###)
209
210    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
211    def bypassQueueCatalog(self,enable=True):
212        """bypass the QueueCatalog by setting all indexes to process imediate,
213        if enable is True (default) the old settings are restored
214        """
215
216    ###)
217
218    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
219    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
220        """measure Object access Time"""
221        import random
222        if hasattr(self,'portal_catalog_real'):
223            aq_portal = self.portal_catalog_real.evalAdvancedQuery
224        else:
225            aq_portal = self.portal_catalog.evalAdvancedQuery
226        nr_pts = int(nr_pts)
227        probe = int(probe)
228        intervall = probe/10
229        objects = ("application","clearance","personal")
230        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
231        #i = random.randrange(num_objects)
232        count = 0
233        found = 0
234        not_found = 0
235        t_found = 0
236        t_not_found = 0
237        time_found = time_not_found = 0.0
238        t_time_found = t_time_not_found = 0.0
239        accessed = []
240        t_min = 1000
241        t_max = 0
242        #import pdb;pdb.set_trace()
243        students = self.portal_catalog(portal_type="Student")
244        num_students = len(students)
245        if method == "d":
246            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
247            res = aq_portal(query)
248            brains = {}
249            for r in res:
250                sid = r.relative_path.split('/')[-2]
251                if brains.has_key(sid):
252                    brains[sid][r.portal_type] = r
253                else:
254                    brains[sid] = {r.portal_type : r}
255            brains_list = brains.keys()
256            num_objects = len(brains_list)
257        else:
258            num_objects = num_students
259        print "="*40
260        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
261                                                                        probe,
262                                                                        nr_pts,
263                                                                        num_objects)
264        print "nr found/not time found/not min/max"
265        elapse = time.time()
266        i_elapse = time.time()
267        c_elapse = time.clock()
268        for c in range(1,probe + 1):
269            i = random.randrange(num_objects)
270            if method in ('a','b','c'):
271                student_brain = students[i]
272            elif method == "d":
273                #import pdb;pdb.set_trace()
274                student_brain = brains[brains_list[i]]
275            if method == "c":
276                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
277                res = aq_portal(query)
278                this_portal_types = [r.portal_type for r in res]
279            for i in range(nr_pts):
280                oid = objects[i]
281                if method == "a":
282                    try:
283                        student_path = student_brain.getPath()
284                        path = "%s/%s" % (student_path,oid)
285                        doc = self.unrestrictedTraverse(path).getContent()
286                        found += 1
287                        i_time = time.time() - i_elapse
288                        time_found += i_time
289                    except:
290                        not_found += 1
291                        i_time = time.time() - i_elapse
292                        time_not_found += i_time
293                        pass
294                elif method == "b":
295                    try:
296                        student_object = student_brain.getObject()
297                        doc = getattr(student_object,oid).getContent()
298                        found += 1
299                        i_time = time.time() - i_elapse
300                        time_found += i_time
301                    except:
302                        i_time = time.time() - i_elapse
303                        time_not_found += i_time
304                        not_found += 1
305                        pass
306                elif method == "c":
307                    if portal_types[i] in this_portal_types:
308                        found += 1
309                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
310                        i_time = time.time() - i_elapse
311                        time_found += i_time
312                    else:
313                        not_found += 1
314                        i_time = time.time() - i_elapse
315                        time_not_found += i_time
316                elif method == "d":
317                    if student_brain.has_key(portal_types[i]):
318                        found += 1
319                        doc = student_brain[portal_types[i]].getObject().getContent()
320                        i_time = time.time() - i_elapse
321                        time_found += i_time
322                    else:
323                        not_found += 1
324                        i_time = time.time() - i_elapse
325                        time_not_found += i_time
326                i_elapse = time.time()
327            if c and (c % intervall == 0):
328                #i_time = time.time() - i_elapse
329                t_per = 0.0
330                if found:
331                    t_per = time_found/found
332                if t_per > t_max:
333                    t_max = t_per
334                if t_per > 0.0 and t_per < t_min:
335                    t_min = t_per
336                itf = 0.0
337                if found:
338                    itf = time_found/found
339                itnf = 0.0
340                if not_found :
341                    itnf = time_not_found / not_found
342                interval_time = time_found + time_not_found
343                s = "%(c)d: %(found)d/%(not_found)d " % vars()
344                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
345                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
346                print s
347                t_found += found
348                t_not_found += not_found
349                t_time_found += time_found
350                t_time_not_found += time_not_found
351                time_found = time_not_found = 0.0
352                found = not_found = 0
353        # t_found += found
354        # t_not_found += not_found
355        elapse = time.time() - elapse
356        itf = 0.0
357        if t_found:
358            itf = t_time_found/t_found
359        itnf = 0.0
360        if t_not_found:
361            itnf = t_time_not_found / t_not_found
362        #c_elapse = time.clock() - c_elapse
363        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
364        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
365        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
366        print "-"*40
367        print s
368        rel_found = float(t_found)/probe
369        rel_not_found = float(t_not_found)/probe
370        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
371        print estimated_total_time
372    ###)
373
374    security.declareProtected(ModifyPortalContent,'writeLog') ###(
375    def writeLog(self,logfile,s):
376        """write to the log file"""
377        logfile.write(s)
378
379###)
380
381    def generateStudentId(self,letter): ###(
382        import random
383        r = random
384        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
385        if letter == '?':
386            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
387        sid = "%c%d" % (letter,r.randint(99999,1000000))
388        students = self.portal_url.getPortalObject().campus.students
389##        while hasattr(students, sid):
390##            sid = "%c%d" % (letter,r.randint(99999,1000000))
391        while self.students_catalog(id = sid):
392            sid = "%c%d" % (letter,r.randint(99999,1000000))
393        return sid
394    ###)
395
396    def generatePassword(self,s=None): ###(
397        import random
398        r = random
399        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
400        if s is None:
401            s = 'abcdefghklmnpqrstuvwxy23456789'
402        pw = ''
403        while len(pw) < 6:
404            pw += r.choice(s)
405        return pw
406    ###)
407
408    security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###(
409    def dumpSchoolfeePayments(self):
410        "dump paid schoolfees"
411        mtool = self.portal_membership
412        member = mtool.getAuthenticatedMember()
413        logger = logging.getLogger('WAeUPTool.dumpSchoolfees')
414        aq_student = self.students_catalog.evalAdvancedQuery
415        query = In('review_state',('schoolfee_paid',
416                                   'courses_registered',
417                                   'courses_validated',
418                                   ))
419        res = aq_student(query)
420        #import pdb;pdb.set_trace()
421        l = []
422        logger.info("start for %d" % len(res))
423        count = 1
424        log_after = 100
425        for student in res:
426            if not count % log_after:
427                logger.info("processed %d total %d" % (log_after,count))
428            count += 1
429            fee_dict =self.getSchoolFee(student)
430            fulltime = student.mode.endswith('_ft')
431            d = {}
432            d['student_id'] = student.id
433            d['name'] = student.name
434            d['amount'] = fee_dict.get(new_returning)
435            l += d,
436        csv_name = self.dumpListToCSV(l,'payments')
437        logger.info('%s dumped payments to %s' % (member,export_file))
438    ###)
439
440    security.declarePublic('dumpListToCSV') ###(
441    def dumpListToCSV(self,l,filename,fields=None):
442        """dump a list of dicts to a CSV file"""
443        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
444        export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,)
445        if fields is None:
446            fields = l[0].keys()
447        headline = ','.join(fields)
448        out = open(export_file,"wb")
449        out.write(headline +'\n')
450        out.close()
451        out = open(export_file,"a")
452        csv_writer = csv.DictWriter(out,fields,)
453        csv_writer.writerows(l)
454        return export_file
455    ###)
456
457    security.declareProtected(ManagePortal, 'listMembers') ###(
458    def listMembers(self):
459        "list all members"
460        mtool = self.portal_membership
461        member = mtool.getAuthenticatedMember()
462        logger = logging.getLogger('WAeUPTool.listMembers')
463        if str(member) not in ('admin','joachim'):
464            logger.info('%s tried to list members' % (member))
465            return None
466        members = self.portal_directories.members
467        all = members.listEntryIdsAndTitles()
468        l = []
469        for user_id,name in all:
470            d = {}
471            d['user_id'] = user_id
472            d['name'] = name
473            d['pw'] = getattr(getattr(members,user_id),'password')
474            d['email'] = getattr(getattr(members,user_id),'email')
475            d['groups'] = " ".join(getattr(getattr(members,user_id),'groups'))
476            d['roles'] = " ".join(getattr(getattr(members,user_id),'roles'))
477            l += d,
478        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
479        export_file = "%s/export/member_list_%s.csv" % (i_home,current,)
480        logger.info('%s dumped member list to %s' % (member,export_file))
481        fields = l[0].keys()
482        headline = ','.join(fields)
483        out = open(export_file,"wb")
484        out.write(headline +'\n')
485        out.close()
486        out = open(export_file,"a")
487        csv_writer = csv.DictWriter(out,fields,)
488        csv_writer.writerows(l)
489    ###)
490
491    security.declareProtected(ManagePortal, 'listStudents') ###(
492    def listStudents(self):
493        "list all students"
494        mtool = self.portal_membership
495        member = mtool.getAuthenticatedMember()
496        logger = logging.getLogger('WAeUPTool.listStudents')
497        if str(member) not in ('admin','joachim'):
498            logger.info('%s tried to list students' % (member))
499            return None
500        students = self.portal_directories.students
501        all = students.listEntryIdsAndTitles()
502        l = []
503        for user_id,name in all:
504            d = {}
505            d['user_id'] = user_id
506            d['name'] = name
507            d['pw'] = getattr(getattr(students,user_id),'password')
508            d['email'] = getattr(getattr(students,user_id),'email')
509            d['groups'] = " ".join(getattr(getattr(students,user_id),'groups'))
510            d['roles'] = " ".join(getattr(getattr(students,user_id),'roles'))
511            l += d,
512        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
513        export_file = "%s/export/student_list_%s.csv" % (i_home,current,)
514        logger.info('%s dumped student list to %s' % (member,export_file))
515        fields = l[0].keys()
516        headline = ','.join(fields)
517        out = open(export_file,"wb")
518        out.write(headline +'\n')
519        out.close()
520        out = open(export_file,"a")
521        csv_writer = csv.DictWriter(out,fields,)
522        csv_writer.writerows(l)
523    ###)
524
525    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
526    def removeDeletedDocIds(self, max=1000):
527        """
528        remove deleted docids from repository commit after max
529        """
530        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
531        repository = getToolByName(self, 'portal_repository')
532        pxtool = getToolByName(self, 'portal_proxies')
533        logger.info('start')
534        pxtool_infos = pxtool.getRevisionsUsed()
535        logger.info('found  %d used revisions ' % (len(pxtool_infos)))
536
537        nb_revs = 0
538        docids_d = {} # all docids
539        unused_docids_d = {} # all docids that are unused
540        ids_unused_revs_docids = [] # ids for revs of unused docids
541        unused_ids = [] # ids for unused revs
542        total = 0
543        idlist = repository.objectIds()
544        to_delete = 0
545        found = False
546        for id in idlist:
547            docid, rev = repository._splitId(id)
548            if docid is None:
549                logger.info("invalid doc_id %s" % docid)
550                continue
551            nb_revs += 1
552            if not pxtool_infos.has_key(docid):
553                found = True
554                to_delete += 1
555                unused_ids.append(id)
556            elif not pxtool_infos[docid].has_key(rev):
557                found = True
558                to_delete += 1
559                unused_ids.append(id)
560            if found and not to_delete % max:
561                found = False
562                #import pdb;pdb.set_trace()
563                repository.manage_delObjects(unused_ids)
564                transaction.commit()
565                logger.info('removed %d total %d unused docids ' % (max,to_delete))
566        else:
567            if unused_ids:
568                repository.manage_delObjects(unused_ids)
569                transaction.commit()
570        logger.info('finished removing %d unused docids ' % (to_delete))
571
572
573###)
574
575    security.declareProtected(View,'getCredential') ###(
576    def getCredential(self,student_id):
577        student_entry = getattr(self.portal_directories.students,student_id,None)
578        if not self.isStaff():
579            mtool = self.portal_membership
580            member = mtool.getAuthenticatedMember()
581            logger = logging.getLogger('WAeUPTool.getCredential')
582            logger.info('%s tried to access password of %s' % (member,student_id))
583            return None
584        if student_entry is None:
585            return None
586        return getattr(student_entry,"password","not set")
587    ###)
588
589    security.declarePublic('checkPassword') ###(
590    def checkPassword(self,student_id,password):
591        student_entry = getattr(self.portal_directories.students,student_id,None)
592        if student_entry is None:
593            return False
594        return getattr(student_entry,"password","not set") == password
595    ###)
596
597    security.declarePublic('checkGenericPassword') ###(
598    def checkGenericPassword(self,member_id):
599        member_entry = getattr(self.portal_directories.members,member_id,None)
600        if member_entry is None:
601            return False
602        ltool = getToolByName(self, 'portal_layouts')
603        unsecure_words = ltool._getOb('members')['w__password'].check_words
604        password = getattr(member_entry,"password","not set")
605        is_unsecure = password in unsecure_words
606        if is_unsecure:
607            logger = logging.getLogger('WAeUPTool.checkGenericPassword')
608            logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password))
609        return is_unsecure
610    ###)
611
612    security.declareProtected(ModifyPortalContent,'editPassword') ###(
613    def editPassword(self,student_id,password):
614        "edit a student password"
615        student_entry = getattr(self.portal_directories.students,student_id,None)
616        if student_entry is None:
617            return
618        setattr(student_entry,'password',password)
619    ###)
620
621    security.declareProtected(ModifyPortalContent,'doCommit') ###(
622    def doCommit(self,logger=None):
623        "commit some transactions"
624        transaction.commit()
625    ###)
626
627    security.declarePublic('loadStudentFoto') ###(
628    def loadStudentFoto(self,student,filename,folder):
629        "return a student passport picture"
630        #import pdb;pdb.set_trace()
631        picture ="%s/import/%s/%s" % (i_home,folder,filename)
632        student_id = student.getId()
633        images_dir = getImagesDir(student_id)
634        if not os.path.exists(images_dir):
635            os.mkdir(images_dir)
636        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
637        for extension in ('.jpg','.JPG'):
638            fullname = "%(picture)s%(extension)s" % vars()
639            if os.path.exists(fullname):
640                copy2(fullname,image_name)
641                return "successfully copied passport picture"
642        return "passport picture not found: %s.jpg or .JPG" % picture
643    ###)
644
645    def old____loadStudentFoto(self,student,filename,folder): ###(
646        "return a student passport picture"
647        app = student.application
648        app_doc = app.getContent()
649        #clear = student.clearance
650        #clear_doc = clear.getContent()
651        #matric_no = clear_doc.matric_no.upper()
652        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
653        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
654        #import pdb;pdb.set_trace()
655        if os.path.exists(picture1):
656            file = open(picture1)
657        elif os.path.exists(picture2):
658            file = open(picture2)
659        else:
660            return "passport picture not found %s" % picture1
661        reopened = False
662        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
663            self.portal_workflow.doActionFor(app,'open')
664            reopened = True
665        outfile = file.read()
666        app_doc.manage_addFile('passport',
667                               file=outfile,
668                               title="%s.jpg" % filename)
669        if reopened:
670            self.portal_workflow.doActionFor(app,'close')
671        return "successfully loaded passport picture"
672    ###)
673
674    security.declareProtected(ModifyPortalContent,'createOne') ###(
675    def createOne(self,students_folder,student_brain,letter,commit=False):
676        sid = self.waeup_tool.generateStudentId(letter)
677        students_folder.invokeFactory('Student', sid)
678        student = getattr(students_folder,sid)
679        self.portal_workflow.doActionFor(student,'return')
680        student.manage_setLocalRoles(sid, ['Owner',])
681        matric_no = student_brain.matric_no
682        jamb_reg_no = student_brain.Entryregno
683        self.students_catalog.addRecord(id = sid,
684                                           matric_no = matric_no,
685                                           jamb_reg_no = jamb_reg_no,
686                                           sex = student_brain.Sex == "F",
687                                           name = "%s %s %s" % (student_brain.Firstname,
688                                                                student_brain.Middlename,
689                                                                student_brain.Lastname)
690                                        )
691        if commit:
692            transaction.commit()
693        return sid,jamb_reg_no
694    ###)
695
696    security.declareProtected(ModifyPortalContent,'addStudent') ###(
697    def addStudent(self,dict):
698        students_folder = self.portal_url.getPortalObject().campus.students
699        sid = self.waeup_tool.generateStudentId('?')
700        students_folder.invokeFactory('Student', sid)
701        student_obj = getattr(students_folder,sid)
702        f2t = self.field2types_student
703        #from pdb import set_trace; set_trace()
704        d = {}
705        #d['jamb_sex']  = 'M'
706        #if dict.get('sex'):
707        #    d['jamb_sex']  = 'F'
708
709        entry_session = dict.get('entry_session')
710        if entry_session == self.getSessionId()[0]:
711            wfaction = 'admit'
712            wft = 'wf_transition_admit'
713            password = None
714        else:
715            wfaction = 'return'
716            wft = 'wf_transition_return'
717            password = self.generatePassword()
718            self.makeStudentMember(sid,password)
719
720        for pt in f2t.keys():
721            student_obj.invokeFactory(pt,f2t[pt]['id'])
722            sub_obj = getattr(student_obj,f2t[pt]['id'])
723            sub_doc = sub_obj.getContent()
724            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
725            d['Title'] = f2t[pt]['title']
726            for field in f2t[pt]['fields']:
727                d[field] = dict.get(field,'')
728            sub_doc.edit(mapping = d)
729            new_state = f2t[pt][wft]
730            if new_state != "remain":
731                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
732        self.portal_workflow.doActionFor(student_obj,wfaction)
733        student_obj.manage_setLocalRoles(sid, ['Owner',])
734        return sid,password
735    ###)
736
737    security.declarePublic('getCertificateBrain') ###(
738    def getCertificateBrain(self,cert_id):
739        "do it"
740        res = ZCatalog.searchResults(self.portal_catalog_real,
741                                {'portal_type':"Certificate",
742                                      'id': cert_id})
743        if res:
744            return res[0]
745        return None
746    ###)
747
748    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
749    def get_csv_filenames(self):
750        "do it"
751        files = [file for file in os.listdir("%s/import/" % (i_home))
752                 if file.endswith('.csv') and (file.find('imported') == -1 and
753                                               file.find('pending') == -1)]
754        return files
755    ###)
756
757    security.declarePublic('findStudentByMatricelNo') ###(
758    def findStudentByMatricelNo(self,matric_no):
759        "do it"
760        res = ZCatalog.searchResults(self.portal_catalog_real,
761                                {'portal_type':"StudentClearance",
762                                 'SearchableText': matric_no})
763        if res:
764            return res[0]
765        return None
766    ###)
767
768    security.declarePublic('makeStudentMember') ###(
769    def makeStudentMember(self,sid,password='uNsEt'):
770        """make the student a member"""
771        membership = self.portal_membership
772        membership.addMember(sid,
773                             password ,
774                             roles=('Member',
775                                     'Student',
776                                     ),
777                             domains='',
778                             properties = {'memberareaCreationFlag': False,
779                                           'homeless': True},)
780        member = membership.getMemberById(sid)
781        self.portal_registration.afterAdd(member, sid, password, None)
782        #self.manage_setLocalRoles(sid, ['Owner',])
783    ###)
784
785    security.declareProtected(View,'makeStudentData') ###(
786    def makeStudentData(self,student_id,email=None,phone_nr=None):
787        "create Datastructure for a returning Student"
788        #import pdb;pdb.set_trace()
789        logger = logging.getLogger('WAeUPTool.makeStudentData')
790        students_folder = self.portal_url.getPortalObject().campus.students
791        #res = self.students_catalog(id=student_id)
792        #if res:
793        #    st = res[0]
794        #res = self.returning_import(matric_no = st.matric_no)
795        res = self.returning_import(id = student_id)
796        if res:
797            student = res[0]
798        else:
799            logger.info('Id %s not found in returning_import' % student_id)
800            return
801        logger.info('%s creates data structure' % student_id)
802        s_results = self.results_import(matric_no = student.matric_no)
803        if s_results:
804            lnr = self.getLevelFromResultsCosCode(s_results)
805            level = "%d00" % lnr
806            verdict,eligible = self.getVerdict(s_results[0].Verdict)
807            #if eligible:
808            #    level = "%d00" % (lnr + 1)
809        else:
810            logger.info('matric_no %s not found in results_import' % student.matric_no)
811            level = ''
812            verdict = ''
813        #student should not be allowed to perform this transition
814        #wftool = self.portal_workflow
815        #wftool.doActionFor(student,'return')
816        certcode_org = student.Coursemajorcode
817        certcode = makeCertificateCode(certcode_org)
818        certificate_brain = self.getCertificateBrain(certcode)
819        if not certificate_brain:
820            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
821            logger.info(em)
822        matric_no = student.matric_no
823        sid = student_id
824        student_obj = getattr(students_folder,sid)
825        if not getattr(student_obj,'application'):
826            student_obj.invokeFactory('StudentApplication','application')
827        application = student_obj.application
828        self.portal_workflow.doActionFor(application,'open',dest_container=application)
829        da = {'Title': 'Application Data'}
830        student_obj.invokeFactory('StudentPersonal','personal')
831        da['jamb_reg_no'] = student.Entryregno
832        em = self.getEntryMode(student.Entryregno)
833        da['entry_mode'] = em
834        personal = student_obj.personal
835        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
836        dp = {'Title': 'Personal Data'}
837        student_obj.invokeFactory('StudentClearance','clearance')
838        clearance = student_obj.clearance
839        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
840        dc = {'Title': 'Clearance/Eligibility Record'}
841        dc['matric_no'] = matric_no
842        state = student.State
843        lga = student.LGA
844        if state and lga:
845            lga =  state + ' / ' + lga
846        else:
847            lga = "None"
848        da['jamb_lga'] = dc['lga'] = lga
849        da['app_email'] = dp['email'] = email
850        da['app_mobile'] = dp['phone'] = phone_nr
851        dp['firstname'] = student.Firstname
852        dp['middlename'] = student.Middlename
853        dp['lastname'] = student.Lastname
854        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
855        da['jamb_sex'] = student.Sex
856        dp['sex'] = student.Sex == 'F'
857        dp['perm_address'] = student.Permanent_Address
858        application.getContent().edit(mapping=da)
859        self.portal_workflow.doActionFor(application,'close',dest_container=application)
860        personal.getContent().edit(mapping=dp)
861        clearance.getContent().edit(mapping=dc)
862        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
863        #
864        # Study Course
865        #
866        student_obj.invokeFactory('StudentStudyCourse','study_course')
867        studycourse = student_obj.study_course
868        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
869        dsc = {}
870        dsc['study_course'] = certcode
871        dsc['current_level'] = level
872        dsc['current_verdict'] = verdict
873        dsc['current_mode'] = em
874        dsc['current_session'] = '05'
875        studycourse.getContent().edit(mapping=dsc)
876        #
877        # Level
878        #
879##        l = getattr(studycourse,level,None)
880##        if l is None:
881##            studycourse.invokeFactory('StudentStudyLevel', level)
882##            l = getattr(studycourse, level)
883##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
884##            l.getContent().edit(mapping={'Title': "Level %s" % level})
885###)
886
887    security.declareProtected(ModifyPortalContent,'admitOneStudent') ###(
888    def admitOneStudent(self,brain,entry_session,pin_password):
889        "create Datastructure for an admitted Student"
890        #import pdb;pdb.set_trace()
891        if not hasattr(self,"_v_certificates"):
892            self._v_certificates = self.getCertificatesDict()
893        students_folder = self.portal_url.getPortalObject().campus.students
894        logger = logging.getLogger('WAeUPTool.admitOneStudent')
895        if brain.screening_type in ('cest','sandwich',):
896            reg_no = "%s%s/07" % (brain.course1,brain.serial)
897        else:
898            reg_no = brain.reg_no
899        res = self.students_catalog(jamb_reg_no = reg_no)
900        if res:
901            logger.info('student with this reg_no already exists %s with id %s' % (reg_no,res[0].id))
902            return
903        if brain.status != "admitted":
904            logger.info('status of %s is %s' % (reg_no,brain.status))
905            return
906        pin_parts = brain.pin.split('-')
907        if pin_parts and len(pin_parts) != 3:
908            logger.info('invalid pin %s for %s' % (brain.pin,reg_no))
909            return
910        if brain.course_admitted not in self._v_certificates:
911            logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no))
912            return
913        if brain.sex not in (True,False):
914            logger.info('sex of %s not available' % (reg_no))
915            return
916        student_id = self.generateStudentId('?')
917        students_folder.invokeFactory('Student', student_id)
918        student_object = getattr(students_folder,student_id)
919        if pin_password:
920            password = pin_parts[2]
921            self.makeStudentMember(student_id,password = password)
922        student_object.manage_setLocalRoles(student_id, ['Owner',])
923        #logger.info("creating %s reg_no %s" % (student_id,reg_no))
924        #
925        # application
926        #
927        student_object.invokeFactory('StudentApplication','application')
928        application = student_object.application
929        #self.portal_workflow.doActionFor(application,'open',dest_container=application)
930        da = {'Title': 'Application Data'}
931        da['jamb_reg_no'] = reg_no
932
933        sex = 'M'
934        if brain.sex:
935            sex = 'F'
936        da['jamb_sex'] = sex
937        da['jamb_age'] = brain.jamb_age
938        #da['app_ac_pin'] = brain.pin
939        da['jamb_lga'] = brain.jamb_lga
940        da['jamb_state'] = brain.jamb_state
941        da['jamb_score'] = brain.aggregate
942        da['app_email'] = brain.email
943        da['app_mobile'] = brain.phone
944
945        da['entry_mode'] = self._v_certificates[brain.course_admitted][1]
946
947        #if brain.entry_mode:
948        #    da['entry_mode'] = brain.entry_mode
949        #elif brain.screening_type == 'pume':
950        #    da['entry_mode'] = 'ume_ft'
951        #elif brain.screening_type == 'pde':
952        #    da['entry_mode'] = 'de_ft'
953        #elif brain.screening_type == 'pce':
954        #    da['entry_mode'] = 'pce'
955        #elif brain.screening_type == 'prence':
956        #    da['entry_mode'] = 'prence'
957        #else:
958        #    da['entry_mode'] = ''
959
960
961        da['entry_session'] = entry_session
962        da['jamb_lastname'] = brain.lastname
963        da['jamb_middlename'] = brain.middlenames   # different field names!
964        da['jamb_firstname'] = brain.firstname
965        da['screening_application_date'] = brain.application_date
966        da['date_of_birth'] = brain.date_of_birth
967        da['jamb_first_cos'] = brain.course1
968        da['jamb_second_cos'] = brain.course2
969        da['course3'] = brain.course3
970        da['screening_type'] = brain.screening_type
971        da['screening_score'] = brain.screening_score
972        da['screening_date'] = brain.screening_date
973        da['hq_type'] = brain.hq_type
974        da['hq_grade'] = brain.hq_grade
975        da['aos'] = brain.aos
976
977        application.getContent().edit(mapping=da)
978        #self.portal_workflow.doActionFor(application,'close',dest_container=application)
979        #
980        # personal
981        #
982        student_object.invokeFactory('StudentPersonal','personal')
983        personal = student_object.personal
984        #self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
985        dp = {'Title': 'Personal Data'}
986        dp['sex'] = brain.sex
987        dp['email'] = brain.email
988        dp['phone'] = brain.phone
989        dp['lastname'] = brain.lastname
990        dp['middlename'] = brain.middlenames   # different field names!
991        dp['firstname'] = brain.firstname
992        personal.getContent().edit(mapping=dp)
993        #
994        # clearance
995        #
996        student_object.invokeFactory('StudentClearance','clearance')
997        clearance = student_object.clearance
998        #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
999        dc = {'Title': 'Clearance/Eligibility Record'}
1000        dc['lga'] = brain.lga
1001        dc['birthday'] = brain.date_of_birth
1002        clearance.getContent().edit(mapping=dc)
1003        #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
1004        #
1005        # study Course
1006        #
1007        student_object.invokeFactory('StudentStudyCourse','study_course')
1008        studycourse = student_object.study_course
1009        #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
1010        dsc = {}
1011        dsc['study_course'] = brain.course_admitted
1012        dsc['current_verdict'] = ''
1013        dsc['current_mode'] = da['entry_mode']
1014        if da['entry_mode'].startswith('de'):
1015            dsc['current_level'] = '200'
1016        elif da['entry_mode'].startswith('pre'):
1017            dsc['current_level'] = '000'
1018        else:
1019            dsc['current_level'] = '100'
1020        dsc['current_session'] = entry_session
1021        studycourse.getContent().edit(mapping=dsc)
1022        #
1023        # payments folder
1024        student_object.invokeFactory('PaymentsFolder','payments')
1025        payments = getattr(student_object,'payments')
1026        dpay = {}
1027        dpay['Title'] = 'Payments'
1028        payments.getContent().edit(mapping=dpay)
1029        self.portal_workflow.doActionFor(payments,'open')
1030        #
1031        # passport foto
1032        app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home,
1033                                                              brain.screening_type,
1034                                                              brain.reg_no)
1035        images_dir = getImagesDir(student_id)
1036        #images_dir = os.path.join("%s" % images_base,student_id)
1037        letter_dir,student_dir = os.path.split(images_dir)
1038        if not os.path.exists(letter_dir):
1039            os.mkdir(letter_dir)
1040        if not os.path.exists(images_dir):
1041            os.mkdir(images_dir)
1042        image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars())
1043        if os.path.exists(app_picture):
1044            copy2(app_picture,image_name)
1045        else:
1046            logger.info('passport of %s/%s not found: %s' % (student_id,
1047                                                             brain.reg_no,
1048                                                             app_picture))
1049
1050        return student_id
1051    ###)
1052
1053    security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###(
1054    def makeStudentLevel(self,student_id):
1055        "create the StudyLevel for a returning Student"
1056        #import pdb;pdb.set_trace()
1057        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
1058        students_folder = self.portal_url.getPortalObject().campus.students
1059        res = self.students_catalog(id=student_id)
1060        if res:
1061            st = res[0]
1062        course = st.course
1063        matric_no = st.matric_no
1064        level = st.level
1065        res = self.results_import(matric_no = matric_no)
1066        if res:
1067            results = res
1068        logger.info('%s creating Level %s' % (student_id,level))
1069        #
1070        # Level
1071        #
1072        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1073        studycourse = getattr(student_obj,"study_course",None)
1074        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
1075        l = getattr(studycourse,level,None)
1076        if l is None:
1077            studycourse.invokeFactory('StudentStudyLevel', level)
1078            l = getattr(studycourse, level)
1079            self.portal_workflow.doActionFor(l,'open',dest_container=l)
1080            l.getContent().edit(mapping={'Title': "Level %s" % level})
1081        ###)
1082
1083    security.declarePublic('getHallInfo') ###(
1084    def getHallInfo(self,bed):
1085        """return Hall Info"""
1086        info = {}
1087        hall,block,room,letter = bed.split('_')
1088        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
1089        if res and len(res) == 1:
1090            hall_brain = res[0]
1091            hall_doc = hall_brain.getObject().getContent()
1092        else:
1093            return info
1094        info['hall_title'] = hall_brain.Title
1095        info['maintenance_code'] = hall_doc.maintenance_code
1096        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
1097        batch_doc = None
1098        for brain in res:
1099            if brain.id.startswith(info['maintenance_code']):
1100                batch_doc = brain.getObject().getContent()
1101                break
1102        if batch_doc is None:
1103            info['maintenance_fee'] = ''
1104        else:
1105            info['maintenance_fee'] = batch_doc.cost
1106        return info
1107    ###)
1108
1109    security.declareProtected(ModifyPortalContent,'removePictureFolder') ###(
1110    def removePictureFolder(self,student_id):
1111        """remove picture_folder by renaming it"""
1112        path = 'images'
1113        picture_path = os.path.join(i_home,path,student_id)
1114        if not os.path.exists(picture_path):
1115            return False
1116        os.rename(picture_path,picture_path + "_removed")
1117        return True
1118    ###)
1119
1120    security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###(
1121    def restorePictureFolder(self,student_id):
1122        """restore picture_folder by renaming it"""
1123        path = 'images'
1124        picture_path = os.path.join(i_home,path,student_id)
1125        if not os.path.exists(picture_path + "_removed"):
1126            return False
1127        os.rename(picture_path + "_removed",picture_path)
1128        return True
1129    ###)
1130
1131    security.declarePublic('picturesExist') ###(
1132    def picturesExist(self, ids,student_id=None):
1133        """check if pictures exist in the filesystem"""
1134        if student_id is None:
1135            student_id = self.getStudentId()
1136        if student_id is None:
1137            return False
1138        picture_path = getImagesDir(student_id)
1139        #picture_path = os.path.join(images_base,student_id)
1140        if not os.path.exists(picture_path):
1141            return False
1142        pictures  = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1143        return set(ids).issubset(set(pictures))
1144    ###)
1145
1146    security.declarePublic('picturesList') ###(
1147    def picturesList(self):
1148        """check if pictures exist in the filesystem"""
1149        path = 'images'
1150        student_id = self.getStudentId()
1151        #picture_path = os.path.join(i_home,path,student_id)
1152        picture_path = getImagesDir(student_id)
1153        if not os.path.exists(picture_path):
1154            return []
1155        return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)]
1156    ###)
1157
1158    security.declarePublic('showFsPicture') ###(
1159    def showFsPicture(self,path):
1160        """return a picture from the filesystem"""
1161        #picture_path = os.path.join(i_home,path)
1162        picture_path = os.path.join(images_base,path)
1163        response = self.REQUEST.RESPONSE
1164        #import pdb;pdb.set_trace()
1165        registry = getToolByName(self, 'mimetypes_registry')
1166        mimetype = str(registry.lookupExtension(path.lower()) or
1167                    registry.lookupExtension('file.bin'))
1168        if os.path.exists(picture_path):
1169            response.setHeader('Content-type',mimetype)
1170            return open(picture_path).read()
1171        picture_path = os.path.join(i_home,'import',path)
1172        if os.path.exists(picture_path):
1173            return open(picture_path).read()
1174    ###)
1175
1176    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
1177    def deleteAllCourses(self,department="All"):
1178        ''' delete the courses'''
1179        pm = self.portal_membership
1180        member = pm.getAuthenticatedMember()
1181
1182        if str(member) not in ("henrik","joachim"):
1183            return "not possible"
1184        if department == "All":
1185            res = self.portal_catalog({'meta_type': 'Department'})
1186        if len(res) < 1:
1187            return "No Departments found"
1188
1189        deleted = []
1190        for dep in res:
1191            cf = dep.getObject().courses
1192            if cf:
1193                cf.manage_delObjects(ids=cf.objectIds())
1194                deleted.append("deleted Courses in %s" % dep.getId)
1195        return "\r".join(deleted)
1196    ###)
1197
1198    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
1199    def getLogfileLines(self,filename="event.log",numlines=20):
1200        """Get last NUMLINES lines of logfile FILENAME.
1201
1202        Return last lines' of a file in the instances logfile directory as
1203        a list. The number of returned lines equals `numlines' or less. If
1204        less than `numlines' lines are available, the whole file ist
1205        returned. If the file can not be opened or some other error
1206        occurs, empty list is returend.
1207        """
1208        result = []
1209        lines_hit = 0
1210
1211        # We only handle files in instances' log directory...
1212        logpath = os.path.join(i_home, "log")
1213        filename = str(os.path.abspath( os.path.join( logpath, filename )))
1214        if not filename.startswith( logpath ):
1215            # Attempt to access file outside log-dir...
1216            return []
1217
1218        try:
1219            fd = file( filename, "rb" )
1220        except IOError:
1221            return []
1222        if not fd:
1223            return []
1224
1225        if os.linesep == None:
1226            linesep = '\n'
1227        else:
1228            linesep = os.linesep
1229
1230        # Try to find 'numlines' times a lineseparator, searching from end
1231        # and moving to the beginning of file...
1232        fd.seek( 0, 2) # Move to end of file...
1233        while lines_hit < numlines:
1234            if fd.read(1) == linesep[-1]: # This moves filedescriptor
1235                                          # one step forward...
1236                lines_hit += 1
1237            try:
1238                fd.seek( -2, 1) # Go two bytes back from current pos...
1239            except IOError:
1240                # We cannot go back two bytes. Maybe the file is too small...
1241                break
1242        fd.seek(2,1)
1243
1244        # Read all lines from current position...
1245        result = fd.readlines()
1246        # Remove line endings...
1247        result = [x.strip() for x in result]
1248        fd.close()
1249        return result
1250    ###)
1251
1252    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
1253    def getCallbacksFromLog(self,filename):
1254        """fix Online Payment Transactions from Z2.log entries"""
1255        import transaction
1256        import random
1257        from cgi import parse_qs
1258        from urlparse import urlparse
1259        #from pdb import set_trace
1260        wftool = self.portal_workflow
1261        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1262        students_folder = self.portal_url.getPortalObject().campus.students
1263        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
1264        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
1265        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
1266        data = re.compile(s)
1267        start = True
1268        tr_count = 1
1269        total = 0
1270        #name = 'pume_results'
1271        #name = 'epaymentsuccessful_z2log2'
1272        name = filename
1273        no_import = []
1274        imported = []
1275        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
1276        try:
1277            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
1278        except:
1279            logger.error('Error reading %s' % name)
1280            return
1281        tas = []
1282        for line in transactions:
1283            dict = {}
1284            items = data.search(line)
1285            dict['idict'] = idict = items.groupdict()
1286            #print idict
1287            #from pdb import set_trace;set_trace()
1288            urlparsed = urlparse(idict['get'][4:])
1289            #print urlparsed
1290            path = urlparsed[2].split('/')
1291            dict['student_id'] = student_id = path[8]
1292            dict['payment_id'] = payment_id = path[10]
1293            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
1294            tas.append(dict)
1295            tr_count += 1
1296        return tas
1297    ###)
1298
1299    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
1300    def importOnlinePaymentTransactions(self):
1301        """load Online Payment Transactions from CSV values"""
1302        import transaction
1303        import random
1304        #from pdb import set_trace
1305        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1306        opt = self.online_payments_import
1307        students_folder = self.portal_url.getPortalObject().campus.students
1308        start = True
1309        tr_count = 1
1310        total = 0
1311        #name = 'pume_results'
1312        name = 'OnlineTransactions'
1313        no_import = []
1314        imported = []
1315        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
1316        try:
1317            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
1318        except:
1319            logger.error('Error reading %s.csv' % name)
1320            return
1321        for pay_transaction in transactions:
1322            if start:
1323                start = False
1324                logger.info('Start loading from %s.csv' % name)
1325                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
1326                no_import.append('%s,"Error"' % s)
1327                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
1328                format_error = format + ',"%(Error)s"'
1329            data = {}
1330
1331            # format of the first file sent by Tayo
1332            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
1333            #data['student_id'] = student_id = pay_transaction['Payer ID']
1334            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1335            #data['response_code'] = response_code = pay_transaction['Resp Code']
1336            #data['amount'] = amount = pay_transaction['Amount']
1337
1338            # format of the second file sent by Tayo
1339            #data['datetime'] = date = 0
1340            #data['student_id'] = student_id = pay_transaction['Payer ID']
1341            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
1342            #data['response_code'] = response_code = '00'
1343            #data['amount'] = amount = pay_transaction['Amount']
1344
1345            # format of the third file sent by Kehinde
1346            data['datetime'] = date = 0
1347            data['student_id'] = student_id = pay_transaction['customer_id']
1348            data['order_id'] = order_id = pay_transaction['merchant_reference']
1349            data['response_code'] = response_code = '00'
1350            data['amount'] = amount = pay_transaction['Amount']
1351
1352            dup = False
1353            if response_code == "12":
1354                continue
1355            try:
1356                opt.addRecord(**data)
1357            except ValueError:
1358                dup = True
1359            #from pdb import set_trace;set_trace()
1360            if dup:
1361                if response_code == "00":
1362                    try:
1363                        opt.modifyRecord(**data)
1364                    except:
1365                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
1366                        continue
1367                else:
1368                    pay_transaction['Error'] = "Duplicate order_id"
1369                    no_import.append( format_error % pay_transaction)
1370                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
1371                    continue
1372            tr_count += 1
1373            if tr_count > 1000:
1374                if len(no_import) > 0:
1375                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1376                             '\n'.join(no_import) + '\n')
1377                    no_import = []
1378                em = '%d transactions committed\n' % (tr_count)
1379                transaction.commit()
1380                regs = []
1381                logger.info(em)
1382                total += tr_count
1383                tr_count = 0
1384        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
1385                                                '\n'.join(no_import))
1386        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
1387    ###)
1388
1389    security.declareProtected(ModifyPortalContent,"importData")###(
1390    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1391        """load data from CSV values"""
1392        import transaction
1393        import random
1394        students_folder = self.portal_url.getPortalObject().campus.students
1395        uploads_folder = self.portal_url.getPortalObject().campus.uploads
1396        pending_only = False
1397        pend_str = '--'
1398        elapse = time.time()
1399        #
1400        # preparations
1401        #
1402        if filename == pend_str:
1403            pending_only = True
1404        # mode = "create"
1405        # if edit:
1406        #     if filename.endswith('_toDelete'):
1407        #         mode = "delete"
1408        #     else:
1409        #         mode = "edit"
1410        importer_name = ''.join([part.capitalize() for part in name.split('_')])
1411        importer = eval("%sImport" % importer_name)(self)
1412        logger = importer.logger
1413        if importer.init_errors:
1414            logger.info(importer.init_errors)
1415            return importer.init_errors
1416        member = importer.member
1417        #current = importer.current
1418        import_date = importer.import_date
1419        #
1420        # not_imported
1421        #
1422        info = importer.info
1423        data_keys = importer.data_keys
1424        csv_keys = importer.csv_keys
1425        #csv_keys.extend(info.keys())
1426        headline_mapping = dict((k,k) for k in csv_keys)
1427        #
1428        # pending
1429        #
1430        pending_path = importer.pending_path
1431        pending_tmp = importer.pending_tmp
1432        pending_backup = importer.pending_backup
1433        pending_fn = importer.pending_fn
1434        imported_path = importer.imported_path
1435        imported_fn = importer.imported_fn
1436        commit_after = importer.commit_after
1437        pending = []
1438        pending_digests = []
1439        #total_added_to_pending = 0
1440        if not pending_only:
1441            pending,pending_digests = importer.makeIdLists()
1442            pending_at_start = len(pending)
1443        datafile = open(pending_tmp,"w")
1444        pending_csv_writer = csv.DictWriter(datafile,
1445                                                    csv_keys,
1446                                                    extrasaction='ignore')
1447        pending_csv_writer.writerow(headline_mapping)
1448        datafile.close()
1449        #
1450        # imported
1451        #
1452        if not os.path.exists(imported_path):
1453            datafile = open(imported_path,"w")
1454            imported_csv_writer = csv.DictWriter(datafile,
1455                                                 csv_keys,
1456                                                 extrasaction='ignore')
1457            imported_csv_writer.writerow(headline_mapping)
1458            datafile.close()
1459        start = True
1460        tr_count = 0
1461        total = 0
1462        total_added_to_imported = 0
1463        total_pending = 0
1464        import_source_done = ""
1465        if pending_only:
1466            import_source_path = pending_path
1467        else:
1468            import_source_path = "%s/import/%s.csv" % (i_home,filename)
1469            import_source_done = "%s/import/%s.done" % (i_home,filename)
1470        if not os.path.exists(import_source_path):
1471            fn = os.path.split(import_source_path)[1]
1472            em = 'no import file %(fn)s' % vars()
1473            return em
1474        import_source_fn = os.path.split(import_source_path)[1]
1475        if not pending_only:
1476            info['imported_from'] = import_source_fn
1477        headline = csv.reader(open(import_source_path,"rb")).next()
1478        if "import_mode" not in headline:
1479            msg = 'import_mode must be in heading'
1480            return msg
1481        invalid_keys = importer.checkHeadline(headline)
1482        if invalid_keys:
1483            return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys)
1484
1485        import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore')
1486                                                         or k.strip() in info.keys())]
1487        # diff2schema = set(import_keys).difference(set(importer.schema.keys()))
1488        # diff2layout = set(import_keys).difference(set(importer.layout.keys()))
1489        # if diff2schema and diff2schema != set(['id',]):
1490        #     msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
1491        #     return msg
1492        #
1493        # start importing
1494        #
1495        try:
1496            reader = csv.DictReader(open(import_source_path,"rb"))
1497        except:
1498            msg = 'Error reading %s.csv' % filename
1499            logger.error(msg)
1500            return msg
1501        items = [item for item in reader]
1502        total_to_import = len(items)
1503        tti_float = float(total_to_import)
1504        if pending_only:
1505            pending_at_start = total_to_import
1506        count = 0
1507        imported = []
1508        old_commit_count = -1
1509        error_count = imported_count = 0
1510        already_in = 0
1511        for record in items:
1512            item = {}
1513            empty_value_keys = []
1514            for k,v in record.items():
1515                if v:
1516                    if v == EMPTY:
1517                        empty_value_keys += k,
1518                        v = ''
1519                    item[k.strip()] = v.strip()
1520            count += 1
1521            if start:
1522                start = False
1523                adapters = [MappingStorageAdapter(importer.schema, item)]
1524                logger.info('%(member)s starts import from %(import_source_fn)s' % vars())
1525            dm = DataModel(item, adapters,context=self)
1526            ds = DataStructure(data=item,datamodel=dm)
1527            error_string = ""
1528            total += 1
1529            import_mode = item.get('import_mode','')
1530            import_method = getattr(importer, '%(import_mode)s' % vars(),None )
1531            if import_method is None:
1532                error_string += "import_mode '%(import_mode)s' is invalid" % vars()
1533            elif (import_mode in importer.required_modes and
1534                not set(importer.required_keys[import_mode]).issubset(set(item.keys()))):
1535                diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys()))
1536                error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import)
1537            else:
1538                for k in import_keys:
1539                    if k not in item.keys() or k not in importer.validators.keys():
1540                        continue
1541                    if not importer.validators[k](ds,mode=import_mode):
1542                        error_string += ' ++ '
1543                        error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
1544                                                                           ds.getErrorMapping(k)))
1545            if error_string:
1546                error = error_string
1547                id = ''
1548                mapping = item
1549            else:
1550                temp_item = item.copy()
1551                temp_item.update(dm)
1552                results = import_method(temp_item)
1553                id = results[0]
1554                error = results[1]
1555                mapping = results[2]
1556            #
1557            # restore some values
1558            #
1559            for k in empty_value_keys:
1560                mapping[k] = EMPTY
1561            if mapping.has_key('sex'):
1562                #import pdb;pdb.set_trace()
1563                if mapping['sex'] == True:
1564                    mapping['sex'] = 'F'
1565                elif mapping['sex'] == False:
1566                    mapping['sex'] = 'M'
1567            if pending_only:
1568                info['imported_from'] = item['imported_from']
1569            data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items())
1570            info['error'] = error
1571            info['import_record_no'] = count + 1
1572            mapping.update(info)
1573            log_list = []
1574            if error:
1575                error_count += 1
1576                digest = makeDigest(mapping,data_keys)
1577                if digest not in pending_digests:
1578                    pending_digests += digest,
1579                    pending.append(mapping)
1580                    if not pending_only:
1581                        log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),
1582                else:
1583                    already_in += 1
1584                    pass
1585            else:
1586                imported_count += 1
1587                imported += mapping,
1588                log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(),
1589            if log_list:
1590                time_till_now = time.time() - elapse
1591                percent_finished = (error_count + imported_count)/tti_float*100
1592                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),)
1593                logger.info(' '.join(log_list))
1594            finished = count > total_to_import - 1
1595            must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after)
1596            if must_commit:
1597                old_commit_count = imported_count
1598
1599            if must_commit or finished:
1600                if len(imported):
1601                    transaction.commit()
1602                    datafile = open(imported_path,"a")
1603                    writer = csv.DictWriter(datafile,
1604                                            csv_keys,
1605                                            extrasaction='ignore')
1606                    writer.writerows(imported)
1607                    datafile.close()
1608                    total_added_to_imported += len(imported)
1609                    imported = []
1610                if len(pending) > 0:
1611                    datafile = open(pending_tmp,"a")
1612                    writer = csv.DictWriter(datafile,
1613                                            csv_keys,
1614                                            extrasaction='ignore')
1615                    writer.writerows(pending)
1616                    datafile.close()
1617                    total_pending += len(pending)
1618                    #total_added_to_pending += len(pending)
1619                    pending = []
1620                if not finished:
1621                    msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars()
1622                    logger.info(msg)
1623        elapse = time.time() - elapse
1624        if os.path.exists(pending_path):
1625            copy2(pending_path,pending_backup)
1626        copy2(pending_tmp,pending_path)
1627        msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars()
1628        msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars()
1629        if pending_only:
1630            removed_pending = pending_at_start - total_pending
1631            msg += "%(removed_pending)d removed from %(pending_fn)s" % vars()
1632        else:
1633            added_pending = total_pending - pending_at_start
1634            msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars()
1635        #msg += "%(total_pending)d totally written" % vars()    # this line does not make any sense
1636        logger.info(msg)
1637        if import_source_done:
1638            copy(import_source_path,import_source_done)
1639            os.remove(import_source_path)
1640            upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None)
1641            if upload is not None:
1642                upload_doc = upload.getContent()
1643                mapping = {}
1644                #mapping['import_date'] = DateTime.DateTime()
1645                mapping['import_date'] = import_date
1646                mapping['imported_by'] = importer.imported_by
1647                mapping['import_message'] = msg
1648                upload_doc.edit(mapping = mapping)
1649        os.remove(pending_tmp)
1650        return msg
1651    ###)
1652
1653    security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
1654    def moveImagesToFS(self,student_id="O738726"):
1655        "move the images to the filesystem"
1656        images_dir = getImagesDir(student_id)
1657        #images_dir = os.path.join("%s" % images_base,student_id)
1658        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1659        stool = getToolByName(self, 'portal_schemas')
1660        schemas = ['student_application',
1661                   'student_clearance',
1662                   ]
1663        created = False
1664        for schema_id in schemas:
1665            schema = stool._getOb(schema_id)
1666            object = getattr(student_folder,schema_id[len('student_'):],None)
1667            if object is None:
1668                continue
1669            doc = object.getContent()
1670            for key in schema.keys():
1671                if schema[key].meta_type != "CPS Image Field":
1672                    continue
1673                #import pdb;pdb.set_trace()
1674                image = getattr(doc,key,None)
1675                if not image or not hasattr(image,"data"):
1676                    continue
1677                if not created:
1678                    if not os.path.exists(images_dir):
1679                        os.mkdir(images_dir)
1680                    created = True
1681                filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars())
1682                open(filename,"wb").write(str(image.data))
1683    ###)
1684
1685    security.declareProtected(ModifyPortalContent,"movePassportToFS")###(
1686    def movePassportToFS(self,student_id="O738726"):
1687        "move the passports to the filesystem"
1688        images_dir = os.path.join("%s" % i_home,'passports')
1689        student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id)
1690        stool = getToolByName(self, 'portal_schemas')
1691        schemas = ['student_application',
1692                   #'student_clearance',
1693                   ]
1694        created = False
1695        for schema_id in schemas:
1696            schema = stool._getOb(schema_id)
1697            object = getattr(student_folder,schema_id[len('student_'):],None)
1698            if object is None:
1699                continue
1700            doc = object.getContent()
1701            for key in schema.keys():
1702                if schema[key].meta_type != "CPS Image Field":
1703                    continue
1704                #import pdb;pdb.set_trace()
1705                image = getattr(doc,key)
1706                if not hasattr(image,"data"):
1707                    continue
1708                if not created:
1709                    if not os.path.exists(images_dir):
1710                        os.mkdir(images_dir)
1711                    created = True
1712                filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars())
1713                open(filename,"wb").write(str(image.data))
1714    ###)
1715
1716InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.