source: WAeUP_SRP/base/WAeUPTool.py @ 2267

Last change on this file since 2267 was 2267, checked in by Henrik Bettermann, 17 years ago

review_state and other fields removed from import schemas and layouts
  • Property svn:keywords set to Id
File size: 55.4 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 The WAeUP group  <http://www.waeup.org>
3# Author: Joachim Schmitz (js@aixtraware.de)
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTool.py 2267 2007-09-21 19:36:08Z henrik $
20"""The WAeUP Tool Box.
21"""
22
23from AccessControl import ClassSecurityInfo
24from Acquisition import aq_inner
25from Acquisition import aq_parent
26from Globals import DTMLFile
27from Globals import InitializeClass
28from OFS.SimpleItem import SimpleItem
29
30from Products.CMFCore.utils import getToolByName
31from Products.CPSSchemas.DataStructure import DataStructure
32from Products.CPSSchemas.DataModel import DataModel
33from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
34from Products.CMFCore.ActionProviderBase import ActionProviderBase
35from Products.CMFCore.permissions import View
36from Products.ZCatalog.ZCatalog import ZCatalog
37from Products.CMFCore.permissions import ModifyPortalContent
38from Products.CMFCore.permissions import ManagePortal
39from Products.CMFCore.utils import UniqueObject
40from Products.CMFCore.URLTool import URLTool
41from Products.CMFCore.utils import getToolByName
42from Students import makeCertificateCode
43from Globals import package_home,INSTANCE_HOME
44p_home = package_home(globals())
45i_home = INSTANCE_HOME
46import DateTime,time
47import logging
48import transaction
49import csv,re,os,sys
50from Products.AdvancedQuery import Eq, Between, Le,In
51
52def getObject(object,name):
53    if object.hasObject(name):
54        return getattr(object,name)
55    return None
56
57class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase):
58    """WAeUP tool"""
59
60    id = 'waeup_tool'
61    meta_type = 'WAeUP Tool'
62    _actions = ()
63    security = ClassSecurityInfo()
64    security.declareObjectProtected(View)
65    manage_options = ( ActionProviderBase.manage_options
66                     + SimpleItem.manage_options
67                     )
68
69    def rwrite(self,s): ###(
70        response = self.REQUEST.RESPONSE
71        response.setHeader('Content-type','text/html; charset=ISO-8859-15')
72        response.write("%s<br />\r\n" % s)
73    ###)
74
75    def sleep(self,secs): ###(
76        "sleep"
77        import time
78        time.sleep(secs)
79        return
80
81###)
82
83    security.declareProtected(ModifyPortalContent,'openLog') ###(
84    def openLog(self,name):
85        """open a log file"""
86        version = 1
87        path = "%s/log/%s_%d.log" % (i_home,name,version)
88        while os.path.exists(path):
89            version += 1
90            path = "%s/log/%s_%d.log" % (i_home,name,version)
91        log = open(path,"w")
92        return log
93    ###)
94
95    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
96    def bypassQueueCatalog(self,enable=True):
97        """bypass the QueueCatalog by setting all indexes to process imediate,
98        if enable is True (default) the old settings are restored
99        """
100
101    ###)
102
103    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
104    def measureOaT(self,method="a",probe="1000",nr_pts="1"):
105        """measure Object access Time"""
106        import random
107        if hasattr(self,'portal_catalog_real'):
108            aq_portal = self.portal_catalog_real.evalAdvancedQuery
109        else:
110            aq_portal = self.portal_catalog.evalAdvancedQuery
111        nr_pts = int(nr_pts)
112        probe = int(probe)
113        intervall = probe/10
114        objects = ("application","clearance","personal")
115        portal_types = ("StudentApplication","StudentClearance","StudentPersonal")
116        #i = random.randrange(num_objects)
117        count = 0
118        found = 0
119        not_found = 0
120        t_found = 0
121        t_not_found = 0
122        time_found = time_not_found = 0.0
123        t_time_found = t_time_not_found = 0.0
124        accessed = []
125        t_min = 1000
126        t_max = 0
127        #import pdb;pdb.set_trace()
128        students = self.portal_catalog(portal_type="Student")
129        num_students = len(students)
130        if method == "d":
131            query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts])
132            res = aq_portal(query)
133            brains = {}
134            for r in res:
135                sid = r.relative_path.split('/')[-2]
136                if brains.has_key(sid):
137                    brains[sid][r.portal_type] = r
138                else:
139                    brains[sid] = {r.portal_type : r}
140            brains_list = brains.keys()
141            num_objects = len(brains_list)
142        else:
143            num_objects = num_students
144        print "="*40
145        print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method,
146                                                                        probe,
147                                                                        nr_pts,
148                                                                        num_objects)
149        print "nr found/not time found/not min/max"
150        elapse = time.time()
151        i_elapse = time.time()
152        c_elapse = time.clock()
153        for c in range(1,probe + 1):
154            i = random.randrange(num_objects)
155            if method in ('a','b','c'):
156                student_brain = students[i]
157            elif method == "d":
158                #import pdb;pdb.set_trace()
159                student_brain = brains[brains_list[i]]
160            if method == "c":
161                query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts])
162                res = aq_portal(query)
163                this_portal_types = [r.portal_type for r in res]
164            for i in range(nr_pts):
165                oid = objects[i]
166                if method == "a":
167                    try:
168                        student_path = student_brain.getPath()
169                        path = "%s/%s" % (student_path,oid)
170                        doc = self.unrestrictedTraverse(path).getContent()
171                        found += 1
172                        i_time = time.time() - i_elapse
173                        time_found += i_time
174                    except:
175                        not_found += 1
176                        i_time = time.time() - i_elapse
177                        time_not_found += i_time
178                        pass
179                elif method == "b":
180                    try:
181                        student_object = student_brain.getObject()
182                        doc = getattr(student_object,oid).getContent()
183                        found += 1
184                        i_time = time.time() - i_elapse
185                        time_found += i_time
186                    except:
187                        i_time = time.time() - i_elapse
188                        time_not_found += i_time
189                        not_found += 1
190                        pass
191                elif method == "c":
192                    if portal_types[i] in this_portal_types:
193                        found += 1
194                        doc = res[this_portal_types.index(portal_types[i])].getObject().getContent()
195                        i_time = time.time() - i_elapse
196                        time_found += i_time
197                    else:
198                        not_found += 1
199                        i_time = time.time() - i_elapse
200                        time_not_found += i_time
201                elif method == "d":
202                    if student_brain.has_key(portal_types[i]):
203                        found += 1
204                        doc = student_brain[portal_types[i]].getObject().getContent()
205                        i_time = time.time() - i_elapse
206                        time_found += i_time
207                    else:
208                        not_found += 1
209                        i_time = time.time() - i_elapse
210                        time_not_found += i_time
211                i_elapse = time.time()
212            if c and (c % intervall == 0):
213                #i_time = time.time() - i_elapse
214                t_per = 0.0
215                if found:
216                    t_per = time_found/found
217                if t_per > t_max:
218                    t_max = t_per
219                if t_per > 0.0 and t_per < t_min:
220                    t_min = t_per
221                itf = 0.0
222                if found:
223                    itf = time_found/found
224                itnf = 0.0
225                if not_found :
226                    itnf = time_not_found / not_found
227                interval_time = time_found + time_not_found
228                s = "%(c)d: %(found)d/%(not_found)d " % vars()
229                s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
230                s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
231                print s
232                t_found += found
233                t_not_found += not_found
234                t_time_found += time_found
235                t_time_not_found += time_not_found
236                time_found = time_not_found = 0.0
237                found = not_found = 0
238        # t_found += found
239        # t_not_found += not_found
240        elapse = time.time() - elapse
241        itf = 0.0
242        if t_found:
243            itf = t_time_found/t_found
244        itnf = 0.0
245        if t_not_found:
246            itnf = t_time_not_found / t_not_found
247        #c_elapse = time.clock() - c_elapse
248        s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars()
249        s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars()
250        s += "%(t_min)6.4f/%(t_max)6.4f" %  vars()
251        print "-"*40
252        print s
253        rel_found = float(t_found)/probe
254        rel_not_found = float(t_not_found)/probe
255        estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf)
256        print estimated_total_time
257    ###)
258
259    security.declareProtected(ModifyPortalContent,'writeLog') ###(
260    def writeLog(self,logfile,s):
261        """write to the log file"""
262        logfile.write(s)
263
264###)
265
266    def generateStudentId(self,letter): ###(
267        import random
268        r = random
269        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
270        if letter == '?':
271            letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY')
272        sid = "%c%d" % (letter,r.randint(99999,1000000))
273        students = self.portal_url.getPortalObject().campus.students
274##        while hasattr(students, sid):
275##            sid = "%c%d" % (letter,r.randint(99999,1000000))
276        while self.students_catalog(id = sid):
277            sid = "%c%d" % (letter,r.randint(99999,1000000))
278        return sid
279    ###)
280
281    def generatePassword(self,s=None): ###(
282        import random
283        r = random
284        ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'):
285        if s is None:
286            s = 'abcdefghklmnpqrstuvwxy23456789'
287        pw = ''
288        while len(pw) < 6:
289            pw += r.choice(s)
290        return pw
291    ###)
292
293    security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###(
294    def removeDeletedDocIds(self, max=1000):
295        """
296        remove deleted docids from repository commit after max
297        """
298        logger = logging.getLogger('WAeUPTool.removeDeletedDocIds')
299        repository = getToolByName(self, 'portal_repository')
300        pxtool = getToolByName(self, 'portal_proxies')
301        pxtool_infos = pxtool.getRevisionsUsed()
302
303        nb_revs = 0
304        docids_d = {} # all docids
305        unused_docids_d = {} # all docids that are unused
306        ids_unused_revs_docids = [] # ids for revs of unused docids
307        ids_unused_revs = [] # ids for unused revs
308        total = 0
309        idlist = repository.objectIds()
310        for id in idlist:
311            docid, rev = repository._splitId(id)
312            if docid is None:
313                logger.info("invalid doc_id %s" % docid)
314                continue
315            nb_revs += 1
316            docids_d[docid] = None
317            if not pxtool_infos.has_key(docid):
318                unused_docids_d[docid] = None
319                ids_unused_revs_docids.append(id)
320                ids_unused_revs.append(id)
321            elif not pxtool_infos[docid].has_key(rev):
322                ids_unused_revs.append(id)
323            if len(ids_unused_revs) >= max:
324                repository.manage_delObjects(ids_unused_revs)
325                #import pdb;pdb.set_trace()
326                transaction.commit()
327                total += max
328                logger.info('removed %d total %d unused docids ' % (max,total))
329        anz = len(ids_unused_revs)
330        if anz > 0:
331            repository.manage_delObjects(ids_unused_revs)
332            transaction.commit()
333            total += anz
334            logger.info('finished removing %d unused docids ' % (total))
335
336
337###)
338
339    security.declareProtected(ModifyPortalContent,'getCredential') ###(
340    def getCredential(self,student_id):
341        "return a student password"
342        student_entry = getattr(self.portal_directories.students,student_id,None)
343        if student_entry is None:
344            return None
345        return getattr(student_entry,"password","not set")
346    ###)
347
348    security.declarePublic('checkPassword') ###(
349    def checkPassword(self,student_id,password):
350        "return a student password"
351        student_entry = getattr(self.portal_directories.students,student_id,None)
352        if student_entry is None:
353            return False
354        return getattr(student_entry,"password","not set") == password
355    ###)
356
357    security.declarePublic('editPassword') ###(
358    def editPassword(self,student_id,password):
359        "edit a student password"
360        student_entry = getattr(self.portal_directories.students,student_id,None)
361        if student_entry is None:
362            return
363        setattr(student_entry,'password',password)
364    ###)
365
366    security.declareProtected(View,'doCommit') ###(
367    def doCommit(self,logger=None):
368        "commit some transactions"
369        transaction.commit()
370    ###)
371
372    security.declarePublic('loadStudentFoto') ###(
373    def loadStudentFoto(self,student,filename,folder):
374        "return a student passport picture"
375        app = student.application
376        app_doc = app.getContent()
377        #clear = student.clearance
378        #clear_doc = clear.getContent()
379        #matric_no = clear_doc.matric_no.upper()
380        picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename)
381        picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename)
382        #import pdb;pdb.set_trace()
383        if os.path.exists(picture1):
384            file = open(picture1)
385        elif os.path.exists(picture2):
386            file = open(picture2)
387        else:
388            return "passport picture not found %s" % picture1
389        reopened = False
390        if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened':
391            self.portal_workflow.doActionFor(app,'open')
392            reopened = True
393        outfile = file.read()
394        app_doc.manage_addFile('passport',
395                               file=outfile,
396                               title="%s.jpg" % filename)
397        if reopened:
398            self.portal_workflow.doActionFor(app,'close')
399        return "successfully loaded passport picture"
400    ###)
401
402    security.declareProtected(ModifyPortalContent,'createOne') ###(
403    def createOne(self,students_folder,student_brain,letter,commit=False):
404        sid = self.waeup_tool.generateStudentId(letter)
405        students_folder.invokeFactory('Student', sid)
406        student = getattr(students_folder,sid)
407        self.portal_workflow.doActionFor(student,'return')
408        student.manage_setLocalRoles(sid, ['Owner',])
409        matric_no = student_brain.matric_no
410        jamb_reg_no = student_brain.Entryregno
411        self.students_catalog.addRecord(id = sid,
412                                           matric_no = matric_no,
413                                           jamb_reg_no = jamb_reg_no,
414                                           sex = student_brain.Sex == "F",
415                                           name = "%s %s %s" % (student_brain.Firstname,
416                                                                student_brain.Middlename,
417                                                                student_brain.Lastname)
418                                        )
419        if commit:
420            transaction.commit()
421        return sid,jamb_reg_no
422    ###)
423
424    security.declareProtected(ModifyPortalContent,'addStudent') ###(
425    def addStudent(self,dict):
426        students_folder = self.portal_url.getPortalObject().campus.students
427        sid = self.waeup_tool.generateStudentId('?')
428        students_folder.invokeFactory('Student', sid)
429        student_obj = getattr(students_folder,sid)
430        f2t = self.student_field2types
431        #from pdb import set_trace; set_trace()
432        d = {}
433        d['jamb_sex']  = 'M'
434        if dict.get('sex'):
435            d['jamb_sex']  = 'F'
436
437        entry_session = dict.get('entry_session')
438        if entry_session == self.getSessionId()[-2:]:
439            wfaction = 'admit'
440            wft = 'wf_transition_admit'
441            password = None
442        else:
443            wfaction = 'return'
444            wft = 'wf_transition_return'
445            password = self.generatePassword()
446            self.makeStudentMember(sid,password)
447
448        for pt in f2t.keys():
449            student_obj.invokeFactory(pt,f2t[pt]['id'])
450            sub_obj = getattr(student_obj,f2t[pt]['id'])
451            sub_doc = sub_obj.getContent()
452            #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj)
453            d['Title'] = f2t[pt]['title']
454            for field in f2t[pt]['fields']:
455                d[field] = dict.get(field,'')
456            sub_doc.edit(mapping = d)
457            new_state = f2t[pt][wft]
458            if new_state != "remain":
459                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
460        self.portal_workflow.doActionFor(student_obj,wfaction)
461        student_obj.manage_setLocalRoles(sid, ['Owner',])
462        return sid,password
463    ###)
464
465    security.declarePublic('getCertificateBrain') ###(
466    def getCertificateBrain(self,cert_id):
467        "do it"
468        res = ZCatalog.searchResults(self.portal_catalog_real,
469                                {'portal_type':"Certificate",
470                                      'id': cert_id})
471        if res:
472            return res[0]
473        return None
474    ###)
475
476    security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###(
477    def get_csv_filenames(self):
478        "do it"
479        files = [file for file in os.listdir("%s/import/" % (i_home))
480                 if file.endswith('.csv') and file.find('imported') == -1]
481        return files
482    ###)
483
484    security.declarePublic('findStudentByMatricelNo') ###(
485    def findStudentByMatricelNo(self,matric_no):
486        "do it"
487        res = ZCatalog.searchResults(self.portal_catalog_real,
488                                {'portal_type':"StudentClearance",
489                                 'SearchableText': matric_no})
490        if res:
491            return res[0]
492        return None
493    ###)
494
495    security.declarePublic('makeStudentMember') ###(
496    def makeStudentMember(self,sid,password='uNsEt'):
497        """make the student a member"""
498        membership = self.portal_membership
499        membership.addMember(sid,
500                             password ,
501                             roles=('Member',
502                                     'Student',
503                                     ),
504                             domains='',
505                             properties = {'memberareaCreationFlag': False,
506                                           'homeless': True},)
507        member = membership.getMemberById(sid)
508        self.portal_registration.afterAdd(member, sid, password, None)
509        #self.manage_setLocalRoles(sid, ['Owner',])
510    ###)
511
512    security.declarePublic('makeStudentData') ###(
513    def makeStudentData(self,student_id,email=None,phone_nr=None):
514        "create Datastructure for a returning Student"
515        #import pdb;pdb.set_trace()
516        logger = logging.getLogger('WAeUPTool.makeStudentData')
517        students_folder = self.portal_url.getPortalObject().campus.students
518        #res = self.students_catalog(id=student_id)
519        #if res:
520        #    st = res[0]
521        #res = self.returning_import(matric_no = st.matric_no)
522        res = self.returning_import(id = student_id)
523        if res:
524            student = res[0]
525        else:
526            logger.info('Id %s not found in returning_import' % student_id)
527            return
528        logger.info('%s creates data structure' % student_id)
529        s_results = self.results_import(matric_no = student.matric_no)
530        if s_results:
531            lnr = self.getLevelFromResultsCosCode(s_results)
532            level = "%d00" % lnr
533            verdict,eligible = self.getVerdict(s_results[0].Verdict)
534            if eligible:
535                level = "%d00" % (lnr + 1)
536        else:
537            logger.info('matric_no %s not found in results_import' % student.matric_no)
538            level = 0
539            verdict = 'N/A'
540        #student should not be allowed to perform this transition
541        #wftool = self.portal_workflow
542        #wftool.doActionFor(student,'return')
543        certcode_org = student.Coursemajorcode
544        certcode = makeCertificateCode(certcode_org)
545        certificate_brain = self.getCertificateBrain(certcode)
546        if not certificate_brain:
547            em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org)
548            logger.info(em)
549        matric_no = student.matric_no
550        sid = student_id
551        student_obj = getattr(students_folder,sid)
552        student_obj.invokeFactory('StudentApplication','application')
553        application = student_obj.application
554        self.portal_workflow.doActionFor(application,'open',dest_container=application)
555        da = {'Title': 'Application Data'}
556        student_obj.invokeFactory('StudentPersonal','personal')
557        da['jamb_reg_no'] = student.Entryregno
558        em = self.getEntryMode(student.Entryregno)
559##        em = student.Mode_of_Entry
560##        if em in ('DIRECT', 'DIRECT ENTRY',):
561##            em = 'DE'
562##        elif em in ('U.M.E', 'UNE',):
563##            em = 'UME'
564##        elif not em:
565##            em = "unknown"
566        da['entry_mode'] = em
567        personal = student_obj.personal
568        self.portal_workflow.doActionFor(personal,'open',dest_container=personal)
569        dp = {'Title': 'Personal Data'}
570        student_obj.invokeFactory('StudentClearance','clearance')
571        clearance = student_obj.clearance
572        self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance)
573        dc = {'Title': 'Clearance/Eligibility Record'}
574        dc['matric_no'] = matric_no
575        state = student.State
576        lga = student.LGA
577        if state and lga:
578            lga =  state + ' / ' + lga
579        else:
580            lga = "None"
581        da['jamb_lga'] = dc['lga'] = lga
582        da['app_email'] = dp['email'] = email
583        da['app_mobile'] = dp['phone'] = phone_nr
584        dp['firstname'] = student.Firstname
585        dp['middlename'] = student.Middlename
586        dp['lastname'] = student.Lastname
587        da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname)
588        da['jamb_sex'] = student.Sex
589        dp['sex'] = student.Sex == 'F'
590        dp['perm_address'] = student.Permanent_Address
591        application.getContent().edit(mapping=da)
592        self.portal_workflow.doActionFor(application,'close',dest_container=application)
593        personal.getContent().edit(mapping=dp)
594        clearance.getContent().edit(mapping=dc)
595        self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance)
596##        catd = {}
597##        catd['id'] = sid
598##        catd['entry_mode']= da['entry_mode']
599##        catd['matric_no'] = matric_no
600##        catd['jamb_reg_no'] = da['jamb_reg_no']
601##        catd['name'] = "%(firstname)s %(middlename)s %(lastname)s" % dp
602##        catd['sex'] = dp['sex']
603##        catd['level'] = level
604##        catd['verdict'] = verdict
605##        if certificate_brain:
606##            cpath = certificate_brain.getPath().split('/')
607##            catd['faculty'] = cpath[-4]
608##            catd['department'] = cpath[-3]
609##            catd['course'] = certcode
610##        self.students_catalog.modifyRecord(**catd)
611        #
612        # Study Course
613        #
614        student_obj.invokeFactory('StudentStudyCourse','study_course')
615        studycourse = student_obj.study_course
616        self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse)
617        dsc = {}
618        dsc['study_course'] = certcode
619        dsc['current_level'] = level
620        dsc['current_verdict'] = verdict
621        dsc['current_mode'] = em
622        dsc['current_session'] = '06'
623        studycourse.getContent().edit(mapping=dsc)
624        #
625        # Level
626        #
627##        l = getattr(studycourse,level,None)
628##        if l is None:
629##            studycourse.invokeFactory('StudentStudyLevel', level)
630##            l = getattr(studycourse, level)
631##            self.portal_workflow.doActionFor(l,'open',dest_container=l)
632##            l.getContent().edit(mapping={'Title': "Level %s" % level})
633###)
634
635    security.declarePublic('makeStudentLevel') ###(
636    def makeStudentLevel(self,student_id):
637        "create the StudyLevel for a returning Student"
638        #import pdb;pdb.set_trace()
639        logger = logging.getLogger('WAeUPTool.makeStudentLevel')
640        students_folder = self.portal_url.getPortalObject().campus.students
641        res = self.students_catalog(id=student_id)
642        if res:
643            st = res[0]
644        course = st.course
645        matric_no = st.matric_no
646        level = st.level
647        res = self.results_import(matric_no = matric_no)
648        if res:
649            results = res
650        logger.info('%s creating Level %s' % (student_id,level))
651        #
652        # Level
653        #
654        student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id)
655        studycourse = getattr(student_obj,"study_course",None)
656        self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse)
657        l = getattr(studycourse,level,None)
658        if l is None:
659            studycourse.invokeFactory('StudentStudyLevel', level)
660            l = getattr(studycourse, level)
661            self.portal_workflow.doActionFor(l,'open',dest_container=l)
662            l.getContent().edit(mapping={'Title': "Level %s" % level})
663        ###)
664
665    security.declarePublic('getHallInfo') ###(
666    def getHallInfo(self,bed):
667        """return Hall Info"""
668        info = {}
669        hall,block,room,letter = bed.split('_')
670        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall)
671        if res and len(res) == 1:
672            hall_brain = res[0]
673            hall_doc = hall_brain.getObject().getContent()
674        else:
675            return info
676        info['hall_title'] = hall_brain.Title
677        info['maintenance_code'] = hall_doc.maintenance_code
678        res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch")
679        batch_doc = None
680        for brain in res:
681            if brain.id.startswith(info['maintenance_code']):
682                batch_doc = brain.getObject().getContent()
683                break
684        if batch_doc is None:
685            info['maintenance_fee'] = None
686        else:
687            info['maintenance_fee'] = batch_doc.cost
688        return info
689    ###)
690
691    security.declarePublic('showFsPicture') ###(
692    def showFsPicture(self,path):
693        """return a picture from the filesystem"""
694        picture_path = "%s/import/%s" % (i_home,path)
695        if os.path.exists(picture_path):
696            return open(picture_path).read()
697
698    ###)
699
700    security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###(
701    def deleteAllCourses(self,department="All"):
702        ''' delete the courses'''
703        pm = self.portal_membership
704        member = pm.getAuthenticatedMember()
705
706        if str(member) not in ("henrik","joachim"):
707            return "not possible"
708        if department == "All":
709            res = self.portal_catalog({'meta_type': 'Department'})
710        if len(res) < 1:
711            return "No Departments found"
712
713        deleted = []
714        for dep in res:
715            cf = dep.getObject().courses
716            if cf:
717                cf.manage_delObjects(ids=cf.objectIds())
718                deleted.append("deleted Courses in %s" % dep.getId)
719        return "\r".join(deleted)
720    ###)
721
722    security.declareProtected(ModifyPortalContent,'getLogfileLines') ###(
723    def getLogfileLines(self,filename="event.log",numlines=20):
724        """Get last NUMLINES lines of logfile FILENAME.
725
726        Return last lines' of a file in the instances logfile directory as
727        a list. The number of returned lines equals `numlines' or less. If
728        less than `numlines' lines are available, the whole file ist
729        returned. If the file can not be opened or some other error
730        occurs, empty list is returend.
731        """
732        result = []
733        lines_hit = 0
734
735        # We only handle files in instances' log directory...
736        logpath = os.path.join(i_home, "log")
737        filename = str(os.path.abspath( os.path.join( logpath, filename )))
738        if not filename.startswith( logpath ):
739            # Attempt to access file outside log-dir...
740            return []
741
742        try:
743            fd = file( filename, "rb" )
744        except IOError:
745            return []
746        if not fd:
747            return []
748
749        if os.linesep == None:
750            linesep = '\n'
751        else:
752            linesep = os.linesep
753
754        # Try to find 'numlines' times a lineseparator, searching from end
755        # and moving to the beginning of file...
756        fd.seek( 0, 2) # Move to end of file...
757        while lines_hit < numlines:
758            if fd.read(1) == linesep[-1]: # This moves filedescriptor
759                                          # one step forward...
760                lines_hit += 1
761            try:
762                fd.seek( -2, 1) # Go two bytes back from current pos...
763            except IOError:
764                # We cannot go back two bytes. Maybe the file is too small...
765                break
766        fd.seek(2,1)
767
768        # Read all lines from current position...
769        result = fd.readlines()
770        # Remove line endings...
771        result = [x.strip() for x in result]
772        fd.close()
773        return result
774    ###)
775
776    security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###(
777    def getCallbacksFromLog(self,filename):
778        """fix Online Payment Transactions from Z2.log entries"""
779        import transaction
780        import random
781        from cgi import parse_qs
782        from urlparse import urlparse
783        #from pdb import set_trace
784        wftool = self.portal_workflow
785        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
786        students_folder = self.portal_url.getPortalObject().campus.students
787        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
788        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
789        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
790        data = re.compile(s)
791        start = True
792        tr_count = 1
793        total = 0
794        #name = 'pume_results'
795        #name = 'epaymentsuccessful_z2log2'
796        name = filename
797        no_import = []
798        imported = []
799        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
800        try:
801            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
802        except:
803            logger.error('Error reading %s' % name)
804            return
805        tas = []
806        for line in transactions:
807            dict = {}
808            items = data.search(line)
809            dict['idict'] = idict = items.groupdict()
810            #print idict
811            #from pdb import set_trace;set_trace()
812            urlparsed = urlparse(idict['get'][4:])
813            #print urlparsed
814            path = urlparsed[2].split('/')
815            dict['student_id'] = student_id = path[8]
816            dict['payment_id'] = payment_id = path[10]
817            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
818            tas.append(dict)
819            tr_count += 1
820        return tas
821    ###)
822
823    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
824    def importOnlinePaymentTransactions(self):
825        """load Online Payment Transactions from CSV values"""
826        import transaction
827        import random
828        #from pdb import set_trace
829        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
830        opt = self.online_payments_import
831        students_folder = self.portal_url.getPortalObject().campus.students
832        start = True
833        tr_count = 1
834        total = 0
835        #name = 'pume_results'
836        name = 'OnlineTransactions'
837        no_import = []
838        imported = []
839        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
840        try:
841            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
842        except:
843            logger.error('Error reading %s.csv' % name)
844            return
845        for pay_transaction in transactions:
846            if start:
847                start = False
848                logger.info('Start loading from %s.csv' % name)
849                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
850                no_import.append('%s,"Error"' % s)
851                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
852                format_error = format + ',"%(Error)s"'
853            data = {}
854
855            # format of the first file sent by Tayo
856            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
857            #data['student_id'] = student_id = pay_transaction['Payer ID']
858            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
859            #data['response_code'] = response_code = pay_transaction['Resp Code']
860            #data['amount'] = amount = pay_transaction['Amount']
861
862            # format of the second file sent by Tayo
863            #data['datetime'] = date = 0
864            #data['student_id'] = student_id = pay_transaction['Payer ID']
865            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
866            #data['response_code'] = response_code = '00'
867            #data['amount'] = amount = pay_transaction['Amount']
868
869            # format of the third file sent by Kehinde
870            data['datetime'] = date = 0
871            data['student_id'] = student_id = pay_transaction['customer_id']
872            data['order_id'] = order_id = pay_transaction['merchant_reference']
873            data['response_code'] = response_code = '00'
874            data['amount'] = amount = pay_transaction['Amount']
875
876            dup = False
877            if response_code == "12":
878                continue
879            try:
880                opt.addRecord(**data)
881            except ValueError:
882                dup = True
883            #from pdb import set_trace;set_trace()
884            if dup:
885                if response_code == "00":
886                    try:
887                        opt.modifyRecord(**data)
888                    except:
889                        logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data)
890                        continue
891                else:
892                    pay_transaction['Error'] = "Duplicate order_id"
893                    no_import.append( format_error % pay_transaction)
894                    logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
895                    continue
896            tr_count += 1
897            if tr_count > 1000:
898                if len(no_import) > 0:
899                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
900                             '\n'.join(no_import) + '\n')
901                    no_import = []
902                em = '%d transactions committed\n' % (tr_count)
903                transaction.commit()
904                regs = []
905                logger.info(em)
906                total += tr_count
907                tr_count = 0
908        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
909                                                '\n'.join(no_import))
910        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
911    ###)
912
913    security.declareProtected(ModifyPortalContent,'mass_create_faculty') ###(
914    def mass_create_faculty(self,mapping):
915        "create a faculty"
916        logger = logging.getLogger('WAeUPTool.mass_create_faculty')
917        academics_folder = self.portal_url.getPortalObject().campus.academics
918        fid = mapping['code']
919        if getattr(academics_folder,fid,None) is not None:
920            return '', "Faculty with ID: %s exists" % fid
921        logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
922        academics_folder.invokeFactory('Faculty', fid)
923        f = getattr(academics_folder,fid,None)
924        f.getContent().edit(mapping=mapping)
925        return fid,''
926    ###)
927
928    security.declareProtected(ModifyPortalContent,'mass_edit_Faculty') ###(
929    def mass_edit_faculty(self,mapping):
930        "edit a faculty"
931        logger = logging.getLogger('WAeUPTool.mass_edit_faculty')
932        academics_folder = self.portal_url.getPortalObject().campus.academics
933        fid = mapping['code']
934        f = getattr(academics_folder,fid,None)
935        if f is None:
936            return '', "Faculty with ID: %s does not exist" % fid
937        logger.info('Editing Faculty %(code)s, %(title)s' % mapping)
938        f.getContent().edit(mapping=mapping)
939        return fid,''
940    ###)
941
942    security.declareProtected(ModifyPortalContent,'mass_create_department') ###(
943    def mass_create_department(self,mapping):
944        "create a department in the correct faculty"
945        logger = logging.getLogger('WAeUPTool.mass_create_department')
946        fid = mapping['faculty_code']
947        if getattr(self,'_v_faculties',None) is None:
948            res = self.portal_catalog(portal_type = "Faculty")
949            self._v_faculties = {}
950            for f in res:
951                self._v_faculties[f.getId] = f.getObject()
952        f = self._v_faculties.get(fid,None)
953        if f is None:
954            return '', "No Faculty with ID: %s" % fid
955        else:
956            did = mapping.get('code')
957            d = getattr(f,did,None)
958            if d is None or d.portal_type == "Faculty":
959                logger.info('Creating Department %(code)s, %(title)s' % mapping)
960                f.invokeFactory('Department', did)
961                d = getattr(f,did)
962                d.invokeFactory('CoursesFolder','courses')
963                courses = getattr(d,'courses')
964                dict = {'Title': 'Courses'}
965                courses.getContent().edit(mapping=dict)
966                d.invokeFactory('CertificatesFolder','certificates')
967                certificates = getattr(d,'certificates')
968                dict = {'Title': 'Certificates'}
969                certificates.getContent().edit(mapping=dict)
970            d.getContent().edit(mapping=mapping)
971        return did,''
972    ###)
973
974    security.declareProtected(ModifyPortalContent,'mass_edit_department') ###(
975    def mass_edit_department(self,mapping):
976        "create a department in the correct faculty"
977        logger = logging.getLogger('WAeUPTool.mass_create_department')
978        academics_folder = self.portal_url.getPortalObject().campus.academics
979        fid = mapping['faculty_code']
980        did = mapping.get('code')
981        try:
982            d = getattr(getattr(academics_folder,fid),did,None)
983        except KeyError:
984            return '', "Department %s or Faculty %s wrong" % (did,fid)
985        else:
986            if d is None or d.portal_type == "Faculty":
987                logger.info('Editing Department %(code)s, %(title)s' % mapping)
988            d.getContent().edit(mapping=mapping)
989        return did,''
990    ###)
991
992    security.declareProtected(ModifyPortalContent,'mass_create_course') ###(
993    def mass_create_course(self,mapping):
994        if getattr(self,'_v_course_list',None) is None:
995            self._v_course_list = []
996        if getattr(self,'_v_departments',None) is None:
997            res = self.portal_catalog(portal_type = "Department")
998            self._v_departments = {}
999            for d in res:
1000                self._v_departments[d.getId] = d.getObject()
1001        did = mapping['department_code']
1002        d = self._v_departments.get(did,None)
1003        if d is None:
1004            return '', "No Department with ID: %s" % fid
1005        course_id = mapping.get('code')
1006        if course_id in self._v_course_list:
1007            return '', "Duplicate Department ID: %s" % did
1008        c = getattr(d,course_id,None)
1009        if c is not None:
1010            return '', "Duplicate Department ID: %s" % did
1011        d.invokeFactory('Course', course_id)
1012        self._v_course_list.append(course_id)
1013        c = getattr(d,course_id)
1014        c.getContent().edit(mapping=mapping)
1015        return course_id,''
1016    ###)
1017
1018    field2types_student = {   ###(
1019                      'StudentApplication':
1020                          {'id': 'application',
1021                           'title': 'Application Data',
1022                           'wf_transition_return': 'close',
1023                           'wf_transition_admit': 'remain',
1024                           'fields':
1025                             ('jamb_reg_no',
1026                              'entry_mode',
1027                              'entry_session',
1028                              'jamb_score',
1029                              'app_email',
1030                              'jamb_age',
1031                              'jamb_state',
1032                              'jamb_lga',
1033                              )
1034                              },
1035                      #'StudentPume':
1036                      #    {'id': 'pume',
1037                      #     'title': 'Pume Data',
1038                      #     'wf_transition_return': 'close',
1039                      #     'wf_transition_admit': 'close',
1040                      #     'fields':
1041                      #       ('pume_score',
1042                      #        )
1043                      #        },
1044                      'StudentClearance':
1045                          {'id': 'clearance',
1046                           'title': 'Clearance/Eligibility Record',
1047                           'wf_transition_return': 'close',
1048                           'wf_transition_admit': 'remain',
1049                           'fields':
1050                             ('matric_no',
1051                              'nationality',
1052                              'lga',
1053                              'birthday',
1054                              )
1055                              },
1056                         'StudentPersonal':
1057                          {'id': 'personal',
1058                           'title': 'Personal Data',
1059                           'wf_transition_return': 'open',
1060                           'wf_transition_admit': 'remain',
1061                           'fields':
1062                             ('firstname',
1063                              'middlename',
1064                              'lastname',
1065                              'sex',
1066                              'email',
1067                              'phone',
1068                              'perm_address',
1069                              )
1070                              },
1071                         'StudentStudyCourse':
1072                          {'id': 'study_course',
1073                           'title': 'Study Course',
1074                           'wf_transition_return': 'open',
1075                           'wf_transition_admit': 'remain',
1076                           'fields':
1077                             ('study_course',
1078                              'current_level',
1079                              'current_session',
1080                              'current_mode',
1081                              'current_verdict',
1082                              )
1083                              },
1084
1085                         'PaymentsFolder':
1086                          {'id': 'payments',
1087                           'title': 'Payments',
1088                           'wf_transition_return': 'open',
1089                           'wf_transition_admit': 'open',
1090                           'fields':
1091                             ()
1092                              },
1093                         }
1094    ###)
1095
1096    security.declareProtected(ModifyPortalContent,'mass_create_student') ###(
1097    def mass_create_student(self,mapping):
1098        "create a students record due import"
1099        logger = logging.getLogger('WAeUPTool.mass_create_student')
1100        students_folder = self.portal_url.getPortalObject().campus.students
1101        jamb_reg_no = mapping.get('jamb_reg_no',None)
1102        if jamb_reg_no:
1103            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1104            if res:
1105                return '',"jamb_reg_no exists"
1106        matric_no = mapping.get('matric_no',None)
1107        if matric_no:
1108            res = self.students_catalog(matric_no = matric_no)
1109            if res:
1110                return '',"matric_no exists"
1111        sid = self.waeup_tool.generateStudentId('?')
1112        students_folder.invokeFactory('Student', sid)
1113        student_obj = getattr(students_folder,sid)
1114        f2t = self.field2types_student
1115        d = {}
1116        d['jamb_sex']  = 'M'
1117        if mapping.get('sex'):
1118            d['jamb_sex']  = 'F'
1119        transition = mapping.get('reg_transition','admit')
1120        if transition not in ('admit','return'):
1121            return '',"no valid transition provided"
1122        for pt in f2t.keys():
1123            student_obj.invokeFactory(pt,f2t[pt]['id'])
1124            sub_obj = getattr(student_obj,f2t[pt]['id'])
1125            sub_doc = sub_obj.getContent()
1126            d['Title'] = f2t[pt]['title']
1127            for field in f2t[pt]['fields']:
1128                d[field] = mapping.get(field,'')
1129            if pt == "StudyCourse":
1130                for von,zu in (('entry_mode','current_mode'),
1131                               ('entry_session','current_session')):
1132                    if mapping.get(zu,None) is None and mapping.get(von,None) is not None:
1133                        d[zu] = mapping[von]
1134            sub_doc.edit(mapping = d)
1135
1136            #import pdb;pdb.set_trace()
1137            new_state = f2t[pt]['wf_transition_%(transition)s' % vars()]
1138            if new_state != "remain":
1139                self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1140        self.portal_workflow.doActionFor(student_obj,transition)
1141        student_obj.manage_setLocalRoles(sid, ['Owner',])
1142        return sid,''
1143    ###)
1144
1145    security.declareProtected(ModifyPortalContent,'mass_edit_student') ###(
1146    def mass_edit_student(self,mapping):
1147        "edit a students record due import"
1148        logger = logging.getLogger('WAeUPTool.mass_edit_student')
1149        students_folder = self.portal_url.getPortalObject().campus.students
1150        sid = mapping.get('id',None)
1151        jamb_reg_no = mapping.get('jamb_reg_no',None)
1152        matric_no = mapping.get('matric_no',None)
1153        editable_keys = mapping.keys()
1154        if sid:
1155            res = self.students_catalog(id = sid)
1156            if not res:
1157                return '',"no student with id %s" % sid
1158            elif matric_no and res[0].matric_no and\
1159              matric_no != res[0].matric_no:
1160                return '%s' % res[0].id ,"student has no matric_no %s" % matric_no
1161            elif jamb_reg_no and res[0].jamb_reg_no and\
1162              jamb_reg_no != res[0].jamb_reg_no:
1163                return '%s' % res[0].id ,"student has no jamb_reg_no %s" % jamb_reg_no
1164        elif jamb_reg_no:
1165            res = self.students_catalog(jamb_reg_no = jamb_reg_no)
1166            if not res:
1167                return '',"no student with jamb_reg_no %s" % jamb_reg_no
1168            editable_keys.remove('jamb_reg_no')
1169        elif matric_no:
1170            res = self.students_catalog(matric_no = matric_no)
1171            if not res:
1172                return '',"no student with matric_no %s" % matric_no
1173            editable_keys.remove('matric_no')
1174
1175        # included only to change wf state from admitted to returning
1176        if res[0].review_state not in ('admitted','objection_raised'):
1177            return '%s' % res[0].id ,"student is not in state admitted or objection_raised"
1178        # end inclusion
1179
1180        sid = res[0].id
1181        student_obj = getattr(students_folder,sid)
1182        f2t = self.field2types_student
1183        d = {}
1184        #import pdb;pdb.set_trace()
1185        any_change = False
1186        for pt in f2t.keys():
1187            if pt == "student_application":
1188                d['jamb_sex']  = 'M'
1189                if mapping.get('sex'):
1190                    d['jamb_sex']  = 'F'
1191            intersect = set(f2t[pt]['fields']).intersection(set(editable_keys))
1192            if intersect:
1193                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1194                if sub_obj is None:
1195                    try:
1196                        student_obj.invokeFactory(pt,f2t[pt]['id'])
1197                    except:
1198                        continue
1199                    sub_obj = getattr(student_obj,f2t[pt]['id'])
1200                    d['Title'] = f2t[pt]['title']
1201                sub_doc = sub_obj.getContent()
1202                for field in intersect:
1203                    changed = False
1204                    if getattr(sub_doc,field) != mapping.get(field,''):
1205                        any_change = True
1206                        changed = True
1207                        d[field] = mapping.get(field,'')
1208                    if changed:
1209                        sub_doc.edit(mapping = d)
1210
1211
1212        # included only to change wf state from admitted to returning
1213            if res[0].review_state in ('admitted','objection_raised'):
1214                new_state = f2t[pt]['wf_transition_return']
1215                sub_obj = getattr(student_obj,f2t[pt]['id'],None)
1216                if sub_obj and new_state != "remain":
1217                    try:
1218                        self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj)
1219                    except:
1220                        #logger.info('%s, wf transition %s of %s failed' % (sid,new_state,sub_obj.id))
1221                        pass
1222        if res[0].review_state in ('admitted','objection_raised'):
1223            wfaction = 'return'
1224            try:
1225                self.portal_workflow.doActionFor(student_obj,wfaction)
1226                logger.info('%s, wf state changed' % sid)
1227                any_change = True
1228            except:
1229                logger.info('%s, wf transition failed, old state = %s' % (sid,res[0].review_state))
1230                pass
1231        # end inclusion
1232
1233
1234        if any_change:
1235            return sid,''
1236        else:
1237            return sid,'not modified'
1238    ###)
1239
1240    security.declareProtected(ModifyPortalContent,"importData")###(
1241    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
1242        """load data from CSV values"""
1243        import transaction
1244        import random
1245
1246        pm = self.portal_membership
1247        member = pm.getAuthenticatedMember()
1248
1249        logger = logging.getLogger('WAeUPTool.importData')
1250        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1251        students_folder = self.portal_url.getPortalObject().campus.students
1252        start = True
1253        tr_count = 1
1254        total_imported = 0
1255        total_not_imported = 0
1256        total = 0
1257        iname = "import_%s" % name
1258        stool = getToolByName(self, 'portal_schemas')
1259        ltool = getToolByName(self, 'portal_layouts')
1260        schema = stool._getOb(iname)
1261        if schema is None:
1262            em = 'No such schema %s' % iname
1263            logger.error('No such schema %s' % iname)
1264            return em
1265        layout = ltool._getOb(iname)
1266        if layout is None:
1267            em = 'No such layout %s' % iname
1268            logger.error(em)
1269            return em
1270        validators = {}
1271        for widget in layout.keys():
1272            validators[widget] = layout[widget].validate
1273        #import pdb;pdb.set_trace()
1274        if edit:
1275            importer_name = "mass_edit_%s" % name
1276        else:
1277            importer_name = "mass_create_%s" % name
1278        importer = getattr(self, '%s' % importer_name,None)
1279        if importer is None:
1280            em = 'No importer function %s' % importer_name
1281            logger.error(em)
1282            return em
1283        not_imported = []
1284        imported = []
1285        try:
1286            items = csv.DictReader(open("%s/import/%s.csv" % (i_home,filename),"rb"))
1287        except:
1288            em = 'Error reading %s.csv' % filename
1289            logger.error(em)
1290            return em
1291        for item in items:
1292            if start:
1293                start = False
1294                adapters = [MappingStorageAdapter(schema, item)]
1295                logger.info('%s starts import from %s.csv' % (member,filename))
1296                #import_keys = [k for k in item.keys() if not k.startswith('ignore')]
1297                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
1298                import_keys = [k for k in attrs if not k.startswith('ignore')]
1299                diff2schema = set(import_keys).difference(set(schema.keys()))
1300                diff2layout = set(import_keys).difference(set(layout.keys()))
1301                if diff2schema:
1302                    em = "not ignorable key(s) %s found in heading" % diff2schema
1303                    return em
1304                s = ','.join(['"%s"' % fn for fn in import_keys])
1305                open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(s + ',"Error"'+ '\n')
1306                s = '"id",' + s
1307                open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(s + '\n')
1308                format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
1309                format_error = format + ',"%(Error)s"'
1310                format = '"%(id)s",'+ format
1311
1312            dm = DataModel(item, adapters,context=self)
1313            ds = DataStructure(data=item,datamodel=dm)
1314            error_string = ""
1315            for k in import_keys:
1316                #import pdb;pdb.set_trace()
1317                if not validators[k](ds):
1318                    error_string += " %s : %s" % (k,
1319                                                  self.translation_service(ds.getError(k),
1320                                                                           ds.getErrorMapping(k)))
1321            if not error_string:
1322                item.update(dm)
1323                item['id'],error = importer(item)
1324                if error:
1325                    error_string += error
1326            if error_string:
1327                item['Error'] = error_string
1328                not_imported.append(format_error % item)
1329                total_not_imported += 1
1330            else:
1331                em = format % item
1332                imported.append(em)
1333                logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
1334                tr_count += 1
1335                total_imported += 1
1336            total += 1
1337            if total and not total % 100:
1338                transaction.commit()
1339                if len(not_imported) > 0:
1340                    open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1341                             '\n'.join(not_imported) + '\n')
1342                    not_imported = []
1343                if len(imported) > 0:
1344                    open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1345                             '\n'.join(imported) + '\n')
1346                    imported = []
1347                em = '%d transactions committed\n' % (tr_count)
1348                regs = []
1349                logger.info(em)
1350                tr_count = 0
1351        if len(imported) > 0:
1352            open("%s/import/%s_imported%s.csv" % (i_home,filename,current),"a").write(
1353                                                '\n'.join(imported))
1354        if len(not_imported) > 0:
1355            open("%s/import/%s_not_imported%s.csv" % (i_home,filename,current),"a").write(
1356                                                '\n'.join(not_imported))
1357        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
1358        return em
1359    ###)
1360
1361InitializeClass(WAeUPTool)
Note: See TracBrowser for help on using the repository browser.