source: WAeUP_SRP/trunk/WAeUPTables.py @ 5582

Last change on this file since 5582 was 5581, checked in by Henrik Bettermann, 14 years ago

First version of Moodle authentication data export.
  • Property svn:keywords set to Id
File size: 73.8 KB
Line 
1#-*- mode: python; mode: fold -*-
2# (C) Copyright 2005 AixtraWare <http://aixtraware.de>
3# Author: Joachim Schmitz <js@aixtraware.de>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as published
7# by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
17# 02111-1307, USA.
18#
19# $Id: WAeUPTables.py 5581 2010-11-08 12:59:16Z henrik $
20
21from zope.interface import implements
22from Globals import InitializeClass
23from Products.ZCatalog.ZCatalog import ZCatalog
24from Products.ZCatalog.ProgressHandler import ZLogHandler
25from AccessControl import ClassSecurityInfo
26from Products.CMFCore.permissions import ModifyPortalContent
27from Products.CMFCore.utils import getToolByName
28from Products.CMFCore.CatalogTool import CatalogTool
29from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter
30from Products.CPSSchemas.DataStructure import DataStructure
31from Products.CPSSchemas.DataModel import DataModel
32from Products.AdvancedQuery import Eq, Between, Le,In
33import urllib
34import DateTime,time
35import csv,re,os
36import logging
37import Globals
38p_home = Globals.package_home(globals())
39i_home = Globals.INSTANCE_HOME
40
41ADDING_SHEDULED = "adding_sheduled"
42OBJECT_CREATED = "object_created"
43NOT_OCCUPIED = 'not_occupied'
44
45from interfaces import IWAeUPTable
46
47class AttributeHolder(object):
48    pass
49
50def dict2ob(dict):
51    ob = AttributeHolder()
52    for key, value in dict.items():
53        setattr(ob, key, value)
54    return ob
55
56class WAeUPTable(ZCatalog): ###(
57
58    implements(IWAeUPTable)
59    security = ClassSecurityInfo()
60    meta_type = None
61
62    def __init__(self,name=None):
63        if name ==  None:
64            name = self.name
65        ZCatalog.__init__(self,name)
66
67    def refreshCatalog(self, clear=0, pghandler=None): ###(
68        """ don't refresh for a normal table """
69
70        if self.REQUEST and self.REQUEST.RESPONSE:
71            self.REQUEST.RESPONSE.redirect(
72              URL1 +
73              '/manage_catalogAdvanced?manage_tabs_message=Catalog%20refresh%20not%20implemented')
74
75###)
76
77    def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None): ###(
78        """ clears the whole enchilada """
79
80        #if REQUEST and RESPONSE:
81        #    RESPONSE.redirect(
82        #      URL1 +
83        #      '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Clearing%20disabled')
84
85        self._catalog.clear()
86        if REQUEST and RESPONSE:
87            RESPONSE.redirect(
88              URL1 +
89              '/manage_catalogAdvanced?manage_tabs_message=Catalog%20cleared')
90
91###)
92
93    def record2dict(self,fields,record,index): ###(
94        d = {}
95        for key in fields:
96            v = getattr(record, key, None)
97            v_dump = v
98            if key == 'sex':
99                if v == True:
100                    v_dump = 'F'
101                elif v == False:
102                    v_dump = 'M'
103                d[key] = v_dump
104            elif v:
105                if index == 'translate':
106                    if key == 'lga':
107                        v_dump = self.portal_vocabularies.local_gov_areas.get(v)
108                        if not v_dump:
109                            v_dump = v
110                    elif key == 'aos':
111                        v_dump = self.portal_vocabularies.aos.get(v)
112                d[key] = v_dump
113            else:
114                d[key] = ''
115        return d
116
117###)
118
119    def addRecord(self, **data): ###(
120        # The uid is the same as "bed".
121        uid = data[self.key]
122        res = self.searchResults({"%s" % self.key : uid})
123        if len(res) > 0:
124            raise ValueError("More than one record with uid %s" % uid)
125        self.catalog_object(dict2ob(data), uid=uid)
126        return uid
127
128###)
129
130    def deleteRecord(self, uid):
131        self.uncatalog_object(uid)
132
133    def getRecordByKey(self,key):
134        if not key:
135            return None
136        res = self.evalAdvancedQuery(Eq(self.key,key))
137        if res:
138            return res[0]
139        return None
140
141    def searchAndSetRecord(self, **data):
142        raise NotImplemented
143
144    def modifyRecord(self, record=None, **data): ###(
145        #records = self.searchResults(uid=uid)
146        uid = data[self.key]
147        if record is None:
148            records = self.searchResults({"%s" % self.key : uid})
149            if len(records) > 1:
150                # Can not happen, but anyway...
151                raise ValueError("More than one record with uid %s" % uid)
152            if len(records) == 0:
153                raise KeyError("No record for uid %s" % uid)
154            record = records[0]
155        record_data = {}
156        for field in self.schema() + self.indexes():
157            record_data[field] = getattr(record, field)
158        # Add the updated data:
159        record_data.update(data)
160        self.catalog_object(dict2ob(record_data), uid)
161
162###)
163
164    def reindexIndex(self, name, REQUEST,pghandler=None): ###(
165        if isinstance(name, str):
166            name =  (name,)
167        paths = self._catalog.uids.items()
168        i = 0
169        #import pdb;pdb.set_trace()
170        for p,rid in paths:
171            i += 1
172            metadata = self.getMetadataForRID(rid)
173            record_data = {}
174            for field in name:
175                record_data[field] = metadata.get(field)
176            uid = metadata.get(self.key)
177            self.catalog_object(dict2ob(record_data), uid, idxs=name,
178                                update_metadata=0)
179
180###)
181
182    security.declareProtected(ModifyPortalContent,"exportAllRecords") ###(
183    def exportAllRecords(self):
184        "export a WAeUPTable"
185        #import pdb;pdb.set_trace()
186        fields = [field for field in self.schema()]
187        format = ','.join(['"%%(%s)s"' % fn for fn in fields])
188        csv = []
189        csv.append(','.join(['"%s"' % fn for fn in fields]))
190        for uid in self._catalog.uids:
191            records = self.searchResults({"%s" % self.key : uid})
192            if len(records) > 1:
193                # Can not happen, but anyway...
194                raise ValueError("More than one record with uid %s" % uid)
195            if len(records) == 0:
196                raise KeyError("No record for uid %s" % uid)
197            rec = records[0]
198            csv.append(format % rec)
199        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
200        open("%s/import/%s-%s.csv" % (i_home,self.getId(),current),"w+").write('\n'.join(csv))
201
202###)
203
204    security.declareProtected(ModifyPortalContent,"dumpAll")###(
205    def dumpAll(self,index=None,value=None):
206        """dump all data in the table to a csv"""
207        member = self.portal_membership.getAuthenticatedMember()
208        logger = logging.getLogger('WAeUPTables.WAeUPTable.dumpAll')
209        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
210        export_file = "%s/export/%s_%s.csv" % (i_home,self.__name__,current,)
211        res_list = []
212        lines = []
213        if hasattr(self,"export_keys"):
214            fields = self.export_keys
215        else:
216            fields = []
217            for f in self.schema():
218                fields.append(f)
219        headline = ','.join(fields)
220        out = open(export_file,"wb")
221        out.write(headline +'\n')
222        out.close()
223        out = open(export_file,"a")
224        csv_writer = csv.DictWriter(out,fields,)
225        if index is not None and value is not None:
226            records = self.evalAdvancedQuery(Eq(index,value))
227        else:
228            records = self()
229        nr2export = len(records)
230        logger.info('%s starts dumping, %s records to export' % (member,nr2export))
231        chunk = 2000
232        total = 0
233        start = DateTime.DateTime().timeTime()
234        start_chunk = DateTime.DateTime().timeTime()
235        for record in records:
236            not_all = False
237            d = self.record2dict(fields,record,index)
238            lines.append(d)
239            total += 1
240            if total and not total % chunk or total == len(records):
241                csv_writer.writerows(lines)
242                anz = len(lines)
243                logger.info("wrote %(anz)d  total written %(total)d" % vars())
244                end_chunk = DateTime.DateTime().timeTime()
245                duration = end_chunk-start_chunk
246                per_record = duration/anz
247                till_now = end_chunk - start
248                avarage_per_record = till_now/total
249                estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
250                estimated_end = estimated_end.strftime("%H:%M:%S")
251                logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
252                start_chunk = DateTime.DateTime().timeTime()
253                lines = []
254        end = DateTime.DateTime().timeTime()
255        logger.info('total time %6.2f m' % ((end-start)/60))
256        import os
257        filename, extension = os.path.splitext(export_file)
258        from subprocess import call
259        msg = "wrote %(total)d records to %(export_file)s" % vars()
260        #try:
261        #    retcode = call('gzip %s' % (export_file),shell=True)
262        #    if retcode == 0:
263        #        msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
264        #except OSError, e:
265        #    retcode = -99
266        #    logger.info("zip failed with %s" % e)
267        logger.info(msg)
268        args = {'portal_status_message': msg}
269        #url = self.REQUEST.get('URL1') + '?' + urlencode(args)
270        url = self.REQUEST.get('URL2')
271        return 'ready'
272        #return self.REQUEST.RESPONSE.redirect(url)
273    ###)
274
275
276    security.declarePrivate("_import") ###(
277    def _import_new(self,csv_items,schema, layout, mode,logger):
278        "import data from csv.Dictreader Instance"
279        start = True
280        tr_count = 1
281        total_imported = 0
282        total_not_imported = 0
283        total = 0
284        iname =  "%s" % filename
285        not_imported = []
286        valid_records = []
287        invalid_records = []
288        duplicate_records = []
289        d = {}
290        d['mode'] = mode
291        d['valid_records'] = valid_records
292        d['invalid_records'] = invalid_records
293        d['invalid_records'] = duplicate_records
294        # d['import_fn'] = import_fn
295        # d['imported_fn'] = imported_fn
296        # d['not_imported_fn'] = not_imported_fn
297        validators = {}
298        for widget in layout.keys():
299            try:
300                validators[widget] = layout[widget].validate
301            except AttributeError:
302                logger.info('%s has no validate attribute' % widget)
303                return d
304        for item in csv_items:
305            if start:
306                start = False
307                logger.info('%s starts import from %s.csv' % (member,filename))
308                #import_keys = [k for k in item.keys() if not k.startswith('ignore')]
309                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
310                import_keys = [k for k in attrs if not (k.startswith('ignore') or k.isupper())]
311                diff2schema = set(import_keys).difference(set(schema.keys()))
312                diff2layout = set(import_keys).difference(set(layout.keys()))
313                if diff2layout:
314                    em = "not ignorable key(s) %s found in heading" % diff2layout
315                    logger.info(em)
316                    return d
317                # s = ','.join(['"%s"' % fn for fn in import_keys])
318                # open(not_imported_fn,"a").write(s + ',"Error"'+ '\n')
319                # #s = '"id",' + s
320                # open(imported_fn,"a").write(s + '\n')
321                # format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
322                # format_error = format + ',"%(Error)s"'
323                # #format = '"%(id)s",'+ format
324                adapters = [MappingStorageAdapter(schema, item)]
325            dm = DataModel(item, adapters,context=self)
326            ds = DataStructure(data=item,datamodel=dm)
327            error_string = ""
328            for k in import_keys:
329                if not validators[k](ds,mode=mode):
330                    error_string += " %s : %s" % (k,ds.getError(k))
331            if error_string:
332                item['Error'] = error_string
333                #invalid_records.append(dm)
334                invalid_records.append(item)
335                total_not_imported += 1
336            else:
337                em = format % item
338                valid_records.append(dm)
339                #logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
340                tr_count += 1
341                total_imported += 1
342            total += 1
343        # if len(imported) > 0:
344        #     open(imported_fn,"a").write('\n'.join(imported))
345        # if len(not_imported) > 0:
346        #     open(not_imported_fn,"a").write('\n'.join(not_imported))
347        #em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
348        d['imported'] = total_imported
349        d['not_imported'] = total_not_imported
350        d['valid_records'] = valid_records
351        d['invalid_records'] = invalid_records
352        return d
353    ###)
354
355    security.declarePublic("missingValue")###(
356    def missingValue(self):
357        from Missing import MV
358        return MV
359    ###)
360###)
361
362class AccommodationTable(WAeUPTable): ###(
363
364    meta_type = 'WAeUP Accommodation Tool'
365    name = "portal_accommodation"
366    key = "bed"
367    not_occupied = NOT_OCCUPIED
368    def __init__(self,name=None):
369        if name ==  None:
370            name = self.name
371        WAeUPTable.__init__(self, name)
372
373    def searchAndReserveBed(self, student_id,bed_type,random_order=False): ###(
374        logger = logging.getLogger('WAeUPTables.AccommodationTable.searchAndReserveBed')
375        records = self.evalAdvancedQuery(Eq('student',student_id))
376        if len(records) == 1:
377            #return -1,"Student with Id %s already booked bed %s." % (student_id,records[0].bed)
378            logger.info('%s found (reserved) bed %s' % (student_id,records[0].bed))
379            return -1,records[0].bed
380        elif len(records) > 1:
381            logger.info('%s found more than one (reserved) bed' % (student_id))
382            return -3,'more than one bed'
383        query = Eq('bed_type',bed_type) & Eq('student',NOT_OCCUPIED)
384        records = self.evalAdvancedQuery(query,sortSpecs=('sort_id','bed'))
385        if len(records) == 0:
386            logger.info('no bed %s available for %s' % (bed_type,student_id))
387            return -2,"no bed"
388        if random_order:
389            import random
390            bed_no = random.randint(0,len(records)-1)
391        else:
392            bed_no = 0
393        rec = records[bed_no]
394        self.modifyRecord(bed=rec.bed,student=student_id)
395        logger.info('%s booked bed %s' % (student_id,rec.bed))
396        return 1,rec.bed
397    ###)
398
399
400InitializeClass(AccommodationTable)
401
402###)
403
404class PinTable(WAeUPTable): ###(
405    from ZODB.POSException import ConflictError
406    security = ClassSecurityInfo()
407    meta_type = 'WAeUP Pin Tool'
408    name = "portal_pins"
409    key = 'pin'
410
411    def __init__(self,name=None):
412        if name ==  None:
413            name = self.name
414        WAeUPTable.__init__(self, name)
415
416    security.declareProtected(ModifyPortalContent,"dumpAll")###(
417    def dumpAll(self,include_unused=None,index=None):
418        """dump all data in the table to a csv"""
419        member = self.portal_membership.getAuthenticatedMember()
420        logger = logging.getLogger('WAeUPTables.PinTable.dumpAll')
421        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
422        export_file = "%s/export/%s_%s.csv" % (i_home,self.__name__,current,)
423        res_list = []
424        lines = []
425        if hasattr(self,"export_keys"):
426            fields = self.export_keys
427        else:
428            fields = []
429            for f in self.schema():
430                fields.append(f)
431        headline = ','.join(fields)
432        out = open(export_file,"wb")
433        out.write(headline +'\n')
434        out.close()
435        out = open(export_file,"a")
436        csv_writer = csv.DictWriter(out,fields,)
437        if include_unused is not None and str(member) not in ('admin'):
438            logger.info('%s tries to dump pintable with unused pins' % (member))
439            return
440        if include_unused is not None:
441            records = self()
442        else:
443            records = self.evalAdvancedQuery(~Eq('student',''))
444        nr2export = len(records)
445        logger.info('%s starts dumping, %s records to export' % (member,nr2export))
446        chunk = 2000
447        total = 0
448        start = DateTime.DateTime().timeTime()
449        start_chunk = DateTime.DateTime().timeTime()
450        for record in records:
451            not_all = False
452            d = self.record2dict(fields,record,index)
453            lines.append(d)
454            total += 1
455            if total and not total % chunk or total == len(records):
456                csv_writer.writerows(lines)
457                anz = len(lines)
458                logger.info("wrote %(anz)d  total written %(total)d" % vars())
459                end_chunk = DateTime.DateTime().timeTime()
460                duration = end_chunk-start_chunk
461                per_record = duration/anz
462                till_now = end_chunk - start
463                avarage_per_record = till_now/total
464                estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
465                estimated_end = estimated_end.strftime("%H:%M:%S")
466                logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
467                start_chunk = DateTime.DateTime().timeTime()
468                lines = []
469        end = DateTime.DateTime().timeTime()
470        logger.info('total time %6.2f m' % ((end-start)/60))
471        import os
472        filename, extension = os.path.splitext(export_file)
473        from subprocess import call
474        msg = "wrote %(total)d records to %(export_file)s" % vars()
475        #try:
476        #    retcode = call('gzip %s' % (export_file),shell=True)
477        #    if retcode == 0:
478        #        msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
479        #except OSError, e:
480        #    retcode = -99
481        #    logger.info("zip failed with %s" % e)
482        logger.info(msg)
483        args = {'portal_status_message': msg}
484        #url = self.REQUEST.get('URL1') + '?' + urlencode(args)
485        url = self.REQUEST.get('URL2')
486        return self.REQUEST.RESPONSE.redirect(url)
487    ###)
488
489
490
491    def searchAndSetRecord(self, uid, student_id,prefix):
492
493        # The following line must be activated after resetting the
494        # the portal_pins table. This is to avoid duplicate entries
495        # and disable duplicate payments.
496
497        #student_id = student_id.upper()
498
499        #records = self.searchResults(student = student_id)
500        #if len(records) > 0 and prefix in ('CLR','APP'):
501        #    for r in records:
502        #        if r.pin != uid and r.prefix_batch.startswith(prefix):
503        #            return -2
504        records = self.searchResults({"%s" % self.key : uid})
505        if len(records) > 1:
506            # Can not happen, but anyway...
507            raise ValueError("More than one record with uid %s" % uid)
508        if len(records) == 0:
509            return -1,None
510        record = records[0]
511        if record.student == "":
512            record_data = {}
513            for field in self.schema() + self.indexes():
514                record_data[field] = getattr(record, field)
515            # Add the updated data:
516            record_data['student'] = student_id
517            try:
518                self.catalog_object(dict2ob(record_data), uid)
519                return 1,record
520            except ConflictError:
521                return 2,record
522        if record.student.upper() != student_id.upper():
523            return 0,record
524        if record.student.upper() == student_id.upper():
525            return 2,record
526        return -3,record
527InitializeClass(PinTable)
528###)
529
530class PumeResultsTable(WAeUPTable): ###(
531
532    meta_type = 'WAeUP PumeResults Tool'
533    name = "portal_pumeresults"
534    key = "jamb_reg_no"
535    def __init__(self,name=None):
536        if name ==  None:
537            name = self.name
538        WAeUPTable.__init__(self, name)
539
540
541InitializeClass(PumeResultsTable)
542
543###)
544
545class ApplicantsCatalog(WAeUPTable): ###(
546
547    meta_type = 'WAeUP Applicants Catalog'
548    name = "applicants_catalog"
549    key = "reg_no"
550    security = ClassSecurityInfo()
551    #export_keys = (
552    #               "reg_no",
553    #               "status",
554    #               "lastname",
555    #               "sex",
556    #               "date_of_birth",
557    #               "lga",
558    #               "email",
559    #               "phone",
560    #               "passport",
561    #               "entry_mode",
562    #               "pin",
563    #               "screening_type",
564    #               "registration_date",
565    #               "testdate",
566    #               "application_date",
567    #               "screening_date",
568    #               "faculty",
569    #               "department",
570    #               "course1",
571    #               "course2",
572    #               "course3",
573    #               "eng_score",
574    #               "subj1",
575    #               "subj1score",
576    #               "subj2",
577    #               "subj2score",
578    #               "subj3",
579    #               "subj3score",
580    #               "aggregate",
581    #               "course_admitted",
582    #               )
583
584    def __init__(self,name=None):
585        if name ==  None:
586            name = self.name
587        WAeUPTable.__init__(self, name)
588
589    security.declareProtected(ModifyPortalContent,"new_importCSV")###(
590    def new_importCSV(self,filename="JAMB_data",
591                  schema_id="application",
592                  layout_id="import_application",
593                  mode='add'):
594        """ import JAMB data """
595        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
596        pm = self.portal_membership
597        member = pm.getAuthenticatedMember()
598        logger = logging.getLogger('WAeUPTables.ApplicantsCatalog.importCSV')
599        lock_fn = "%s/import/%s_import_lock" % (i_home,filename)
600        import_fn = "%s/import/%s.csv" % (i_home,filename)
601        if mode not in ('add','edit'):
602            logger.info("invalid mode: %s" % mode)
603        if os.path.exists(lock_fn):
604            logger.info("import of %(import_fn)s already in progress" % vars())
605            return
606        lock_file = open(lock_fn,"w")
607        lock_file.write("%(current)s \n" % vars())
608        lock_file.close()
609        invalid_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
610        duplicate_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
611        stool = getToolByName(self, 'portal_schemas')
612        ltool = getToolByName(self, 'portal_layouts')
613        schema = stool._getOb(schema_id)
614        if schema is None:
615            em = 'No such schema %s' % schema_id
616            logger.error(em)
617            return
618        for postfix in ('_import',''):
619            layout_name = "%(layout_id)s%(postfix)s" % vars()
620            if hasattr(ltool,layout_name):
621                break
622        layout = ltool._getOb(layout_name)
623        if layout is None:
624            em = 'No such layout %s' % layout_id
625            logger.error(em)
626            return
627        try:
628            csv_file = csv.DictReader(open(import_fn,"rb"))
629        except:
630            em = 'Error reading %s.csv' % filename
631            logger.error(em)
632            return
633        d = self._import_new(csv_items,schema,layout,mode,logger)
634        imported = []
635        edited = []
636        duplicates = []
637        not_found = []
638        if len(d['valid_records']) > 0:
639            for record in d['valid_records']:
640                #import pdb;pdb.set_trace()
641                if mode == "add":
642                    try:
643                        self.addRecord(**dict(record.items()))
644                        imported.append(**dict(record.items()))
645                        logger.info("added %s" % record.items())
646                    except ValueError:
647                        dupplicate.append(**dict(record.items()))
648                        logger.info("duplicate %s" % record.items())
649                elif mode == "edit":
650                    try:
651                        self.modifyRecord(**dict(record.items()))
652                        edited.append(**dict(record.items()))
653                        logger.info("edited %s" % record.items())
654                    except KeyError:
655                        not_found.append(**dict(record.items()))
656                        logger.info("not found %s" % record.items())
657        invalid = d['invalid_records']
658        for itype in ("imported","edited","not_found","duplicate","invalid"):
659            outlist = locals[itype]
660            if len(outlist):
661                d = {}
662                for k in outlist[0].keys():
663                    d[k] = k
664                outlist[0] = d
665                outfile = open("file_name_%s" % itype,'w')
666                csv.DictWriter(outfile,outlist[0].keys()).writerows(outlist)
667                logger.info("wrote %(itype)s records to %(, written to %(not_imported_fn)s" % d)
668###)
669
670    security.declareProtected(ModifyPortalContent,"importCSV")###(
671    def importCSV(self,filename="JAMB_data",
672                  schema_id="application",
673                  layout_id="application_pce",
674                  mode='add'):
675        """ import JAMB data """
676        stool = getToolByName(self, 'portal_schemas')
677        ltool = getToolByName(self, 'portal_layouts')
678        schema = stool._getOb(schema_id)
679        if schema is None:
680            em = 'No such schema %s' % schema_id
681            logger.error(em)
682            return
683        layout = ltool._getOb(layout_id)
684        if layout is None:
685            em = 'No such layout %s' % layout_id
686            logger.error(em)
687            return
688        logger = logging.getLogger('WAeUPTables.ApplicantsCatalog.importCSV')
689        d = self._import_old(filename,schema,layout,mode,logger)
690        if len(d['valid_records']) > 0:
691            for record in d['valid_records']:
692                #import pdb;pdb.set_trace()
693                if mode == "add":
694                    self.addRecord(**dict(record.items()))
695                    logger.info("added %s" % record.items())
696                elif mode == "edit":
697                    self.modifyRecord(**dict(record.items()))
698                    logger.info("edited %s" % record.items())
699                else:
700                    logger.info("invalid mode: %s" % mode)
701        logger.info("%(mode)sed %(imported)d records, invalid written to %(not_imported_fn)s" % d)
702    ###)
703
704InitializeClass(ApplicantsCatalog)
705
706###)
707
708class StudentsCatalog(WAeUPTable): ###(
709    security = ClassSecurityInfo()
710
711    meta_type = 'WAeUP Students Catalog'
712    name = "students_catalog"
713    key = "id"
714    affected_types = {   ###(
715                      'StudentApplication':
716                      {'id': 'application',
717                       'fields':
718                       ('jamb_reg_no',
719                        'entry_mode',
720                        #'entry_level',
721                        'entry_session',
722                       )
723                      },
724                      'StudentClearance':
725                      {'id': 'clearance',
726                       'fields':
727                       ('matric_no',
728                        'lga',
729                        'date_of_birth',  # birthday
730                       )
731                      },
732                      'StudentPersonal':
733                      {'id': 'personal',
734                       'fields':
735                       ('name',
736                        'sex',
737                        'perm_address',
738                        'email',
739                        'phone',
740                        'marit_stat',
741                        'firstname',
742                        'middlename',
743                        'lastname',
744                       )
745                      },
746                      'StudentStudyCourse':
747                      {'id': 'study_course',
748                       'fields':
749                       ('course', # study_course
750                        'faculty', # from certificate
751                        'department', # from certificate
752                        'end_level', # from certificate
753                        'level', # current_level
754                        'mode',  # from certificate
755                        'session', # current_session
756                        'verdict', # current_verdict
757                       )
758                      },
759                     }
760    ###)
761
762    def __init__(self,name=None):
763        if name ==  None:
764            name = self.name
765        WAeUPTable.__init__(self, name)
766        return
767       
768    security.declareProtected(ModifyPortalContent,"exportMoodleData") ###(
769    def exportMoodleData(self):
770        "export a WAeUPTable"
771        member = self.portal_membership.getAuthenticatedMember()
772        logger = logging.getLogger('WAeUPTables.StudentsCatalog.exportMoodleData')
773        if str(member) not in ('admin'):
774            logger.info('%s tries to dump Moodle Data' % (member))
775            return       
776        fields = [field for field in self.schema()]
777        export_fields = ['username','firstname','lastname','email','current_session','reg_state','password']
778        format = ','.join(['"%%(%s)s"' % fn for fn in export_fields])
779        csv = []
780        csv.append(','.join(['"%s"' % fn for fn in export_fields]))
781        #import pdb;pdb.set_trace()
782        for uid in self._catalog.uids:
783            records = self.searchResults({"%s" % self.key : uid})
784            if len(records) > 1:
785                # Can not happen, but anyway...
786                raise ValueError("More than one record with uid %s" % uid)
787            if len(records) == 0:
788                raise KeyError("No record for uid %s" % uid)
789            rec = self.record2dict(fields,records[0],None)           
790            rec['firstname'] = rec['firstname'] + rec['middlename']
791            rec.update({'reg_state':rec['review_state']})
792            rec['current_session'] = rec['session']
793            rec['username'] = rec['id']
794            rec['password'] = self.waeup_tool.getCredential(rec['id'])
795            csv.append(format % rec)
796        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
797        open("%s/export/moodle-%s.csv" % (i_home,current),"w+").write('\n'.join(csv))       
798        return 'ready'
799
800    def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
801        """ clears the whole enchilada """
802        self._catalog.clear()
803
804        if REQUEST and RESPONSE:
805            RESPONSE.redirect(
806              URL1 +
807              '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
808
809    def manage_catalogReindex(self, REQUEST, RESPONSE, URL1): ###(
810        """ clear the catalog, then re-index everything """
811
812        elapse = time.time()
813        c_elapse = time.clock()
814
815        pgthreshold = self._getProgressThreshold()
816        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
817        self.refreshCatalog(clear=1, pghandler=handler)
818
819        elapse = time.time() - elapse
820        c_elapse = time.clock() - c_elapse
821
822        RESPONSE.redirect(
823            URL1 +
824            '/manage_catalogAdvanced?manage_tabs_message=' +
825            urllib.quote('Catalog Updated \n'
826                         'Total time: %s\n'
827                         'Total CPU time: %s' % (`elapse`, `c_elapse`)))
828    ###)
829
830    def fill_certificates_dict(self): ###(
831        "return certificate data in  dict"
832        certificates_brains = self.portal_catalog(portal_type ='Certificate')
833        d = {}
834        for cb in certificates_brains:
835            certificate_doc = cb.getObject().getContent()
836            cb_path = cb.getPath().split('/')
837            ld = {}
838            ld['faculty'] = cb_path[-4]
839            ld['department'] = cb_path[-3]
840            ld['end_level'] = getattr(certificate_doc,'end_level','999')
841            ld['study_mode'] = getattr(certificate_doc,'study_mode','')
842            d[cb.getId] = ld
843        return d
844    ###)
845
846    def get_from_doc_department(self,doc,cached_data={}): ###(
847        "return the students department"
848        if doc is None:
849            return None
850        if hasattr(self,"_v_certificates") and self._v_certificates.has_key(doc.study_course):
851            return self._v_certificates[doc.study_course]['department']
852        certificate_res = self.portal_catalog(id = doc.study_course)
853        if len(certificate_res) != 1:
854            return None
855        return certificate_res[0].getPath().split('/')[-3]
856
857    def get_from_doc_faculty(self,doc,cached_data={}):
858        "return the students faculty"
859        if doc is None:
860            return None
861        if hasattr(self,"_v_certificates") and self._v_certificates.has_key(doc.study_course):
862            return self._v_certificates[doc.study_course]['faculty']
863        certificate_res = self.portal_catalog(id = doc.study_course)
864        if len(certificate_res) != 1:
865            return None
866        return certificate_res[0].getPath().split('/')[-4]
867
868    def get_from_doc_end_level(self,doc,cached_data={}):
869        "return the students end_level"
870        if doc is None:
871            return None
872        if hasattr(self,"_v_certificates") and self._v_certificates.has_key(doc.study_course):
873            return self._v_certificates[doc.study_course]['end_level']
874        certificate_res = self.portal_catalog(id = doc.study_course)
875        if len(certificate_res) != 1:
876            return None
877        return getattr(certificate_res[0].getObject().getContent(),'end_level','unknown')
878
879    def get_from_doc_level(self,doc,cached_data={}):
880        "return the students level"
881        if doc is None:
882            return None
883        return getattr(doc,'current_level',None)
884
885    #def get_from_doc_mode(self,doc,cached_data={}):
886    #    "return the students mode"
887    #    if doc is None:
888    #        return None
889    #    cm = getattr(doc,'current_mode',None)
890    #    return cm
891   
892    def get_from_doc_mode(self,doc,cached_data={}):
893        "return the students mode"
894        if doc is None:
895            return None
896        if hasattr(self,"_v_certificates") and self._v_certificates.has_key(doc.study_course):
897            return self._v_certificates[doc.study_course]['study_mode']
898        certificate_res = self.portal_catalog(id = doc.study_course)
899        if len(certificate_res) != 1:
900            return None
901        return getattr(certificate_res[0].getObject().getContent(),'study_mode','unknown')   
902
903
904    def get_from_doc_marit_stat(self,doc,cached_data={}):
905        "return the students marit_stat"
906        if doc is None:
907            return None
908        ms = getattr(doc,'marit_stat',None)
909        if ms == True:
910            return 'married'
911        elif ms == False:
912            return 'single'
913        else:
914            return 'undefined'
915           
916    def get_from_doc_date_of_birth(self,doc,cached_data={}):
917        "return the students date of birth"
918        if doc is None:
919            return None
920        return getattr(doc,'birthday',None)           
921
922    def get_from_doc_session(self,doc,cached_data={}):
923        "return the students current_session"
924        if doc is None:
925            return None
926        return getattr(doc,'current_session',None)
927
928    def get_from_doc_entry_session(self,doc,cached_data={}):
929        "return the students entry_session"
930        if doc is None:
931            return None
932        es = getattr(doc,'entry_session',None)
933        if es is not None and len(es) < 3:
934            return es
935        elif len(es) == 9:
936            return es[2:4]   
937        try:
938            digit = int(doc.jamb_reg_no[0])
939        except:
940            return "-1"
941        if digit < 9:
942            return "0%c" % doc.jamb_reg_no[0]
943        return "9%c" % doc.jamb_reg_no[0]
944
945    def get_from_doc_course(self,doc,cached_data={}):
946        "return the students study_course"
947        if doc is None:
948            return None
949        return getattr(doc,'study_course',None)
950
951    def get_from_doc_name(self,doc,cached_data={}):
952        "return the students name from the personal"
953        if doc is None:
954            return None
955        return "%s %s %s" % (doc.firstname,doc.middlename,doc.lastname)
956
957    def get_from_doc_verdict(self,doc,cached_data={}):
958        "return the students study_course"
959        if doc is None:
960            return None
961        return getattr(doc,'current_verdict',None)
962    ###)
963
964    def reindexIndex(self, name, REQUEST,pghandler=None): ###(
965        if not hasattr(self,'_v_certificates'):
966            self._v_certificates = self.fill_certificates_dict()
967        if isinstance(name, str):
968            name = (name,)
969        reindextypes = {}
970        reindex_special = []
971        for n in name:
972            if n in ("review_state"):
973                reindex_special.append(n)
974            else:
975                for pt in self.affected_types.keys():
976                    if n in self.affected_types[pt]['fields']:
977                        if reindextypes.has_key(pt):
978                            reindextypes[pt].append(n)
979                        else:
980                            reindextypes[pt]= [n]
981                        break
982        #cached_data = {}
983        #if set(name).intersection(set(('faculty','department','end_level','mode'))):
984        #    cached_data = self.fill_certificates_dict()
985        students = self.portal_catalog(portal_type="Student")
986        if hasattr(self,'portal_catalog_real'):
987            aq_portal = self.portal_catalog_real.evalAdvancedQuery
988        else:
989            aq_portal = self.portal_catalog.evalAdvancedQuery
990        num_objects = len(students)
991        if pghandler:
992            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
993        noattr = set(('StudentClearance','StudentPersonal')) & set(reindextypes.keys())
994        #import pdb;pdb.set_trace()
995        for i in xrange(num_objects):
996            if pghandler: pghandler.report(i)
997            student_brain = students[i]
998            student_object = student_brain.getObject()
999            data = {}
1000            modified = False
1001            sid = data['id'] = student_brain.getId
1002            if reindex_special and 'review_state' in reindex_special:
1003                modified = True
1004                data['review_state'] = student_object.portal_workflow.getInfoFor(student_object,'review_state',None)
1005            sub_objects = False
1006            for pt in reindextypes.keys():
1007                modified = True
1008                try:
1009                    doc = getattr(student_object,self.affected_types[pt]['id']).getContent()
1010                    sub_objects = True
1011                except:
1012                    continue
1013                for field in set(name).intersection(self.affected_types[pt]['fields']):
1014                    if hasattr(self,'get_from_doc_%s' % field):
1015                        data[field] = getattr(self,'get_from_doc_%s' % field)(doc)
1016                    else:
1017                        data[field] = getattr(doc,field)
1018            if not sub_objects and noattr:
1019                import_res = self.returning_import(id = sid)
1020                if not import_res:
1021                    continue
1022                import_record = import_res[0]
1023                data['matric_no'] = import_record.matric_no
1024                data['sex'] = import_record.Sex == 'F'
1025                data['name'] = "%s %s %s" % (import_record.Firstname,
1026                                             import_record.Middlename,
1027                                             import_record.Lastname)
1028                data['jamb_reg_no'] = import_record.Entryregno
1029            if modified:
1030                self.modifyRecord(**data)
1031        if pghandler: pghandler.finish()
1032    ###)
1033
1034    def refreshCatalog(self, clear=0, pghandler=None): ###(
1035        """ re-index everything we can find """
1036        students_folder = self.portal_url.getPortalObject().campus.students
1037        if clear:
1038            self._catalog.clear()
1039        students = self.portal_catalog(portal_type="Student")
1040        num_objects = len(students)
1041        #cached_data = self.fill_certificates_dict()
1042        if not hasattr(self,'_v_certificates'):
1043            self._v_certificates = self.fill_certificates_dict()
1044        if pghandler:
1045            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
1046        for i in xrange(num_objects):
1047            if pghandler: pghandler.report(i)
1048            student_brain = students[i]
1049            spath = student_brain.getPath()
1050            student_object = student_brain.getObject()
1051            data = {}
1052            sid = data['id'] = student_brain.getId
1053            #data['review_state'] = student_brain.review_state
1054            data['review_state'] = student_object.portal_workflow.getInfoFor(student_object,'review_state',None)
1055            sub_objects = False
1056            for pt in self.affected_types.keys():
1057                modified = True
1058                try:
1059                    doc = getattr(student_object,self.affected_types[pt]['id']).getContent()
1060                    sub_objects = True
1061                except:
1062                    #from pdb import set_trace;set_trace()
1063                    continue
1064                for field in self.affected_types[pt]['fields']:
1065                    if hasattr(self,'get_from_doc_%s' % field):
1066                        data[field] = getattr(self,'get_from_doc_%s' % field)(doc,
1067                                                                              cached_data=cached_data)
1068                    else:
1069                        data[field] = getattr(doc,field,None)
1070            if not sub_objects:
1071                import_res = self.returning_import(id = sid)
1072                if not import_res:
1073                    continue
1074                import_record = import_res[0]
1075                data['matric_no'] = import_record.matric_no
1076                data['sex'] = import_record.Sex == 'F'
1077                data['name'] = "%s %s %s" % (import_record.Firstname,
1078                                             import_record.Middlename,
1079                                             import_record.Lastname)
1080                data['jamb_reg_no'] = import_record.Entryregno
1081            self.addRecord(**data)
1082        if pghandler: pghandler.finish()
1083    ###)
1084
1085    security.declarePrivate('notify_event_listener') ###(
1086    def notify_event_listener(self,event_type,object,infos):
1087        "listen for events"
1088        if not infos.has_key('rpath'):
1089            return
1090        pt = getattr(object,'portal_type',None)
1091        mt = getattr(object,'meta_type',None)
1092        students_catalog = self
1093        data = {}
1094        if pt == 'Student' and\
1095           mt == 'CPS Proxy Folder' and\
1096           event_type.startswith('workflow'):
1097            data['id'] = object.getId()
1098            data['review_state'] = self.portal_workflow.getInfoFor(object,'review_state',None)
1099            students_catalog.modifyRecord(**data)
1100            return
1101        rpl = infos['rpath'].split('/')
1102        if pt == 'Student' and mt == 'CPS Proxy Folder':
1103            student_id = object.id
1104            if event_type == "sys_add_object":
1105                try:
1106                    self.addRecord(id = student_id)
1107                except ValueError:
1108                    pass
1109                return
1110            elif event_type == 'sys_del_object':
1111                self.deleteRecord(student_id)
1112        if pt not in self.affected_types.keys():
1113            return
1114        if event_type not in ('sys_modify_object'):
1115            return
1116        if mt == 'CPS Proxy Folder':
1117            return
1118        if not hasattr(self,'_v_certificates'):
1119            self._v_certificates = self.fill_certificates_dict()
1120        for field in self.affected_types[pt]['fields']:
1121            if hasattr(self,'get_from_doc_%s' % field):
1122                data[field] = getattr(self,'get_from_doc_%s' % field)(object)
1123            else:
1124                data[field] = getattr(object,field)
1125        data['id'] = rpl[2]
1126        self.modifyRecord(**data)
1127    ###)
1128
1129
1130InitializeClass(StudentsCatalog)
1131
1132###)
1133
1134class CertificatesCatalog(WAeUPTable): ###(
1135    security = ClassSecurityInfo()
1136
1137    meta_type = 'WAeUP Certificates Catalog'
1138    name =  "certificates_catalog"
1139    key = "code"
1140    def __init__(self,name=None):
1141        if name ==  None:
1142            name =  self.name
1143        WAeUPTable.__init__(self, name)
1144
1145    def manage_catalogReindex(self, REQUEST, RESPONSE, URL1): ###(
1146        """ clear the catalog, then re-index everything """
1147
1148        elapse = time.time()
1149        c_elapse = time.clock()
1150
1151        pgthreshold = self._getProgressThreshold()
1152        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
1153        self.refreshCatalog(clear=1, pghandler=handler)
1154
1155        elapse = time.time() - elapse
1156        c_elapse = time.clock() - c_elapse
1157
1158        RESPONSE.redirect(
1159            URL1 +
1160            '/manage_catalogAdvanced?manage_tabs_message=' +
1161            urllib.quote('Catalog Updated \n'
1162                         'Total time: %s\n'
1163                         'Total CPU time: %s' % (`elapse`, `c_elapse`)))
1164    ###)
1165
1166    def reindexIndex(self, name, REQUEST,pghandler=None): ###(
1167        if isinstance(name, str):
1168            name = (name,)
1169        certificates = self.portal_catalog(portal_type="Certificate")
1170        num_objects = len(certificates)
1171        if pghandler:
1172            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
1173        for i in xrange(num_objects):
1174            if pghandler: pghandler.report(i)
1175            certificate_brain = certificates[i]
1176            certificate_object = certificate_brain.getObject()
1177            pl = certificate_brain.getPath().split('/')
1178            data = {}
1179            cid = data[self.key] = certificate_brain.getId
1180            data['faculty'] = pl[-4]
1181            data['department'] = pl[-3]
1182            doc = certificate_object.getContent()
1183            for field in name:
1184                if field not in (self.key,'faculty','department'):
1185                    data[field] = getattr(doc,field)
1186            self.modifyRecord(**data)
1187        if pghandler: pghandler.finish()
1188    ###)
1189
1190    def refreshCatalog(self, clear=0, pghandler=None): ###(
1191        """ re-index everything we can find """
1192        if clear:
1193            self._catalog.clear()
1194        certificates = self.portal_catalog(portal_type="Certificate")
1195        num_objects = len(certificates)
1196        if pghandler:
1197            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
1198        #from pdb import set_trace;set_trace()
1199        for i in xrange(num_objects):
1200            if pghandler: pghandler.report(i)
1201            certificate_brain = certificates[i]
1202            certificate_doc = certificate_brain.getObject().getContent()
1203            pl = certificate_brain.getPath().split('/')
1204            data = {}
1205            for field in self.schema():
1206                data[field] = getattr(certificate_doc,field,None)
1207            data[self.key] = certificate_brain.getId
1208            ai = pl.index('academics')
1209            data['faculty'] = pl[ai +1]
1210            data['department'] = pl[ai +2]
1211            if clear:
1212                self.addRecord(**data)
1213            else:
1214                self.modifyRecord(**data)
1215        if pghandler: pghandler.finish()
1216    ###)
1217
1218    security.declarePrivate('notify_event_listener') ###(
1219    def notify_event_listener(self,event_type,object,infos):
1220        "listen for events"
1221        if not infos.has_key('rpath'):
1222            return
1223        pt = getattr(object,'portal_type',None)
1224        mt = getattr(object,'meta_type',None)
1225        if pt != 'Certificate':
1226            return
1227        data = {}
1228        rpl = infos['rpath'].split('/')
1229        if event_type not in ("sys_add_object","sys_modify_object","sys_del_object"):
1230            return
1231        certificate_id = object.getId()
1232        data[self.key] = certificate_id
1233        if event_type == "sys_add_object" and mt == 'CPS Proxy Folder':
1234            try:
1235                self.addRecord(**data)
1236            except ValueError:
1237                return
1238            certificate_id = object.getId()
1239            doc = object.getContent()
1240            if doc is None:
1241                return
1242            for field in self.schema():
1243                data[field] = getattr(doc,field,None)
1244            data[self.key] = certificate_id
1245            ai = rpl.index('academics')
1246            data['faculty'] = rpl[ai +1]
1247            data['department'] = rpl[ai +2]
1248            self.modifyRecord(**data)
1249            return
1250        if event_type == "sys_del_object":
1251            self.deleteRecord(certificate_id)
1252            return
1253        if event_type == "sys_modify_object" and mt == 'Certificate':
1254            #from pdb import set_trace;set_trace()
1255            for field in self.schema():
1256                data[field] = getattr(object,field,None)
1257            certificate_id = object.aq_parent.getId()
1258            data[self.key] = certificate_id
1259            ai = rpl.index('academics')
1260            data['faculty'] = rpl[ai +1]
1261            data['department'] = rpl[ai +2]
1262            self.modifyRecord(**data)
1263    ###)
1264
1265
1266InitializeClass(CertificatesCatalog)
1267###)
1268
1269class CoursesCatalog(WAeUPTable): ###(
1270    security = ClassSecurityInfo()
1271
1272    meta_type = 'WAeUP Courses Catalog'
1273    name =  "courses_catalog"
1274    key = "code"
1275    def __init__(self,name=None):
1276        if name ==  None:
1277            name =  self.name
1278        WAeUPTable.__init__(self, name)
1279
1280    def manage_catalogReindex(self, REQUEST, RESPONSE, URL1): ###(
1281        """ clear the catalog, then re-index everything """
1282
1283        elapse = time.time()
1284        c_elapse = time.clock()
1285
1286        pgthreshold = self._getProgressThreshold()
1287        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
1288        self.refreshCatalog(clear=1, pghandler=handler)
1289
1290        elapse = time.time() - elapse
1291        c_elapse = time.clock() - c_elapse
1292
1293        RESPONSE.redirect(
1294            URL1 +
1295            '/manage_catalogAdvanced?manage_tabs_message=' +
1296            urllib.quote('Catalog Updated \n'
1297                         'Total time: %s\n'
1298                         'Total CPU time: %s' % (`elapse`, `c_elapse`)))
1299    ###)
1300
1301    def reindexIndex(self, name, REQUEST,pghandler=None): ###(
1302        if isinstance(name, str):
1303            name = (name,)
1304        courses = self.portal_catalog(portal_type="Course")
1305        num_objects = len(courses)
1306        if pghandler:
1307            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
1308        for i in xrange(num_objects):
1309            if pghandler: pghandler.report(i)
1310            course_brain = courses[i]
1311            course_object = course_brain.getObject()
1312            pl = course_brain.getPath().split('/')
1313            data = {}
1314            cid = data[self.key] = course_brain.getId
1315            data['faculty'] = pl[-4]
1316            data['department'] = pl[-3]
1317            doc = course_object.getContent()
1318            for field in name:
1319                if field not in (self.key,'faculty','department'):
1320                    data[field] = getattr(doc,field)
1321            self.modifyRecord(**data)
1322        if pghandler: pghandler.finish()
1323    ###)
1324
1325    def refreshCatalog(self, clear=0, pghandler=None): ###(
1326        """ re-index everything we can find """
1327        if clear:
1328            self._catalog.clear()
1329        courses = self.portal_catalog(portal_type="Course")
1330        num_objects = len(courses)
1331        if pghandler:
1332            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
1333        #from pdb import set_trace;set_trace()
1334        for i in xrange(num_objects):
1335            if pghandler: pghandler.report(i)
1336            course_brain = courses[i]
1337            course_doc = course_brain.getObject().getContent()
1338            pl = course_brain.getPath().split('/')
1339            data = {}
1340            for field in self.schema():
1341                data[field] = getattr(course_doc,field,None)
1342            data[self.key] = course_brain.getId
1343            ai = pl.index('academics')
1344            data['faculty'] = pl[ai +1]
1345            data['department'] = pl[ai +2]
1346            if clear:
1347                self.addRecord(**data)
1348            else:
1349                self.modifyRecord(**data)
1350        if pghandler: pghandler.finish()
1351    ###)
1352
1353    security.declarePrivate('notify_event_listener') ###(
1354    def notify_event_listener(self,event_type,object,infos):
1355        "listen for events"
1356        if not infos.has_key('rpath'):
1357            return
1358        pt = getattr(object,'portal_type',None)
1359        mt = getattr(object,'meta_type',None)
1360        if pt != 'Course':
1361            return
1362        data = {}
1363        rpl = infos['rpath'].split('/')
1364        if event_type not in ("sys_add_object","sys_modify_object","sys_del_object"):
1365            return
1366        course_id = object.getId()
1367        data[self.key] = course_id
1368        if event_type == "sys_add_object" and mt == 'CPS Proxy Folder':
1369            try:
1370                self.addRecord(**data)
1371            except ValueError:
1372                return
1373            course_id = object.getId()
1374            doc = object.getContent()
1375            if doc is None:
1376                return
1377            for field in self.schema():
1378                data[field] = getattr(doc,field,None)
1379            data[self.key] = course_id
1380            ai = rpl.index('academics')
1381            data['faculty'] = rpl[ai +1]
1382            data['department'] = rpl[ai +2]
1383            self.modifyRecord(**data)
1384            return
1385        if event_type == "sys_del_object":
1386            self.deleteRecord(course_id)
1387            return
1388        if event_type == "sys_modify_object" and mt == 'Course':
1389            #from pdb import set_trace;set_trace()
1390            for field in self.schema():
1391                data[field] = getattr(object,field,None)
1392            course_id = object.aq_parent.getId()
1393            data[self.key] = course_id
1394            ai = rpl.index('academics')
1395            data['faculty'] = rpl[ai +1]
1396            data['department'] = rpl[ai +2]
1397            self.modifyRecord(**data)
1398    ###)
1399
1400
1401InitializeClass(CoursesCatalog)
1402###)
1403
1404class CourseResults(WAeUPTable): ###(
1405    security = ClassSecurityInfo()
1406
1407    meta_type = 'WAeUP Results Catalog'
1408    name = "course_results"
1409    key = "key" #student_id + level + course_id
1410    def __init__(self,name=None):
1411        if name ==  None:
1412            name = self.name
1413        WAeUPTable.__init__(self, name)
1414        self._queue = []
1415
1416    def addMultipleRecords(self, records): ###(
1417        """add many records"""
1418        existing_uids = []
1419        for data in records:
1420            uid = "%(student_id)s|%(level_id)s|%(course_id)s" % data
1421            data['%s' % self.key] = uid
1422            query = Eq(self.key, uid)
1423            res = self.course_results.evalAdvancedQuery(query)
1424            if len(res) > 0:
1425                rec = res[0]
1426                equal = True
1427                for attr in ('student_id','level_id','course_id'):
1428                    if getattr(rec,attr,'') != data[attr]:
1429                        equal = False
1430                        break
1431                if equal:
1432                    existing_uids += uid,
1433                    continue
1434            self.catalog_object(dict2ob(data), uid=uid)
1435        return existing_uids
1436    ###)
1437
1438    def deleteResultsHere(self,level_id,student_id): ###(
1439        query = Eq('student_id',student_id) & Eq('level_id', level_id)
1440        course_results = self.course_results.evalAdvancedQuery(query)
1441        #import pdb;pdb.set_trace()
1442        for result in course_results:
1443            self.deleteRecord(result.key)
1444    ###)
1445
1446    def moveResultsHere(self,level,student_id): ###(
1447        #import pdb;pdb.set_trace()
1448        level_id = level.getId()
1449        query = Eq('student_id',student_id) & Eq('level_id', level_id)
1450        course_results = self.course_results.evalAdvancedQuery(query)
1451        existing_courses = [cr.code for cr in course_results]
1452        to_delete = []
1453        for code,obj in level.objectItems():
1454            to_delete.append(code)
1455            carry_over = False
1456            if code.endswith('_co'):
1457                carry_over = True
1458                code  = code[:-3]
1459            if code in existing_courses:
1460                continue
1461            course_result_doc = obj.getContent()
1462            data = {}
1463            course_id = code
1464            for field in self.schema():
1465                data[field] = getattr(course_result_doc,field,'')
1466            data['key'] = key = "%(student_id)s|%(level_id)s|%(course_id)s" % vars()
1467            data['student_id'] = student_id
1468            data['level_id'] = level_id
1469            session_id = self.getLevelSession(level.getContent(),student_id,level_id)
1470            data['session_id'] = session_id
1471            #data['queue_status'] = OBJECT_CREATED
1472            data['code'] = course_id
1473            data['carry_over'] = carry_over
1474            self.catalog_object(dict2ob(data), uid=key)
1475        level.manage_delObjects(to_delete)
1476    ###)
1477
1478    def getCourses(self,student_id,level_id): ###(
1479        query = Eq('student_id',student_id) & Eq('level_id', level_id)
1480        course_results = self.course_results.evalAdvancedQuery(query)
1481        carry_overs = []
1482        normal1 = []
1483        normal2 = []
1484        normal3 = []
1485        total_credits = 0
1486        gpa = 0
1487        for brain in course_results:
1488            d = {}
1489
1490            for field in self.schema():
1491                d[field] = getattr(brain,field,None)
1492                if repr(d[field]) == 'Missing.Value':
1493                    d[field] = ''
1494            d['weight'] = ''
1495            d['grade'] = ''
1496            d['score'] = ''
1497
1498            if str(brain.credits).isdigit():
1499                credits = int(brain.credits)
1500                total_credits += credits
1501                score = getattr(brain,'score',0)
1502                if score and str(score).isdigit() and int(score) > 0:
1503                    score = int(score)
1504                    grade,weight = self.getGradesFromScore(score,'')
1505                    gpa += weight * credits
1506                    d['weight'] = weight
1507                    d['grade'] = grade
1508                    d['score'] = score
1509
1510            #if str(brain.ca1).isdigit() and str(brain.ca2).isdigit() and str(brain.exam).isdigit():
1511            #    d['score_calc'] = int(brain.ca1) + int(brain.ca2) + int(brain.exam)
1512            #else:
1513            #    d['score_calc'] = ''
1514            try:
1515                d['score_calc'] = float(brain.ca1) + float(brain.ca2) + float(brain.exam)
1516            except:
1517                d['score_calc'] = ''
1518
1519            if d['score_calc']:
1520                grade = self.getGradesFromScore(d['score_calc'],level_id)
1521                d['grade'] = grade
1522
1523            d['coe'] = ''
1524            if brain.core_or_elective:
1525                d['coe'] = 'Core'
1526            elif brain.core_or_elective == False:
1527                d['coe'] = 'Elective'
1528            id = code = d['id'] = brain.code
1529            d['code'] = code
1530            res = self.courses_catalog.evalAdvancedQuery(Eq('code',code))
1531            if res:
1532                course = res[0]
1533                d['title'] = course.title
1534                # The courses_catalog contains strings and integers in its semester field.
1535                # Maybe this can be fixed by reindexing the catalog. The schema of course says 'CPS Int Field'.
1536                d['semester'] = str(course.semester)
1537            else:
1538                d['title'] = "Course has been removed from course list"
1539                d['semester'] = ''
1540            if brain.carry_over:
1541                d['coe'] = 'CO'
1542                carry_overs.append(d)
1543            else:
1544                if d['semester'] == '1':
1545                    normal1.append(d)
1546
1547                elif d['semester'] == '2':
1548                    normal2.append(d)
1549                else:
1550                    normal3.append(d)
1551        #normal.sort(cmp=lambda x,y: cmp("%(semester)s%(code)s" % x,
1552        #                                "%(semester)s%(code)s" % y))
1553        carry_overs.sort(cmp=lambda x,y: cmp("%(semester)s%(code)s" % x,
1554                                             "%(semester)s%(code)s" % y))
1555        return total_credits,gpa,carry_overs,normal1,normal2,normal3
1556    ###)
1557
1558   
1559    # for transcript only
1560    def getAllCourses(self,student_id): ###(
1561        query = Eq('student_id',student_id)
1562        course_results = self.course_results.evalAdvancedQuery(query)
1563        courses = []
1564        for brain in course_results:
1565            d = {}
1566
1567            for field in self.schema():
1568                d[field] = getattr(brain,field,'')
1569
1570            d['weight'] = ''
1571            d['grade'] = ''
1572            d['score'] = ''
1573
1574            if str(brain.credits).isdigit():
1575                credits = int(brain.credits)
1576                score = getattr(brain,'score',0)
1577                if score and str(score).isdigit() and int(score) > 0:
1578                    score = int(score)
1579                    grade,weight = self.getGradesFromScore(score,'')
1580                    d['weight'] = weight
1581                    d['grade'] = grade
1582                    d['score'] = score
1583            d['coe'] = ''
1584            if brain.core_or_elective:
1585                d['coe'] = 'Core'
1586            elif brain.core_or_elective == False:
1587                d['coe'] = 'Elective'
1588            id = code = d['id'] = brain.code
1589            d['code'] = code
1590            res = self.courses_catalog.evalAdvancedQuery(Eq('code',code))
1591            if res:
1592                course = res[0]
1593                d['title'] = course.title
1594                # The courses_catalog contains strings and integers in its semester field.
1595                # Maybe this can be fixed by reindexing the catalog. The schema of course says 'CPS Int Field'.
1596                d['semester'] = str(course.semester)
1597            else:
1598                d['title'] = "Course has been removed from course list"
1599                d['semester'] = ''
1600            if brain.carry_over:
1601                d['coe'] = 'CO'
1602            courses.append(d)
1603        return courses
1604    ###)
1605   
1606    def getYearGroupAverage(self,session_id,level_id): ###(
1607        query = Eq('session_id',session_id) & Eq('level_id',level_id)
1608        course_results = self.course_results.evalAdvancedQuery(query)
1609        yga1 = 0
1610        yg1 = []
1611        counter1 = 0
1612        yga2 = 0
1613        yg2 = []
1614        counter2 = 0
1615        yga3 = 0
1616        yg3 = []
1617        counter3 = 0       
1618        #import pdb;pdb.set_trace()
1619        for brain in course_results:
1620            try:
1621                om = float(brain.ca1) + float(brain.ca2) + float(brain.exam)
1622                if not om > 0:
1623                    continue
1624                code = brain.code               
1625                res = self.courses_catalog.evalAdvancedQuery(Eq('code',code))
1626                if res:
1627                    course = res[0]
1628                    # The courses_catalog contains strings and integers in its semester field.
1629                    # Maybe this can be fixed by reindexing the catalog. The schema of course says 'CPS Int Field'.
1630                    semester = str(course.semester)
1631                else:
1632                    semester = ''
1633                if semester == '1':
1634                    counter1 += 1
1635                    yga1 += om
1636                    yg1.append(om)
1637                elif semester == '2':
1638                    counter2 += 1
1639                    yga2 += om     
1640                    yg2.append(om)   
1641                if semester == '3':
1642                    counter3 += 1
1643                    yga3 += om
1644                    yg3.append(om)
1645            except:
1646                continue               
1647        if counter1:
1648            yga1 /= counter1
1649            yga1 = '%.2f' % yga1   
1650        if counter2:
1651            yga2 /= counter2
1652            yga2 = '%.2f' % yga2   
1653        if counter3:
1654            yga3 /= counter3
1655            yga3 = '%.2f' % yga3                                   
1656        return yga1, yga2, yga3, counter1, counter2, counter3, yg1, yg2, yg3
1657    ###)
1658   
1659   
1660    #security.declarePublic("calculateCoursePosition")
1661    def calculateCoursePosition(self,session_id,level_id,code,score,semester=None):
1662        #"""calculate Course Position"""
1663        query = Eq('session_id',session_id) & Eq('level_id',level_id) & Eq('code',code)
1664        course_results = self.course_results.evalAdvancedQuery(query)
1665        ygc = []
1666        #import pdb;pdb.set_trace() 
1667        for brain in course_results:
1668            try:
1669                if not float(brain.ca1) + float(brain.ca2) + float(brain.exam) > 0:
1670                    continue
1671                #code = brain.code   
1672                if semester:
1673                    res = self.courses_catalog.evalAdvancedQuery(Eq('code',code))
1674                    if res:
1675                        course = res[0]
1676                        # The courses_catalog contains strings and integers in its semester field.
1677                        # Maybe this can be fixed by reindexing the catalog. The schema of course says 'CPS Int Field'.
1678                        semester_from_course = str(course.semester)
1679                    else:
1680                        continue
1681                    if semester != semester_from_course:
1682                        continue
1683                ygc.append(float(brain.ca1) + float(brain.ca2) + float(brain.exam))
1684            except:
1685                continue     
1686        ygc.sort(reverse=True)
1687        if not len(ygc):
1688            return 'no result'
1689        #import pdb;pdb.set_trace()       
1690        for pos in range(len(ygc)):
1691            if ygc[pos] <= float(score):
1692                break
1693        output = {}   
1694        output['pos'] =  '%d of %d' % (pos+1,len(ygc))
1695        output['ygc'] = ygc
1696        return output
1697       
1698    security.declareProtected(ModifyPortalContent,"calculateAllCoursePositions")
1699    def calculateAllCoursePositions(self,session_id=None):
1700        """calculate All Course Positions"""
1701        logger = logging.getLogger('WAeUPTables.CourseResults.calculateAllCoursePositions')
1702        member = self.portal_membership.getAuthenticatedMember()
1703        logger.info('%s starts recalculation of positions in session %s' % (member,session_id))
1704        if session_id:
1705            query = Eq('session_id',session_id)
1706        else:
1707            return 'no session_id provided'
1708        course_results = self.course_results.evalAdvancedQuery(query)
1709        for brain in course_results:
1710            try:
1711                if not float(brain.ca1) + float(brain.ca2) + float(brain.exam) > 0:
1712                    data = {}
1713                    data[self.key] = brain.key
1714                    data['pic'] = ''
1715                    self.modifyRecord(**data)                   
1716                    continue
1717                res = self.courses_catalog.evalAdvancedQuery(Eq('code',brain.code))
1718                if res:
1719                    course = res[0]
1720                    semester_from_course = str(course.semester)
1721                else:
1722                    continue                   
1723                score = float(brain.ca1) + float(brain.ca2) + float(brain.exam)
1724                pic = self.calculateCoursePosition(session_id,brain.level_id,brain.code,score,semester_from_course)['pos']
1725                data = {}
1726                data[self.key] = brain.key
1727                data['pic'] = pic
1728                self.modifyRecord(**data)
1729            except:
1730                data = {}
1731                data[self.key] = brain.key
1732                data['pic'] = ''
1733                self.modifyRecord(**data)
1734                continue       
1735        logger.info('recalculation finished')             
1736        return 'ready'   
1737   
1738    def exportRemoveAllCourses(self,student_id,export=False,remove=False): ###(
1739        ""
1740        query = Eq('student_id',student_id)
1741        cr_catalog = self.course_results
1742        course_results = cr_catalog.evalAdvancedQuery(query)
1743        courses = []
1744        fields = self.schema()
1745        format = '"%(' + ')s","%('.join(fields) + ')s"'
1746        for brain in course_results:
1747            d = {}
1748            for field in fields:
1749                d[field] = getattr(brain,field,'')
1750            courses.append(format % d)
1751               
1752        if export:
1753            export_file = "%s/export/course_results_removed.csv" % (i_home)
1754            if not os.path.exists(export_file): 
1755                file_handler = open(export_file,"a")
1756                headline = ','.join(fields)
1757                file_handler.write(headline +'\n')
1758            else:
1759                file_handler = open(export_file,"a")
1760            for line in courses:
1761                file_handler.write(line +'\n')
1762
1763        if remove:
1764            for brain in course_results:
1765                key = getattr(brain,'key','')
1766                cr_catalog.deleteRecord(key)
1767       
1768        return courses
1769    ###)   
1770   
1771   
1772
1773InitializeClass(CourseResults)
1774###)
1775
1776class OnlinePaymentsImport(WAeUPTable): ###(
1777
1778    meta_type = 'WAeUP Online Payment Transactions'
1779    name = "online_payments_import"
1780    key = "order_id"
1781    def __init__(self,name=None):
1782        if name ==  None:
1783            name = self.name
1784        WAeUPTable.__init__(self, name)
1785
1786
1787InitializeClass(OnlinePaymentsImport)
1788###)
1789
1790class ReturningImport(WAeUPTable): ###(
1791
1792    meta_type = 'Returning Import Table'
1793    name = "returning_import"
1794    key = "matric_no"
1795    def __init__(self,name=None):
1796        if name ==  None:
1797            name = self.name
1798        WAeUPTable.__init__(self, name)
1799
1800
1801InitializeClass(ReturningImport)
1802###)
1803
1804class ResultsImport(WAeUPTable): ###(
1805
1806    meta_type = 'Results Import Table'
1807    name = "results_import"
1808    key = "key"
1809    def __init__(self,name=None):
1810        if name ==  None:
1811            name = self.name
1812        WAeUPTable.__init__(self, name)
1813
1814
1815InitializeClass(ResultsImport)
1816
1817###)
1818
1819class PaymentsCatalog(WAeUPTable): ###(
1820    security = ClassSecurityInfo()
1821
1822    meta_type = 'WAeUP Payments Catalog'
1823    name = "payments_catalog"
1824    key = "order_id"
1825    def __init__(self,name=None):
1826        if name ==  None:
1827            name = self.name
1828        WAeUPTable.__init__(self, name)
1829
1830
1831    security.declarePrivate('notify_event_listener') ###(
1832    def notify_event_listener(self,event_type,object,infos):
1833        "listen for events"
1834        if not infos.has_key('rpath'):
1835            return
1836        pt = getattr(object,'portal_type',None)
1837        mt = getattr(object,'meta_type',None)
1838        data = {}
1839        if pt != 'Payment':
1840            return
1841        if event_type == 'sys_del_object' and mt == 'CPS Proxy Folder':
1842            self.deleteRecord(object.getContent().order_id)
1843        if mt == 'CPS Proxy Folder':
1844            return # is handled only for the real object
1845        if event_type not in ('sys_modify_object'):
1846            return
1847        for field in self.schema():
1848            data[field] = getattr(object,field,'')
1849        rpl = infos['rpath'].split('/')
1850        #import pdb;pdb.set_trace()
1851        student_id = rpl[-4]
1852        data['student_id'] = student_id
1853        modified = False
1854        try:
1855            self.modifyRecord(**data)
1856            modified = True
1857        except KeyError:
1858            #logger = logging.getLogger('WAeUPTables.PaymentsCatalog.%s' % self.__name__)
1859            #logger.info("could not modify entry for %(student_id)s with %(order_id)s" % data)
1860            pass
1861        if not modified:
1862            try:
1863                self.addRecord(**data)
1864            except:
1865                logger = logging.getLogger('WAeUPTables.PaymentsCatalog.notify_event_listener')
1866                logger.info("could not add or modify entry for %(student_id)s with %(order_id)s" % data)
1867        ###)
1868
1869
1870    def exportRemoveAllPayments(self,student_id,export=False,remove=False): ###(
1871        ""
1872        query = Eq('student_id',student_id)
1873        pm_catalog = self.payments_catalog
1874        payments = pm_catalog.evalAdvancedQuery(query)
1875        payments_dic = []
1876        fields = self.schema()
1877        format = '"%(' + ')s","%('.join(fields) + ')s"'
1878        for brain in payments:
1879            d = {}
1880            for field in fields:
1881                d[field] = getattr(brain,field,'')
1882            payments_dic.append(format % d)
1883               
1884        if export:
1885            export_file = "%s/export/payments_removed.csv" % (i_home)
1886            if not os.path.exists(export_file): 
1887                file_handler = open(export_file,"a")
1888                headline = ','.join(fields)
1889                file_handler.write(headline +'\n')
1890            else:
1891                file_handler = open(export_file,"a")
1892            for line in payments_dic:
1893                file_handler.write(line +'\n')
1894
1895        if remove:
1896            for brain in payments:
1897                order_id = getattr(brain,'order_id','')
1898                pm_catalog.deleteRecord(order_id)
1899       
1900        return payments_dic
1901    ###)   
1902
1903    security.declareProtected(ModifyPortalContent,"dumpPayments")###(
1904    def dumpPayments(self,session_id=''):
1905        """dump all valid payments and combine with student data """
1906       
1907        if not session_id:
1908            session_id  = self.getSessionId()[0]
1909        member = self.portal_membership.getAuthenticatedMember()
1910        logger = logging.getLogger('WAeUPTables.WAeUPTable.dumpPayments')
1911        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
1912        export_file = "%s/export/valid_payments%s_%s.csv" % (i_home,session_id,current,)       
1913       
1914        pm_catalog = self.payments_catalog
1915        query = Eq('status','paid') & Eq('type','online') & Eq('session_id',session_id)
1916        payments = pm_catalog.evalAdvancedQuery(query)
1917        payments_dic = []
1918        s_catalog = self.students_catalog
1919        fields_pm = pm_catalog.schema()
1920        fields_s = s_catalog.schema()
1921        fields =  fields_pm + fields_s
1922       
1923        format = '"%(' + ')s","%('.join(fields) + ')s"'
1924        #import pdb;pdb.set_trace()
1925        for brain in payments:
1926            d = {}
1927            for field in fields_pm:
1928                d[field] = getattr(brain,field,'')
1929           
1930            student_id = getattr(brain,'student_id','')
1931            query = Eq('id',student_id)   
1932            student = s_catalog.evalAdvancedQuery(query)
1933            if student:
1934                for field in fields_s:
1935                    d[field] = getattr(student[0],field,'')               
1936            payments_dic.append(format % d)     
1937           
1938        if not os.path.exists(export_file): 
1939            file_handler = open(export_file,"a")
1940            headline = ','.join(fields)
1941            file_handler.write(headline +'\n')
1942        else:
1943            file_handler = open(export_file,"a")
1944        for line in payments_dic:
1945            file_handler.write(line +'\n')     
1946       
1947        return 'ready'       
1948           
1949
1950InitializeClass(PaymentsCatalog)
1951
1952###)
1953
1954class RemovedStudentIds(WAeUPTable): ###(
1955
1956    meta_type = 'WAeUP Removed StudentIds'
1957    name = "removed_student_ids"
1958    key = "id"
1959    def __init__(self,name=None):
1960        if name ==  None:
1961            name = self.name
1962        WAeUPTable.__init__(self, name)
1963
1964
1965InitializeClass(RemovedStudentIds)
1966
1967###)
1968
1969# BBB:
1970AccomodationTable = AccommodationTable
Note: See TracBrowser for help on using the repository browser.