source: main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py @ 7655

Last change on this file since 7655 was 7652, checked in by Henrik Bettermann, 13 years ago

Do not retrieve current principal in derived logging components. This is now done by the parent Logger class.
  • Property svn:keywords set to Id
File size: 23.4 KB
Line 
1## $Id: batching.py 7652 2012-02-15 10:51:53Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState
34from waeup.sirp.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory)
37from waeup.sirp.students.interfaces import (
38    IStudent, IStudentStudyCourse,
39    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
40    IStudentStudyLevel, ICourseTicket,
41    IStudentOnlinePayment)
42from waeup.sirp.students.workflow import  IMPORTABLE_STATES
43from waeup.sirp.utils.batching import BatchProcessor
44
45class StudentProcessor(BatchProcessor):
46    """A batch processor for IStudent objects.
47    """
48    grok.implements(IBatchProcessor)
49    grok.provides(IBatchProcessor)
50    grok.context(Interface)
51    util_name = 'studentimporter'
52    grok.name(util_name)
53
54    name = u'Student Importer'
55    iface = IStudent
56
57    location_fields = []
58    factory_name = 'waeup.Student'
59
60    mode = None
61
62    @property
63    def available_fields(self):
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'reg_state'] + getFields(
67                self.iface).keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if not 'reg_number' in headerfields and not 'student_id' \
71            in headerfields and not 'matric_number' in headerfields:
72            raise FatalCSVError(
73                "Need at least columns student_id or reg_number " +
74                "or matric_number for import!")
75        if mode == 'create':
76            for field in self.required_fields:
77                if not field in headerfields:
78                    raise FatalCSVError(
79                        "Need at least columns %s for import!" %
80                        ', '.join(["'%s'" % x for x in self.required_fields]))
81        # Check for fields to be ignored...
82        not_ignored_fields = [x for x in headerfields
83                              if not x.startswith('--')]
84        if len(set(not_ignored_fields)) < len(not_ignored_fields):
85            raise FatalCSVError(
86                "Double headers: each column name may only appear once.")
87        return True
88
89    def parentsExist(self, row, site):
90        return 'students' in site.keys()
91
92    def getLocator(self, row):
93        if row.get('student_id',None):
94            return 'student_id'
95        elif row.get('reg_number',None):
96            return 'reg_number'
97        elif row.get('matric_number',None):
98            return 'matric_number'
99        else:
100            return None
101
102    # The entry never exists in create mode.
103    def entryExists(self, row, site):
104        return self.getEntry(row, site) is not None
105
106    def getParent(self, row, site):
107        return site['students']
108
109    def getEntry(self, row, site):
110        if not 'students' in site.keys():
111            return None
112        if self.getLocator(row) == 'student_id':
113            if row['student_id'] in site['students']:
114                student = site['students'][row['student_id']]
115                return student
116        elif self.getLocator(row) == 'reg_number':
117            reg_number = row['reg_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(reg_number=(reg_number, reg_number)))
121            if results:
122                return results[0]
123        elif self.getLocator(row) == 'matric_number':
124            matric_number = row['matric_number']
125            cat = queryUtility(ICatalog, name='students_catalog')
126            results = list(
127                cat.searchResults(matric_number=(matric_number, matric_number)))
128            if results:
129                return results[0]
130        return None
131
132       
133    def addEntry(self, obj, row, site):
134        parent = self.getParent(row, site)
135        parent.addStudent(obj)
136        # In some tests we don't have a students container
137        try:
138            parent.logger.info('%s - Student record imported' % (
139                obj.student_id))
140            history = IObjectHistory(obj)
141            history.addMessage('Student record imported')
142        except (TypeError, AttributeError):
143            pass
144        return
145
146    def delEntry(self, row, site):
147        student = self.getEntry(row, site)
148        if student is not None:
149            parent = self.getParent(row, site)
150            del parent[student.student_id]
151        pass
152
153    def updateEntry(self, obj, row, site):
154        """Update obj to the values given in row.
155        """
156        # Remove student_id from row if empty
157        if row.has_key('student_id') and row['student_id'] is None:
158            row.pop('student_id')
159        for key, value in row.items():
160            # Set student password and all fields declared in interface.
161            if key == 'password' and value != '':
162                IUserAccount(obj).setPassword(value)
163            elif key == 'reg_state':
164                IWorkflowState(obj).setState(value)
165                msg = "State '%s' set" % value
166                history = IObjectHistory(obj)
167                history.addMessage(msg)
168            elif hasattr(obj, key):
169                setattr(obj, key, value)
170        return
171
172    def getMapping(self, path, headerfields, mode):
173        """Get a mapping from CSV file headerfields to actually used fieldnames.
174        """
175        result = dict()
176        reader = csv.reader(open(path, 'rb'))
177        raw_header = reader.next()
178        for num, field in enumerate(headerfields):
179            if field not in [
180                'student_id', 'reg_number', 'matric_number'] and mode == 'remove':
181                continue
182            if field == u'--IGNORE--':
183                # Skip ignored columns in failed and finished data files.
184                continue
185            result[raw_header[num]] = field
186        return result
187
188    def checkConversion(self, row, mode='create'):
189        """Validates all values in row.
190        """
191        iface = self.iface
192        if mode in ['update', 'remove']:
193            if self.getLocator(row) == 'reg_number':
194                iface = IStudentUpdateByRegNo
195            elif self.getLocator(row) == 'matric_number':
196                iface = IStudentUpdateByMatricNo
197        converter = IObjectConverter(iface)
198        errs, inv_errs, conv_dict =  converter.fromStringDict(
199            row, self.factory_name)
200        if row.has_key('reg_state') and \
201            not row['reg_state'] in IMPORTABLE_STATES:
202            if row['reg_state'] != '':
203                errs.append(('reg_state','not allowed'))
204            else:
205                errs.append(('reg_state','no value provided'))
206        return errs, inv_errs, conv_dict
207
208class StudentStudyCourseProcessor(BatchProcessor):
209    """A batch processor for IStudentStudyCourse objects.
210    """
211    grok.implements(IBatchProcessor)
212    grok.provides(IBatchProcessor)
213    grok.context(Interface)
214    util_name = 'studycourseupdater'
215    grok.name(util_name)
216
217    name = u'StudentStudyCourse Importer (update only)'
218    iface = IStudentStudyCourse
219    factory_name = 'waeup.StudentStudyCourse'
220
221    location_fields = []
222
223    mode = None
224
225    @property
226    def available_fields(self):
227        return sorted(list(set(
228            ['student_id','reg_number','matric_number'] + getFields(
229                self.iface).keys())))
230
231    def checkHeaders(self, headerfields, mode='ignore'):
232        if not 'reg_number' in headerfields and not 'student_id' \
233            in headerfields and not 'matric_number' in headerfields:
234            raise FatalCSVError(
235                "Need at least columns student_id " +
236                "or reg_number or matric_number for import!")
237        # Check for fields to be ignored...
238        not_ignored_fields = [x for x in headerfields
239                              if not x.startswith('--')]
240        if len(set(not_ignored_fields)) < len(not_ignored_fields):
241            raise FatalCSVError(
242                "Double headers: each column name may only appear once.")
243        return True
244
245    def getParent(self, row, site):
246        if not 'students' in site.keys():
247            return None
248        if 'student_id' in row.keys() and row['student_id']:
249            if row['student_id'] in site['students']:
250                student = site['students'][row['student_id']]
251                return student
252        elif 'reg_number' in row.keys() and row['reg_number']:
253            reg_number = row['reg_number']
254            cat = queryUtility(ICatalog, name='students_catalog')
255            results = list(
256                cat.searchResults(reg_number=(reg_number, reg_number)))
257            if results:
258                return results[0]
259        elif 'matric_number' in row.keys() and row['matric_number']:
260            matric_number = row['matric_number']
261            cat = queryUtility(ICatalog, name='students_catalog')
262            results = list(
263                cat.searchResults(matric_number=(matric_number, matric_number)))
264            if results:
265                return results[0]
266        return None
267
268    def parentsExist(self, row, site):
269        return self.getParent(row, site) is not None
270
271    def entryExists(self, row, site):
272        return self.getEntry(row, site) is not None
273
274    def getEntry(self, row, site):
275        student = self.getParent(row, site)
276        if student is None:
277            return None
278        return student.get('studycourse')
279
280    def updateEntry(self, obj, row, site):
281        """Update obj to the values given in row.
282        """
283        for key, value in row.items():
284            # Skip fields not declared in interface.
285            if hasattr(obj, key):
286                setattr(obj, key, value)
287        # Update the students_catalog
288        notify(grok.ObjectModifiedEvent(obj.__parent__))
289        return
290
291    def checkConversion(self, row, mode='ignore'):
292        """Validates all values in row.
293        """
294        converter = IObjectConverter(self.iface)
295        errs, inv_errs, conv_dict =  converter.fromStringDict(
296            row, self.factory_name)
297        # We have to check if current_level is in range of certificate.
298        # This is not done by the converter. This kind of conversion
299        # checking does only work if a combination of certificate and
300        # current_level is provided.
301        if conv_dict.has_key('certificate'):
302          certificate = conv_dict['certificate']
303          start_level = certificate.start_level
304          end_level = certificate.end_level
305          if conv_dict['current_level'] < start_level or \
306              conv_dict['current_level'] > end_level+120:
307              errs.append(('current_level','not in range'))
308        return errs, inv_errs, conv_dict
309
310class StudentStudyLevelProcessor(BatchProcessor):
311    """A batch processor for IStudentStudyLevel objects.
312    """
313    grok.implements(IBatchProcessor)
314    grok.provides(IBatchProcessor)
315    grok.context(Interface)
316    util_name = 'studylevelimporter'
317    grok.name(util_name)
318
319    name = u'StudentStudyLevel Importer'
320    iface = IStudentStudyLevel
321    factory_name = 'waeup.StudentStudyLevel'
322
323    location_fields = []
324
325    mode = None
326
327    @property
328    def available_fields(self):
329        return sorted(list(set(
330            ['student_id','reg_number','matric_number','level'] + getFields(
331                self.iface).keys())))
332
333    def checkHeaders(self, headerfields, mode='ignore'):
334        if not 'reg_number' in headerfields and not 'student_id' \
335            in headerfields and not 'matric_number' in headerfields:
336            raise FatalCSVError(
337                "Need at least columns student_id " +
338                "or reg_number or matric_number for import!")
339        if not 'level' in headerfields:
340            raise FatalCSVError(
341                "Need level for import!")
342        # Check for fields to be ignored...
343        not_ignored_fields = [x for x in headerfields
344                              if not x.startswith('--')]
345        if len(set(not_ignored_fields)) < len(not_ignored_fields):
346            raise FatalCSVError(
347                "Double headers: each column name may only appear once.")
348        return True
349
350    def getParent(self, row, site):
351        if not 'students' in site.keys():
352            return None
353        if 'student_id' in row.keys() and row['student_id']:
354            if row['student_id'] in site['students']:
355                student = site['students'][row['student_id']]
356                return student['studycourse']
357        elif 'reg_number' in row.keys() and row['reg_number']:
358            reg_number = row['reg_number']
359            cat = queryUtility(ICatalog, name='students_catalog')
360            results = list(
361                cat.searchResults(reg_number=(reg_number, reg_number)))
362            if results:
363                return results[0]['studycourse']
364        elif 'matric_number' in row.keys() and row['matric_number']:
365            matric_number = row['matric_number']
366            cat = queryUtility(ICatalog, name='students_catalog')
367            results = list(
368                cat.searchResults(matric_number=(matric_number, matric_number)))
369            if results:
370                return results[0]['studycourse']
371        return None
372
373    def parentsExist(self, row, site):
374        return self.getParent(row, site) is not None
375
376    def entryExists(self, row, site):
377        return self.getEntry(row, site) is not None
378
379    def getEntry(self, row, site):
380        studycourse = self.getParent(row, site)
381        if studycourse is None:
382            return None
383        return studycourse.get(row['level'])
384
385    def addEntry(self, obj, row, site):
386        parent = self.getParent(row, site)
387        obj.level = int(row['level'])
388        parent[row['level']] = obj
389        return
390
391    def checkConversion(self, row, mode='ignore'):
392        """Validates all values in row.
393        """
394        converter = IObjectConverter(self.iface)
395        errs, inv_errs, conv_dict =  converter.fromStringDict(
396            row, self.factory_name)
397        # We have to check if level is a valid integer.
398        # This is not done by the converter.
399        try:
400            level = int(row['level'])
401            if level not in range(0,700,10):
402                errs.append(('level','no valid integer'))
403        except ValueError:
404            errs.append(('level','no integer'))
405        return errs, inv_errs, conv_dict
406
407class CourseTicketProcessor(BatchProcessor):
408    """A batch processor for ICourseTicket objects.
409    """
410    grok.implements(IBatchProcessor)
411    grok.provides(IBatchProcessor)
412    grok.context(Interface)
413    util_name = 'courseticketimporter'
414    grok.name(util_name)
415
416    name = u'CourseTicket Importer'
417    iface = ICourseTicket
418    factory_name = 'waeup.CourseTicket'
419
420    location_fields = []
421
422    mode = None
423
424    @property
425    def available_fields(self):
426        return sorted(list(set(
427            ['student_id','reg_number','matric_number','level','code'] + getFields(
428                self.iface).keys())))
429
430    def checkHeaders(self, headerfields, mode='ignore'):
431        if not 'reg_number' in headerfields and not 'student_id' \
432            in headerfields and not 'matric_number' in headerfields:
433            raise FatalCSVError(
434                "Need at least columns student_id " +
435                "or reg_number or matric_number for import!")
436        if not 'level' in headerfields:
437            raise FatalCSVError(
438                "Need level for import!")
439        if not 'code' in headerfields:
440            raise FatalCSVError(
441                "Need code for import!")
442        # Check for fields to be ignored...
443        not_ignored_fields = [x for x in headerfields
444                              if not x.startswith('--')]
445        if len(set(not_ignored_fields)) < len(not_ignored_fields):
446            raise FatalCSVError(
447                "Double headers: each column name may only appear once.")
448        return True
449
450    def getParent(self, row, site):
451        if not 'students' in site.keys():
452            return None
453        if 'student_id' in row.keys() and row['student_id']:
454            if row['student_id'] in site['students']:
455                student = site['students'][row['student_id']]
456                return student['studycourse'].get(row['level'])
457        elif 'reg_number' in row.keys() and row['reg_number']:
458            reg_number = row['reg_number']
459            cat = queryUtility(ICatalog, name='students_catalog')
460            results = list(
461                cat.searchResults(reg_number=(reg_number, reg_number)))
462            if results:
463                return results[0]['studycourse'].get(row['level'])
464        elif 'matric_number' in row.keys() and row['matric_number']:
465            matric_number = row['matric_number']
466            cat = queryUtility(ICatalog, name='students_catalog')
467            results = list(
468                cat.searchResults(matric_number=(matric_number, matric_number)))
469            if results:
470                return results[0]['studycourse'].get(row['level'])
471        return None
472
473    def parentsExist(self, row, site):
474        return self.getParent(row, site) is not None
475
476    def entryExists(self, row, site):
477        return self.getEntry(row, site) is not None
478
479    def getEntry(self, row, site):
480        level = self.getParent(row, site)
481        if level is None:
482            return None
483        return level.get(row['code'])
484
485    def addEntry(self, obj, row, site):
486        parent = self.getParent(row, site)
487        catalog = getUtility(ICatalog, name='courses_catalog')
488        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
489        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
490        obj.dcode = entries[0].__parent__.__parent__.code
491        obj.title = entries[0].title
492        obj.credits = entries[0].credits
493        obj.passmark = entries[0].passmark
494        obj.semester = entries[0].semester
495        parent[row['code']] = obj
496        return
497
498    def checkConversion(self, row, mode='ignore'):
499        """Validates all values in row.
500        """
501        converter = IObjectConverter(self.iface)
502        errs, inv_errs, conv_dict =  converter.fromStringDict(
503            row, self.factory_name)
504        # We have to check if course really exists.
505        # This is not done by the converter.
506        catalog = getUtility(ICatalog, name='courses_catalog')
507        entries = catalog.searchResults(code=(row['code'],row['code']))
508        if len(entries) == 0:
509            errs.append(('code','non-existent'))
510            return errs, inv_errs, conv_dict
511        return errs, inv_errs, conv_dict
512
513class StudentOnlinePaymentProcessor(BatchProcessor):
514    """A batch processor for IStudentOnlinePayment objects.
515    """
516    grok.implements(IBatchProcessor)
517    grok.provides(IBatchProcessor)
518    grok.context(Interface)
519    util_name = 'paymentimporter'
520    grok.name(util_name)
521
522    name = u'Payment Importer'
523    iface = IStudentOnlinePayment
524    factory_name = 'waeup.StudentOnlinePayment'
525
526    location_fields = []
527
528    mode = None
529
530    @property
531    def available_fields(self):
532        return sorted(list(set(
533            ['student_id','reg_number','matric_number','p_id'] + getFields(
534                self.iface).keys())))
535
536    def checkHeaders(self, headerfields, mode='ignore'):
537        if not 'reg_number' in headerfields and not 'student_id' \
538            in headerfields and not 'matric_number' in headerfields:
539            raise FatalCSVError(
540                "Need at least columns student_id " +
541                "or reg_number or matric_number for import!")
542        if not 'p_id' in headerfields:
543            raise FatalCSVError(
544                "Need p_id for import!")
545        # Check for fields to be ignored...
546        not_ignored_fields = [x for x in headerfields
547                              if not x.startswith('--')]
548        if len(set(not_ignored_fields)) < len(not_ignored_fields):
549            raise FatalCSVError(
550                "Double headers: each column name may only appear once.")
551        return True
552
553    def getParent(self, row, site):
554        if not 'students' in site.keys():
555            return None
556        if 'student_id' in row.keys() and row['student_id']:
557            if row['student_id'] in site['students']:
558                student = site['students'][row['student_id']]
559                return student['payments']
560        elif 'reg_number' in row.keys() and row['reg_number']:
561            reg_number = row['reg_number']
562            cat = queryUtility(ICatalog, name='students_catalog')
563            results = list(
564                cat.searchResults(reg_number=(reg_number, reg_number)))
565            if results:
566                return results[0]['payments']
567        elif 'matric_number' in row.keys() and row['matric_number']:
568            matric_number = row['matric_number']
569            cat = queryUtility(ICatalog, name='students_catalog')
570            results = list(
571                cat.searchResults(matric_number=(matric_number, matric_number)))
572            if results:
573                return results[0]['payments']
574        return None
575
576    def parentsExist(self, row, site):
577        return self.getParent(row, site) is not None
578
579    def entryExists(self, row, site):
580        return self.getEntry(row, site) is not None
581
582    def getEntry(self, row, site):
583        payments = self.getParent(row, site)
584        if payments is None:
585            return None
586        # We can use the hash symbol at the end of p_id in import files
587        # to avoid annoying automatic number transformation
588        # by Excel or Calc
589        p_id = row['p_id'].strip('#')
590        if p_id.startswith('p'):
591            entry = payments.get(p_id)
592        else:
593            # For data migration from old SRP
594            entry = payments.get('p' + p_id[6:])
595        return entry
596
597    def addEntry(self, obj, row, site):
598        parent = self.getParent(row, site)
599        p_id = row['p_id'].strip('#')
600        if not p_id.startswith('p'):
601            # For data migration from old SRP
602            obj.p_id = 'p' + p_id[6:]
603            parent[obj.p_id] = obj
604        else:
605            parent[p_id] = obj
606        return
607
608    def checkConversion(self, row, mode='ignore'):
609        """Validates all values in row.
610        """
611        converter = IObjectConverter(self.iface)
612        errs, inv_errs, conv_dict =  converter.fromStringDict(
613            row, self.factory_name)
614        # We have to check p_id.
615        p_id = row['p_id'].strip('#')
616        if p_id.startswith('p'):
617            if not len(p_id) == 14:
618                errs.append(('p_id','invalid length'))
619                return errs, inv_errs, conv_dict
620        else:
621            if not len(p_id) == 19:
622                errs.append(('p_id','invalid length'))
623                return errs, inv_errs, conv_dict
624        return errs, inv_errs, conv_dict
Note: See TracBrowser for help on using the repository browser.