source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8786

Last change on this file since 8786 was 8736, checked in by Henrik Bettermann, 12 years ago

Replace getStudent method by a 'student' attribute.
  • Property svn:keywords set to Id
File size: 24.8 KB
Line 
1## $Id: batching.py 8736 2012-06-17 07:09:21Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58    iface_byregnumber = IStudentUpdateByRegNo
59    iface_bymatricnumber = IStudentUpdateByMatricNo
60
61    location_fields = []
62    factory_name = 'waeup.Student'
63
64    @property
65    def available_fields(self):
66        fields = getFields(self.iface)
67        return sorted(list(set(
68            ['student_id','reg_number','matric_number',
69            'password', 'state', 'transition'] + fields.keys())))
70
71    def checkHeaders(self, headerfields, mode='create'):
72        if 'state' in headerfields and 'transition' in headerfields:
73            raise FatalCSVError(
74                "State and transition can't be  imported at the same time!")
75        if not 'reg_number' in headerfields and not 'student_id' \
76            in headerfields and not 'matric_number' in headerfields:
77            raise FatalCSVError(
78                "Need at least columns student_id or reg_number " +
79                "or matric_number for import!")
80        if mode == 'create':
81            for field in self.required_fields:
82                if not field in headerfields:
83                    raise FatalCSVError(
84                        "Need at least columns %s for import!" %
85                        ', '.join(["'%s'" % x for x in self.required_fields]))
86        # Check for fields to be ignored...
87        not_ignored_fields = [x for x in headerfields
88                              if not x.startswith('--')]
89        if len(set(not_ignored_fields)) < len(not_ignored_fields):
90            raise FatalCSVError(
91                "Double headers: each column name may only appear once.")
92        return True
93
94    def parentsExist(self, row, site):
95        return 'students' in site.keys()
96
97    def getLocator(self, row):
98        if row.get('student_id',None) not in (None, IGNORE_MARKER):
99            return 'student_id'
100        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
101            return 'reg_number'
102        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
103            return 'matric_number'
104        else:
105            return None
106
107    # The entry never exists in create mode.
108    def entryExists(self, row, site):
109        return self.getEntry(row, site) is not None
110
111    def getParent(self, row, site):
112        return site['students']
113
114    def getEntry(self, row, site):
115        if not 'students' in site.keys():
116            return None
117        if self.getLocator(row) == 'student_id':
118            if row['student_id'] in site['students']:
119                student = site['students'][row['student_id']]
120                return student
121        elif self.getLocator(row) == 'reg_number':
122            reg_number = row['reg_number']
123            cat = queryUtility(ICatalog, name='students_catalog')
124            results = list(
125                cat.searchResults(reg_number=(reg_number, reg_number)))
126            if results:
127                return results[0]
128        elif self.getLocator(row) == 'matric_number':
129            matric_number = row['matric_number']
130            cat = queryUtility(ICatalog, name='students_catalog')
131            results = list(
132                cat.searchResults(matric_number=(matric_number, matric_number)))
133            if results:
134                return results[0]
135        return None
136
137    def addEntry(self, obj, row, site):
138        parent = self.getParent(row, site)
139        parent.addStudent(obj)
140        # Reset _curr_stud_id if student_id has been imported
141        if self.getLocator(row) == 'student_id':
142            parent._curr_stud_id -= 1
143        # We have to log this if state is provided. If not,
144        # logging is done by the event handler handle_student_added
145        if row.has_key('state'):
146            parent.logger.info('%s - Student record created' % obj.student_id)
147        history = IObjectHistory(obj)
148        history.addMessage(_('Student record created'))
149        return
150
151    def delEntry(self, row, site):
152        student = self.getEntry(row, site)
153        if student is not None:
154            parent = self.getParent(row, site)
155            parent.logger.info('%s - Student removed' % student.student_id)
156            del parent[student.student_id]
157        pass
158
159    def checkUpdateRequirements(self, obj, row, site):
160        """Checks requirements the object must fulfill when being updated.
161
162        This method is not used in case of deleting or adding objects.
163
164        Returns error messages as strings in case of requirement
165        problems.
166        """
167        transition = row.get('transition', IGNORE_MARKER)
168        if transition not in (IGNORE_MARKER, ''):
169            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
170            if transition not in allowed_transitions:
171                return 'Transition not allowed.'
172        return None
173
174    def updateEntry(self, obj, row, site):
175        """Update obj to the values given in row.
176        """
177        items_changed = ''
178
179        # Remove student_id from row if empty
180        if row.has_key('student_id') and row['student_id'] in (
181            None, IGNORE_MARKER):
182            row.pop('student_id')
183
184        # Update password
185        if row.has_key('password'):
186            passwd = row.get('password', IGNORE_MARKER)
187            if passwd not in ('', IGNORE_MARKER):
188                if passwd.startswith('{SSHA}'):
189                    # already encrypted password
190                    obj.password = passwd
191                else:
192                    # not yet encrypted password
193                    IUserAccount(obj).setPassword(passwd)
194                items_changed += ('%s=%s, ' % ('password', passwd))
195            row.pop('password')
196
197        # Update registration state
198        if row.has_key('state'):
199            state = row.get('state', IGNORE_MARKER)
200            if state not in (IGNORE_MARKER, ''):
201                value = row['state']
202                IWorkflowState(obj).setState(value)
203                msg = _("State '${a}' set", mapping = {'a':value})
204                history = IObjectHistory(obj)
205                history.addMessage(msg)
206                items_changed += ('%s=%s, ' % ('state', state))
207            row.pop('state')
208
209        if row.has_key('transition'):
210            transition = row.get('transition', IGNORE_MARKER)
211            if transition not in (IGNORE_MARKER, ''):
212                value = row['transition']
213                IWorkflowInfo(obj).fireTransition(value)
214                items_changed += ('%s=%s, ' % ('transition', transition))
215            row.pop('transition')
216
217        # apply other values...
218        items_changed += super(StudentProcessor, self).updateEntry(
219            obj, row, site)
220
221        # Log actions...
222        parent = self.getParent(row, site)
223        if hasattr(obj,'student_id'):
224            # Update mode: the student exists and we can get the student_id
225            parent.logger.info(
226                '%s - Student record updated: %s'
227                % (obj.student_id, items_changed))
228        else:
229            # Create mode: the student does not yet exist
230            parent.logger.info('Student data imported: %s' % items_changed)
231        return items_changed
232
233    def getMapping(self, path, headerfields, mode):
234        """Get a mapping from CSV file headerfields to actually used fieldnames.
235        """
236        result = dict()
237        reader = csv.reader(open(path, 'rb'))
238        raw_header = reader.next()
239        for num, field in enumerate(headerfields):
240            if field not in ['student_id', 'reg_number', 'matric_number'
241                             ] and mode == 'remove':
242                continue
243            if field == u'--IGNORE--':
244                # Skip ignored columns in failed and finished data files.
245                continue
246            result[raw_header[num]] = field
247        return result
248
249    def checkConversion(self, row, mode='create'):
250        """Validates all values in row.
251        """
252        iface = self.iface
253        if mode in ['update', 'remove']:
254            if self.getLocator(row) == 'reg_number':
255                iface = self.iface_byregnumber
256            elif self.getLocator(row) == 'matric_number':
257                iface = self.iface_bymatricnumber
258        converter = IObjectConverter(iface)
259        errs, inv_errs, conv_dict =  converter.fromStringDict(
260            row, self.factory_name, mode=mode)
261        if row.has_key('transition') and \
262            not row['transition'] in IMPORTABLE_TRANSITIONS:
263            if row['transition'] not in (IGNORE_MARKER, ''):
264                errs.append(('transition','not allowed'))
265        if row.has_key('state') and \
266            not row['state'] in IMPORTABLE_STATES:
267            if row['state'] not in (IGNORE_MARKER, ''):
268                errs.append(('state','not allowed'))
269            else:
270                # state is an attribute of Student and must not
271                # be changed if empty
272                conv_dict['state'] = IGNORE_MARKER
273
274        try:
275            # Correct stud_id counter. As the IConverter for students
276            # creates student objects that are not used afterwards, we
277            # have to fix the site-wide student_id counter.
278            site = grok.getSite()
279            students = site['students']
280            students._curr_stud_id -= 1
281        except (KeyError, TypeError, AttributeError):
282                pass
283        return errs, inv_errs, conv_dict
284
285
286class StudentProcessorBase(BatchProcessor):
287    """A base for student subitem processor.
288
289    Helps reducing redundancy.
290    """
291    grok.baseclass()
292
293    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
294    additional_fields = []
295
296    #: header fields additional required
297    additional_headers = []
298
299    @property
300    def available_fields(self):
301        fields = ['student_id','reg_number','matric_number'
302                  ] + self.additional_fields
303        return sorted(list(set(fields + getFields(
304                self.iface).keys())))
305
306    def checkHeaders(self, headerfields, mode='ignore'):
307        if not 'reg_number' in headerfields and not 'student_id' \
308            in headerfields and not 'matric_number' in headerfields:
309            raise FatalCSVError(
310                "Need at least columns student_id " +
311                "or reg_number or matric_number for import!")
312        for name in self.additional_headers:
313            if not name in headerfields:
314                raise FatalCSVError(
315                    "Need %s for import!" % name)
316
317        # Check for fields to be ignored...
318        not_ignored_fields = [x for x in headerfields
319                              if not x.startswith('--')]
320        if len(set(not_ignored_fields)) < len(not_ignored_fields):
321            raise FatalCSVError(
322                "Double headers: each column name may only appear once.")
323        return True
324
325    def _getStudent(self, row, site):
326        NON_VALUES = ['', IGNORE_MARKER]
327        if not 'students' in site.keys():
328            return None
329        if row.get('student_id', '') not in NON_VALUES:
330            if row['student_id'] in site['students']:
331                student = site['students'][row['student_id']]
332                return student
333        elif row.get('reg_number', '') not in NON_VALUES:
334            reg_number = row['reg_number']
335            cat = queryUtility(ICatalog, name='students_catalog')
336            results = list(
337                cat.searchResults(reg_number=(reg_number, reg_number)))
338            if results:
339                return results[0]
340        elif row.get('matric_number', '') not in NON_VALUES:
341            matric_number = row['matric_number']
342            cat = queryUtility(ICatalog, name='students_catalog')
343            results = list(
344                cat.searchResults(matric_number=(matric_number, matric_number)))
345            if results:
346                return results[0]
347        return None
348
349    def parentsExist(self, row, site):
350        return self.getParent(row, site) is not None
351
352    def entryExists(self, row, site):
353        return self.getEntry(row, site) is not None
354
355    def checkConversion(self, row, mode='ignore'):
356        """Validates all values in row.
357        """
358        converter = IObjectConverter(self.iface)
359        errs, inv_errs, conv_dict =  converter.fromStringDict(
360            row, self.factory_name, mode=mode)
361        return errs, inv_errs, conv_dict
362
363
364class StudentStudyCourseProcessor(StudentProcessorBase):
365    """A batch processor for IStudentStudyCourse objects.
366    """
367    grok.implements(IBatchProcessor)
368    grok.provides(IBatchProcessor)
369    grok.context(Interface)
370    util_name = 'studycourseupdater'
371    grok.name(util_name)
372
373    name = u'StudentStudyCourse Processor (update only)'
374    iface = IStudentStudyCourse
375    factory_name = 'waeup.StudentStudyCourse'
376
377    location_fields = []
378    additional_fields = []
379
380    def getParent(self, row, site):
381        return self._getStudent(row, site)
382
383    def getEntry(self, row, site):
384        student = self.getParent(row, site)
385        if student is None:
386            return None
387        return student.get('studycourse')
388
389    def updateEntry(self, obj, row, site):
390        """Update obj to the values given in row.
391        """
392        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
393            obj, row, site)
394        parent = self.getParent(row, site)
395        parent.__parent__.logger.info(
396            '%s - Study course updated: %s'
397            % (parent.student_id, items_changed))
398        # Update the students_catalog
399        notify(grok.ObjectModifiedEvent(obj.__parent__))
400        return
401
402    def checkConversion(self, row, mode='ignore'):
403        """Validates all values in row.
404        """
405        errs, inv_errs, conv_dict = super(
406            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
407        # We have to check if current_level is in range of certificate.
408        if conv_dict.has_key('certificate'):
409          cert = conv_dict['certificate']
410          if conv_dict['current_level'] < cert.start_level or \
411              conv_dict['current_level'] > cert.end_level+120:
412              errs.append(('current_level','not in range'))
413        return errs, inv_errs, conv_dict
414
415class StudentStudyLevelProcessor(StudentProcessorBase):
416    """A batch processor for IStudentStudyLevel objects.
417    """
418    grok.implements(IBatchProcessor)
419    grok.provides(IBatchProcessor)
420    grok.context(Interface)
421    util_name = 'studylevelprocessor'
422    grok.name(util_name)
423
424    name = u'StudentStudyLevel Processor'
425    iface = IStudentStudyLevel
426    factory_name = 'waeup.StudentStudyLevel'
427
428    location_fields = []
429    additional_fields = ['level']
430    additional_headers = ['level']
431
432    def getParent(self, row, site):
433        student = self._getStudent(row, site)
434        if student is None:
435            return None
436        return student['studycourse']
437
438    def getEntry(self, row, site):
439        studycourse = self.getParent(row, site)
440        if studycourse is None:
441            return None
442        return studycourse.get(row['level'])
443
444    def updateEntry(self, obj, row, site):
445        """Update obj to the values given in row.
446        """
447        items_changed = super(StudentStudyLevelProcessor, self).updateEntry(
448            obj, row, site)
449        student = self.getParent(row, site).__parent__
450        student.__parent__.logger.info(
451            '%s - Study level updated: %s'
452            % (student.student_id, items_changed))
453        return
454
455    def addEntry(self, obj, row, site):
456        parent = self.getParent(row, site)
457        obj.level = int(row['level'])
458        parent[row['level']] = obj
459        return
460
461    def checkConversion(self, row, mode='ignore'):
462        """Validates all values in row.
463        """
464        errs, inv_errs, conv_dict = super(
465            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
466
467        # We have to check if level is a valid integer.
468        # This is not done by the converter.
469        try:
470            level = int(row['level'])
471            if level not in range(0,700,10):
472                errs.append(('level','no valid integer'))
473        except ValueError:
474            errs.append(('level','no integer'))
475        return errs, inv_errs, conv_dict
476
477class CourseTicketProcessor(StudentProcessorBase):
478    """A batch processor for ICourseTicket objects.
479    """
480    grok.implements(IBatchProcessor)
481    grok.provides(IBatchProcessor)
482    grok.context(Interface)
483    util_name = 'courseticketprocessor'
484    grok.name(util_name)
485
486    name = u'CourseTicket Processor'
487    iface = ICourseTicket
488    factory_name = 'waeup.CourseTicket'
489
490    location_fields = []
491    additional_fields = ['level', 'code']
492    additional_headers = ['level', 'code']
493
494    def getParent(self, row, site):
495        student = self._getStudent(row, site)
496        if student is None:
497            return None
498        return student['studycourse'].get(row['level'])
499
500    def getEntry(self, row, site):
501        level = self.getParent(row, site)
502        if level is None:
503            return None
504        return level.get(row['code'])
505
506    def updateEntry(self, obj, row, site):
507        """Update obj to the values given in row.
508        """
509        items_changed = super(CourseTicketProcessor, self).updateEntry(
510            obj, row, site)
511        student = self.getParent(row, site).__parent__.__parent__
512        student.__parent__.logger.info(
513            '%s - Course ticket updated: %s'
514            % (student.student_id, items_changed))
515        return
516
517    def addEntry(self, obj, row, site):
518        parent = self.getParent(row, site)
519        catalog = getUtility(ICatalog, name='courses_catalog')
520        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
521        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
522        obj.dcode = entries[0].__parent__.__parent__.code
523        obj.title = entries[0].title
524        obj.credits = entries[0].credits
525        obj.passmark = entries[0].passmark
526        obj.semester = entries[0].semester
527        parent[row['code']] = obj
528        return
529
530    def checkConversion(self, row, mode='ignore'):
531        """Validates all values in row.
532        """
533        errs, inv_errs, conv_dict = super(
534            CourseTicketProcessor, self).checkConversion(row, mode=mode)
535
536        # We have to check if course really exists.
537        # This is not done by the converter.
538        catalog = getUtility(ICatalog, name='courses_catalog')
539        entries = catalog.searchResults(code=(row['code'],row['code']))
540        if len(entries) == 0:
541            errs.append(('code','non-existent'))
542            return errs, inv_errs, conv_dict
543        return errs, inv_errs, conv_dict
544
545class StudentOnlinePaymentProcessor(StudentProcessorBase):
546    """A batch processor for IStudentOnlinePayment objects.
547    """
548    grok.implements(IBatchProcessor)
549    grok.provides(IBatchProcessor)
550    grok.context(Interface)
551    util_name = 'paymentprocessor'
552    grok.name(util_name)
553
554    name = u'Payment Processor'
555    iface = IStudentOnlinePayment
556    factory_name = 'waeup.StudentOnlinePayment'
557
558    location_fields = []
559    additional_fields = ['p_id']
560    additional_headers = ['p_id']
561
562    def parentsExist(self, row, site):
563        return self.getParent(row, site) is not None
564
565    def getParent(self, row, site):
566        student = self._getStudent(row, site)
567        if student is None:
568            return None
569        return student['payments']
570
571    def getEntry(self, row, site):
572        payments = self.getParent(row, site)
573        if payments is None:
574            return None
575        # We can use the hash symbol at the end of p_id in import files
576        # to avoid annoying automatic number transformation
577        # by Excel or Calc
578        p_id = row['p_id'].strip('#')
579        if p_id.startswith('p'):
580            entry = payments.get(p_id)
581        else:
582            # For data migration from old SRP
583            entry = payments.get('p' + p_id[6:])
584        return entry
585
586    def updateEntry(self, obj, row, site):
587        """Update obj to the values given in row.
588        """
589        items_changed = super(StudentOnlinePaymentProcessor, self).updateEntry(
590            obj, row, site)
591        student = self.getParent(row, site).__parent__
592        student.__parent__.logger.info(
593            '%s - Payment ticket updated: %s'
594            % (student.student_id, items_changed))
595        return
596
597    def addEntry(self, obj, row, site):
598        parent = self.getParent(row, site)
599        p_id = row['p_id'].strip('#')
600        if not p_id.startswith('p'):
601            # For data migration from old SRP
602            obj.p_id = 'p' + p_id[6:]
603            parent[obj.p_id] = obj
604        else:
605            parent[p_id] = obj
606        return
607
608    def checkConversion(self, row, mode='ignore'):
609        """Validates all values in row.
610        """
611        errs, inv_errs, conv_dict = super(
612            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
613
614        # We have to check p_id.
615        p_id = row['p_id'].strip('#')
616        if p_id.startswith('p'):
617            if not len(p_id) == 14:
618                errs.append(('p_id','invalid length'))
619                return errs, inv_errs, conv_dict
620        else:
621            if not len(p_id) == 19:
622                errs.append(('p_id','invalid length'))
623                return errs, inv_errs, conv_dict
624        return errs, inv_errs, conv_dict
625
626class StudentVerdictProcessor(StudentStudyCourseProcessor):
627    """A batch processor for verdicts.
628
629    Import verdicts and perform workflow transitions.
630    """
631
632    util_name = 'verdictupdater'
633    grok.name(util_name)
634
635    name = u'Verdict Processor (update only)'
636    iface = IStudentVerdictUpdate
637    factory_name = 'waeup.StudentStudyCourse'
638
639    additional_fields = ['current_session', 'current_level']
640
641    def checkUpdateRequirements(self, obj, row, site):
642        """Checks requirements the studycourse and the student must fulfill
643        before being updated.
644        """
645        # Check if current_levels correspond
646        if obj.current_level != row['current_level']:
647            return 'Current level does not correspond.'
648        # Check if current_sessions correspond
649        if obj.current_session != row['current_session']:
650            return 'Current session does not correspond.'
651        # Check if student is in state REGISTERED
652        if obj.student.state != VALIDATED:
653            return 'Student in wrong state.'
654        return None
655
656    def updateEntry(self, obj, row, site):
657        """Update obj to the values given in row.
658        """
659        # Don't set current_session, current_level
660        vals_to_set = dict((key, val) for key, val in row.items()
661                           if key not in ('current_session','current_level'))
662        items_changed = super(StudentVerdictProcessor, self).updateEntry(
663            obj, vals_to_set, site)
664        parent = self.getParent(row, site)
665        parent.__parent__.logger.info(
666            '%s - Verdict updated: %s'
667            % (parent.student_id, items_changed))
668        # Fire transition
669        IWorkflowInfo(obj.__parent__).fireTransition('return')
670        # Update the students_catalog
671        notify(grok.ObjectModifiedEvent(obj.__parent__))
672        return
Note: See TracBrowser for help on using the repository browser.