source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8312

Last change on this file since 8312 was 8309, checked in by Henrik Bettermann, 13 years ago

Implement import of student transitions.
  • Property svn:keywords set to Id
File size: 22.5 KB
Line 
1## $Id: batching.py 8309 2012-04-29 12:55:52Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58
59    location_fields = []
60    factory_name = 'waeup.Student'
61
62    @property
63    def available_fields(self):
64        fields = getFields(self.iface)
65        return sorted(list(set(
66            ['student_id','reg_number','matric_number',
67            'password', 'state', 'transition'] + fields.keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if 'state' in headerfields and 'transition' in headerfields:
71            raise FatalCSVError(
72                "State and transition can't be  imported at the same time!")
73        if not 'reg_number' in headerfields and not 'student_id' \
74            in headerfields and not 'matric_number' in headerfields:
75            raise FatalCSVError(
76                "Need at least columns student_id or reg_number " +
77                "or matric_number for import!")
78        if mode == 'create':
79            for field in self.required_fields:
80                if not field in headerfields:
81                    raise FatalCSVError(
82                        "Need at least columns %s for import!" %
83                        ', '.join(["'%s'" % x for x in self.required_fields]))
84        # Check for fields to be ignored...
85        not_ignored_fields = [x for x in headerfields
86                              if not x.startswith('--')]
87        if len(set(not_ignored_fields)) < len(not_ignored_fields):
88            raise FatalCSVError(
89                "Double headers: each column name may only appear once.")
90        return True
91
92    def parentsExist(self, row, site):
93        return 'students' in site.keys()
94
95    def getLocator(self, row):
96        if row.get('student_id',None) not in (None, IGNORE_MARKER):
97            return 'student_id'
98        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
99            return 'reg_number'
100        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
101            return 'matric_number'
102        else:
103            return None
104
105    # The entry never exists in create mode.
106    def entryExists(self, row, site):
107        return self.getEntry(row, site) is not None
108
109    def getParent(self, row, site):
110        return site['students']
111
112    def getEntry(self, row, site):
113        if not 'students' in site.keys():
114            return None
115        if self.getLocator(row) == 'student_id':
116            if row['student_id'] in site['students']:
117                student = site['students'][row['student_id']]
118                return student
119        elif self.getLocator(row) == 'reg_number':
120            reg_number = row['reg_number']
121            cat = queryUtility(ICatalog, name='students_catalog')
122            results = list(
123                cat.searchResults(reg_number=(reg_number, reg_number)))
124            if results:
125                return results[0]
126        elif self.getLocator(row) == 'matric_number':
127            matric_number = row['matric_number']
128            cat = queryUtility(ICatalog, name='students_catalog')
129            results = list(
130                cat.searchResults(matric_number=(matric_number, matric_number)))
131            if results:
132                return results[0]
133        return None
134
135    def addEntry(self, obj, row, site):
136        parent = self.getParent(row, site)
137        parent.addStudent(obj)
138        # We have to log this if state is provided. If not,
139        # logging is done by the event handler handle_student_added
140        if row.has_key('state'):
141            parent.logger.info('%s - Student record created' % obj.student_id)
142        history = IObjectHistory(obj)
143        history.addMessage(_('Student record created'))
144        return
145
146    def delEntry(self, row, site):
147        student = self.getEntry(row, site)
148        if student is not None:
149            parent = self.getParent(row, site)
150            parent.logger.info('%s - Student removed' % student.student_id)
151            del parent[student.student_id]
152        pass
153
154    def checkUpdateRequirements(self, obj, row, site):
155        """Checks requirements the object must fulfill when being updated.
156
157        This method is not used in case of deleting or adding objects.
158
159        Returns error messages as strings in case of requirement
160        problems.
161        """
162        transition = row.get('transition', IGNORE_MARKER)
163        if transition not in (IGNORE_MARKER, ''):
164            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
165            if transition not in allowed_transitions:
166                return 'Transition not allowed.'
167        return None
168
169    def updateEntry(self, obj, row, site):
170        """Update obj to the values given in row.
171        """
172        items_changed = ''
173
174        # Remove student_id from row if empty
175        if row.has_key('student_id') and row['student_id'] in (
176            None, IGNORE_MARKER):
177            row.pop('student_id')
178
179        # Update password
180        passwd = row.get('password', IGNORE_MARKER)
181        if passwd not in ('', IGNORE_MARKER):
182            IUserAccount(obj).setPassword(passwd)
183            row.pop('password')
184
185        # Update registration state
186        state = row.get('state', IGNORE_MARKER)
187        if state not in (IGNORE_MARKER, ''):
188            value = row['state']
189            IWorkflowState(obj).setState(value)
190            msg = _("State '${a}' set", mapping = {'a':value})
191            history = IObjectHistory(obj)
192            history.addMessage(msg)
193            row.pop('state')
194        transition = row.get('transition', IGNORE_MARKER)
195        if transition not in (IGNORE_MARKER, ''):
196            value = row['transition']
197            IWorkflowInfo(obj).fireTransition(value)
198            row.pop('transition')
199
200        # apply other values...
201        items_changed = super(StudentProcessor, self).updateEntry(
202            obj, row, site)
203
204        # Log actions...
205        parent = self.getParent(row, site)
206        if hasattr(obj,'student_id'):
207            # Update mode: the student exists and we can get the student_id
208            parent.logger.info(
209                '%s - Student record updated: %s'
210                % (obj.student_id, items_changed))
211        else:
212            # Create mode: the student does not yet exist
213            parent.logger.info('Student data imported: %s' % items_changed)
214        return items_changed
215
216    def getMapping(self, path, headerfields, mode):
217        """Get a mapping from CSV file headerfields to actually used fieldnames.
218        """
219        result = dict()
220        reader = csv.reader(open(path, 'rb'))
221        raw_header = reader.next()
222        for num, field in enumerate(headerfields):
223            if field not in ['student_id', 'reg_number', 'matric_number'
224                             ] and mode == 'remove':
225                continue
226            if field == u'--IGNORE--':
227                # Skip ignored columns in failed and finished data files.
228                continue
229            result[raw_header[num]] = field
230        return result
231
232    def checkConversion(self, row, mode='create'):
233        """Validates all values in row.
234        """
235        iface = self.iface
236        if mode in ['update', 'remove']:
237            if self.getLocator(row) == 'reg_number':
238                iface = IStudentUpdateByRegNo
239            elif self.getLocator(row) == 'matric_number':
240                iface = IStudentUpdateByMatricNo
241        converter = IObjectConverter(iface)
242        errs, inv_errs, conv_dict =  converter.fromStringDict(
243            row, self.factory_name, mode=mode)
244        if row.has_key('transition') and \
245            not row['transition'] in IMPORTABLE_TRANSITIONS:
246            if row['transition'] not in (IGNORE_MARKER, ''):
247                errs.append(('transition','not allowed'))
248        if row.has_key('state') and \
249            not row['state'] in IMPORTABLE_STATES:
250            if row['state'] not in (IGNORE_MARKER, ''):
251                errs.append(('state','not allowed'))
252            else:
253                # state is an attribute of Student and must not
254                # be changed if empty
255                conv_dict['state'] = IGNORE_MARKER
256        return errs, inv_errs, conv_dict
257
258
259class StudentProcessorBase(BatchProcessor):
260    """A base for student subitem processor.
261
262    Helps reducing redundancy.
263    """
264    grok.baseclass()
265
266    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
267    additional_fields = []
268
269    #: header fields additional required
270    additional_headers = []
271
272    @property
273    def available_fields(self):
274        fields = ['student_id','reg_number','matric_number'
275                  ] + self.additional_fields
276        return sorted(list(set(fields + getFields(
277                self.iface).keys())))
278
279    def checkHeaders(self, headerfields, mode='ignore'):
280        if not 'reg_number' in headerfields and not 'student_id' \
281            in headerfields and not 'matric_number' in headerfields:
282            raise FatalCSVError(
283                "Need at least columns student_id " +
284                "or reg_number or matric_number for import!")
285        for name in self.additional_headers:
286            if not name in headerfields:
287                raise FatalCSVError(
288                    "Need %s for import!" % name)
289
290        # Check for fields to be ignored...
291        not_ignored_fields = [x for x in headerfields
292                              if not x.startswith('--')]
293        if len(set(not_ignored_fields)) < len(not_ignored_fields):
294            raise FatalCSVError(
295                "Double headers: each column name may only appear once.")
296        return True
297
298    def _getStudent(self, row, site):
299        NON_VALUES = ['', IGNORE_MARKER]
300        if not 'students' in site.keys():
301            return None
302        if row.get('student_id', '') not in NON_VALUES:
303            if row['student_id'] in site['students']:
304                student = site['students'][row['student_id']]
305                return student
306        elif row.get('reg_number', '') not in NON_VALUES:
307            reg_number = row['reg_number']
308            cat = queryUtility(ICatalog, name='students_catalog')
309            results = list(
310                cat.searchResults(reg_number=(reg_number, reg_number)))
311            if results:
312                return results[0]
313        elif row.get('matric_number', '') not in NON_VALUES:
314            matric_number = row['matric_number']
315            cat = queryUtility(ICatalog, name='students_catalog')
316            results = list(
317                cat.searchResults(matric_number=(matric_number, matric_number)))
318            if results:
319                return results[0]
320        return None
321
322    def parentsExist(self, row, site):
323        result = self.getParent(row, site) is not None
324        return self.getParent(row, site) is not None
325
326    def entryExists(self, row, site):
327        return self.getEntry(row, site) is not None
328
329    def checkConversion(self, row, mode='ignore'):
330        """Validates all values in row.
331        """
332        converter = IObjectConverter(self.iface)
333        errs, inv_errs, conv_dict =  converter.fromStringDict(
334            row, self.factory_name, mode=mode)
335        return errs, inv_errs, conv_dict
336
337
338class StudentStudyCourseProcessor(StudentProcessorBase):
339    """A batch processor for IStudentStudyCourse objects.
340    """
341    grok.implements(IBatchProcessor)
342    grok.provides(IBatchProcessor)
343    grok.context(Interface)
344    util_name = 'studycourseupdater'
345    grok.name(util_name)
346
347    name = u'StudentStudyCourse Processor (update only)'
348    iface = IStudentStudyCourse
349    factory_name = 'waeup.StudentStudyCourse'
350
351    location_fields = []
352    additional_fields = []
353
354    def getParent(self, row, site):
355        return self._getStudent(row, site)
356
357    def getEntry(self, row, site):
358        student = self.getParent(row, site)
359        if student is None:
360            return None
361        return student.get('studycourse')
362
363    def updateEntry(self, obj, row, site):
364        """Update obj to the values given in row.
365        """
366        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
367            obj, row, site)
368        parent = self.getParent(row, site)
369        parent.__parent__.logger.info(
370            '%s - Study course updated: %s'
371            % (parent.student_id, items_changed))
372        # Update the students_catalog
373        notify(grok.ObjectModifiedEvent(obj.__parent__))
374        return
375
376    def checkConversion(self, row, mode='ignore'):
377        """Validates all values in row.
378        """
379        errs, inv_errs, conv_dict = super(
380            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
381        # We have to check if current_level is in range of certificate.
382        if conv_dict.has_key('certificate'):
383          cert = conv_dict['certificate']
384          if conv_dict['current_level'] < cert.start_level or \
385              conv_dict['current_level'] > cert.end_level+120:
386              errs.append(('current_level','not in range'))
387        return errs, inv_errs, conv_dict
388
389class StudentStudyLevelProcessor(StudentProcessorBase):
390    """A batch processor for IStudentStudyLevel objects.
391    """
392    grok.implements(IBatchProcessor)
393    grok.provides(IBatchProcessor)
394    grok.context(Interface)
395    util_name = 'studylevelprocessor'
396    grok.name(util_name)
397
398    name = u'StudentStudyLevel Processor'
399    iface = IStudentStudyLevel
400    factory_name = 'waeup.StudentStudyLevel'
401
402    location_fields = []
403    additional_fields = ['level']
404    additional_headers = ['level']
405
406    def getParent(self, row, site):
407        student = self._getStudent(row, site)
408        if student is None:
409            return None
410        return student['studycourse']
411
412    def getEntry(self, row, site):
413        studycourse = self.getParent(row, site)
414        if studycourse is None:
415            return None
416        return studycourse.get(row['level'])
417
418    def addEntry(self, obj, row, site):
419        parent = self.getParent(row, site)
420        obj.level = int(row['level'])
421        parent[row['level']] = obj
422        return
423
424    def checkConversion(self, row, mode='ignore'):
425        """Validates all values in row.
426        """
427        errs, inv_errs, conv_dict = super(
428            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
429
430        # We have to check if level is a valid integer.
431        # This is not done by the converter.
432        try:
433            level = int(row['level'])
434            if level not in range(0,700,10):
435                errs.append(('level','no valid integer'))
436        except ValueError:
437            errs.append(('level','no integer'))
438        return errs, inv_errs, conv_dict
439
440class CourseTicketProcessor(StudentProcessorBase):
441    """A batch processor for ICourseTicket objects.
442    """
443    grok.implements(IBatchProcessor)
444    grok.provides(IBatchProcessor)
445    grok.context(Interface)
446    util_name = 'courseticketprocessor'
447    grok.name(util_name)
448
449    name = u'CourseTicket Processor'
450    iface = ICourseTicket
451    factory_name = 'waeup.CourseTicket'
452
453    location_fields = []
454    additional_fields = ['level', 'code']
455    additional_headers = ['level', 'code']
456
457    def getParent(self, row, site):
458        student = self._getStudent(row, site)
459        if student is None:
460            return None
461        return student['studycourse'].get(row['level'])
462
463    def getEntry(self, row, site):
464        level = self.getParent(row, site)
465        if level is None:
466            return None
467        return level.get(row['code'])
468
469    def addEntry(self, obj, row, site):
470        parent = self.getParent(row, site)
471        catalog = getUtility(ICatalog, name='courses_catalog')
472        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
473        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
474        obj.dcode = entries[0].__parent__.__parent__.code
475        obj.title = entries[0].title
476        obj.credits = entries[0].credits
477        obj.passmark = entries[0].passmark
478        obj.semester = entries[0].semester
479        parent[row['code']] = obj
480        return
481
482    def checkConversion(self, row, mode='ignore'):
483        """Validates all values in row.
484        """
485        errs, inv_errs, conv_dict = super(
486            CourseTicketProcessor, self).checkConversion(row, mode=mode)
487
488        # We have to check if course really exists.
489        # This is not done by the converter.
490        catalog = getUtility(ICatalog, name='courses_catalog')
491        entries = catalog.searchResults(code=(row['code'],row['code']))
492        if len(entries) == 0:
493            errs.append(('code','non-existent'))
494            return errs, inv_errs, conv_dict
495        return errs, inv_errs, conv_dict
496
497class StudentOnlinePaymentProcessor(StudentProcessorBase):
498    """A batch processor for IStudentOnlinePayment objects.
499    """
500    grok.implements(IBatchProcessor)
501    grok.provides(IBatchProcessor)
502    grok.context(Interface)
503    util_name = 'paymentprocessor'
504    grok.name(util_name)
505
506    name = u'Payment Processor'
507    iface = IStudentOnlinePayment
508    factory_name = 'waeup.StudentOnlinePayment'
509
510    location_fields = []
511    additional_fields = ['p_id']
512    additional_headers = ['p_id']
513
514    def parentsExist(self, row, site):
515        result = self.getParent(row, site) is not None
516        return self.getParent(row, site) is not None
517
518    def getParent(self, row, site):
519        student = self._getStudent(row, site)
520        if student is None:
521            return None
522        return student['payments']
523
524    def getEntry(self, row, site):
525        payments = self.getParent(row, site)
526        if payments is None:
527            return None
528        # We can use the hash symbol at the end of p_id in import files
529        # to avoid annoying automatic number transformation
530        # by Excel or Calc
531        p_id = row['p_id'].strip('#')
532        if p_id.startswith('p'):
533            entry = payments.get(p_id)
534        else:
535            # For data migration from old SRP
536            entry = payments.get('p' + p_id[6:])
537        return entry
538
539    def addEntry(self, obj, row, site):
540        parent = self.getParent(row, site)
541        p_id = row['p_id'].strip('#')
542        if not p_id.startswith('p'):
543            # For data migration from old SRP
544            obj.p_id = 'p' + p_id[6:]
545            parent[obj.p_id] = obj
546        else:
547            parent[p_id] = obj
548        return
549
550    def checkConversion(self, row, mode='ignore'):
551        """Validates all values in row.
552        """
553        errs, inv_errs, conv_dict = super(
554            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
555
556        # We have to check p_id.
557        p_id = row['p_id'].strip('#')
558        if p_id.startswith('p'):
559            if not len(p_id) == 14:
560                errs.append(('p_id','invalid length'))
561                return errs, inv_errs, conv_dict
562        else:
563            if not len(p_id) == 19:
564                errs.append(('p_id','invalid length'))
565                return errs, inv_errs, conv_dict
566        return errs, inv_errs, conv_dict
567
568class StudentVerdictProcessor(StudentStudyCourseProcessor):
569    """A batch processor for verdicts.
570
571    Import verdicts and perform workflow transitions.
572    """
573
574    util_name = 'verdictupdater'
575    grok.name(util_name)
576
577    name = u'Verdict Processor (update only)'
578    iface = IStudentVerdictUpdate
579    factory_name = 'waeup.StudentStudyCourse'
580
581    additional_fields = ['current_session', 'current_level']
582
583    def checkUpdateRequirements(self, obj, row, site):
584        """Checks requirements the studycourse and the student must fulfill
585        before being updated.
586        """
587        # Check if current_levels correspond
588        if obj.current_level != row['current_level']:
589            return 'Current level does not correspond.'
590        # Check if current_sessions correspond
591        if obj.current_session != row['current_session']:
592            return 'Current session does not correspond.'
593        # Check if student is in state REGISTERED
594        if obj.getStudent().state != VALIDATED:
595            return 'Student in wrong state.'
596        return None
597
598    def updateEntry(self, obj, row, site):
599        """Update obj to the values given in row.
600        """
601        # Don't set current_session, current_level
602        vals_to_set = dict((key, val) for key, val in row.items()
603                           if key not in ('current_session','current_level'))
604        items_changed = super(StudentVerdictProcessor, self).updateEntry(
605            obj, vals_to_set, site)
606        parent = self.getParent(row, site)
607        parent.__parent__.logger.info(
608            '%s - Verdict updated: %s'
609            % (parent.student_id, items_changed))
610        # Fire transition
611        IWorkflowInfo(obj.__parent__).fireTransition('return')
612        # Update the students_catalog
613        notify(grok.ObjectModifiedEvent(obj.__parent__))
614        return
Note: See TracBrowser for help on using the repository browser.