source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8286

Last change on this file since 8286 was 8284, checked in by Henrik Bettermann, 13 years ago

Allow empty reg_state cells. This is necessary particularly during mixed updates.
  • Property svn:keywords set to Id
File size: 21.2 KB
Line 
1## $Id: batching.py 8284 2012-04-26 11:30:32Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, DELETION_MARKER, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  IMPORTABLE_STATES
44from waeup.kofa.utils.batching import BatchProcessor
45
46class StudentProcessor(BatchProcessor):
47    """A batch processor for IStudent objects.
48    """
49    grok.implements(IBatchProcessor)
50    grok.provides(IBatchProcessor)
51    grok.context(Interface)
52    util_name = 'studentprocessor'
53    grok.name(util_name)
54
55    name = u'Student Processor'
56    iface = IStudent
57
58    location_fields = []
59    factory_name = 'waeup.Student'
60
61    @property
62    def available_fields(self):
63        fields = getFields(self.iface)
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'reg_state'] + getFields(
67                self.iface).keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if not 'reg_number' in headerfields and not 'student_id' \
71            in headerfields and not 'matric_number' in headerfields:
72            raise FatalCSVError(
73                "Need at least columns student_id or reg_number " +
74                "or matric_number for import!")
75        if mode == 'create':
76            for field in self.required_fields:
77                if not field in headerfields:
78                    raise FatalCSVError(
79                        "Need at least columns %s for import!" %
80                        ', '.join(["'%s'" % x for x in self.required_fields]))
81        # Check for fields to be ignored...
82        not_ignored_fields = [x for x in headerfields
83                              if not x.startswith('--')]
84        if len(set(not_ignored_fields)) < len(not_ignored_fields):
85            raise FatalCSVError(
86                "Double headers: each column name may only appear once.")
87        return True
88
89    def parentsExist(self, row, site):
90        return 'students' in site.keys()
91
92    def getLocator(self, row):
93        if row.get('student_id',None) not in (None, IGNORE_MARKER):
94            return 'student_id'
95        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
96            return 'reg_number'
97        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
98            return 'matric_number'
99        else:
100            return None
101
102    # The entry never exists in create mode.
103    def entryExists(self, row, site):
104        return self.getEntry(row, site) is not None
105
106    def getParent(self, row, site):
107        return site['students']
108
109    def getEntry(self, row, site):
110        if not 'students' in site.keys():
111            return None
112        if self.getLocator(row) == 'student_id':
113            if row['student_id'] in site['students']:
114                student = site['students'][row['student_id']]
115                return student
116        elif self.getLocator(row) == 'reg_number':
117            reg_number = row['reg_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(reg_number=(reg_number, reg_number)))
121            if results:
122                return results[0]
123        elif self.getLocator(row) == 'matric_number':
124            matric_number = row['matric_number']
125            cat = queryUtility(ICatalog, name='students_catalog')
126            results = list(
127                cat.searchResults(matric_number=(matric_number, matric_number)))
128            if results:
129                return results[0]
130        return None
131
132    def addEntry(self, obj, row, site):
133        parent = self.getParent(row, site)
134        parent.addStudent(obj)
135        # We have to log this if reg_state is provided. If not,
136        # logging is done by the event handler handle_student_added
137        if row.has_key('reg_state'):
138            parent.logger.info('%s - Student record created' % obj.student_id)
139        history = IObjectHistory(obj)
140        history.addMessage(_('Student record created'))
141        return
142
143    def delEntry(self, row, site):
144        student = self.getEntry(row, site)
145        if student is not None:
146            parent = self.getParent(row, site)
147            parent.logger.info('%s - Student removed' % student.student_id)
148            del parent[student.student_id]
149        pass
150
151    def updateEntry(self, obj, row, site):
152        """Update obj to the values given in row.
153        """
154        items_changed = ''
155
156        # Remove student_id from row if empty
157        if row.has_key('student_id') and row['student_id'] in (
158            None, IGNORE_MARKER):
159            row.pop('student_id')
160
161        # Update password
162        passwd = row.get('password', IGNORE_MARKER)
163        if passwd not in ('', IGNORE_MARKER):
164            IUserAccount(obj).setPassword(passwd)
165            row.pop('password')
166
167        # Update registration state
168        reg_state = row.get('reg_state', IGNORE_MARKER)
169        if reg_state not in (IGNORE_MARKER, ''):
170            value = row['reg_state']
171            IWorkflowState(obj).setState(value)
172            msg = _("State '${a}' set", mapping = {'a':value})
173            history = IObjectHistory(obj)
174            history.addMessage(msg)
175            row.pop('reg_state')
176
177        # apply other values...
178        items_changed = super(StudentProcessor, self).updateEntry(
179            obj, row, site)
180
181        # Log actions...
182        parent = self.getParent(row, site)
183        if hasattr(obj,'student_id'):
184            # Update mode: the student exists and we can get the student_id
185            parent.logger.info(
186                '%s - Student record updated: %s'
187                % (obj.student_id, items_changed))
188        else:
189            # Create mode: the student does not yet exist
190            parent.logger.info('Student data imported: %s' % items_changed)
191        return items_changed
192
193    def getMapping(self, path, headerfields, mode):
194        """Get a mapping from CSV file headerfields to actually used fieldnames.
195        """
196        result = dict()
197        reader = csv.reader(open(path, 'rb'))
198        raw_header = reader.next()
199        for num, field in enumerate(headerfields):
200            if field not in ['student_id', 'reg_number', 'matric_number'
201                             ] and mode == 'remove':
202                continue
203            if field == u'--IGNORE--':
204                # Skip ignored columns in failed and finished data files.
205                continue
206            result[raw_header[num]] = field
207        return result
208
209    def checkConversion(self, row, mode='create'):
210        """Validates all values in row.
211        """
212        iface = self.iface
213        if mode in ['update', 'remove']:
214            if self.getLocator(row) == 'reg_number':
215                iface = IStudentUpdateByRegNo
216            elif self.getLocator(row) == 'matric_number':
217                iface = IStudentUpdateByMatricNo
218        converter = IObjectConverter(iface)
219        errs, inv_errs, conv_dict =  converter.fromStringDict(
220            row, self.factory_name, mode=mode)
221        if row.has_key('reg_state') and \
222            not row['reg_state'] in IMPORTABLE_STATES:
223            if row['reg_state'] not in (IGNORE_MARKER, ''):
224                errs.append(('reg_state','not allowed'))
225            #else:
226            #    errs.append(('reg_state','no value provided'))
227        return errs, inv_errs, conv_dict
228
229
230class StudentProcessorBase(BatchProcessor):
231    """A base for student subitem processor.
232
233    Helps reducing redundancy.
234    """
235    grok.baseclass()
236
237    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
238    additional_fields = []
239
240    #: header fields additional required
241    additional_headers = []
242
243    @property
244    def available_fields(self):
245        fields = ['student_id','reg_number','matric_number'
246                  ] + self.additional_fields
247        return sorted(list(set(fields + getFields(
248                self.iface).keys())))
249
250    def checkHeaders(self, headerfields, mode='ignore'):
251        if not 'reg_number' in headerfields and not 'student_id' \
252            in headerfields and not 'matric_number' in headerfields:
253            raise FatalCSVError(
254                "Need at least columns student_id " +
255                "or reg_number or matric_number for import!")
256        for name in self.additional_headers:
257            if not name in headerfields:
258                raise FatalCSVError(
259                    "Need %s for import!" % name)
260
261        # Check for fields to be ignored...
262        not_ignored_fields = [x for x in headerfields
263                              if not x.startswith('--')]
264        if len(set(not_ignored_fields)) < len(not_ignored_fields):
265            raise FatalCSVError(
266                "Double headers: each column name may only appear once.")
267        return True
268
269    def _getStudent(self, row, site):
270        NON_VALUES = ['', IGNORE_MARKER]
271        if not 'students' in site.keys():
272            return None
273        if row.get('student_id', '') not in NON_VALUES:
274            if row['student_id'] in site['students']:
275                student = site['students'][row['student_id']]
276                return student
277        elif row.get('reg_number', '') not in NON_VALUES:
278            reg_number = row['reg_number']
279            cat = queryUtility(ICatalog, name='students_catalog')
280            results = list(
281                cat.searchResults(reg_number=(reg_number, reg_number)))
282            if results:
283                return results[0]
284        elif row.get('matric_number', '') not in NON_VALUES:
285            matric_number = row['matric_number']
286            cat = queryUtility(ICatalog, name='students_catalog')
287            results = list(
288                cat.searchResults(matric_number=(matric_number, matric_number)))
289            if results:
290                return results[0]
291        return None
292
293    def parentsExist(self, row, site):
294        result = self.getParent(row, site) is not None
295        return self.getParent(row, site) is not None
296
297    def entryExists(self, row, site):
298        return self.getEntry(row, site) is not None
299
300    def checkConversion(self, row, mode='ignore'):
301        """Validates all values in row.
302        """
303        converter = IObjectConverter(self.iface)
304        errs, inv_errs, conv_dict =  converter.fromStringDict(
305            row, self.factory_name, mode=mode)
306        return errs, inv_errs, conv_dict
307
308
309class StudentStudyCourseProcessor(StudentProcessorBase):
310    """A batch processor for IStudentStudyCourse objects.
311    """
312    grok.implements(IBatchProcessor)
313    grok.provides(IBatchProcessor)
314    grok.context(Interface)
315    util_name = 'studycourseupdater'
316    grok.name(util_name)
317
318    name = u'StudentStudyCourse Processor (update only)'
319    iface = IStudentStudyCourse
320    factory_name = 'waeup.StudentStudyCourse'
321
322    location_fields = []
323    additional_fields = []
324
325    def getParent(self, row, site):
326        return self._getStudent(row, site)
327
328    def getEntry(self, row, site):
329        student = self.getParent(row, site)
330        if student is None:
331            return None
332        return student.get('studycourse')
333
334    def updateEntry(self, obj, row, site):
335        """Update obj to the values given in row.
336        """
337        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
338            obj, row, site)
339        parent = self.getParent(row, site)
340        parent.__parent__.logger.info(
341            '%s - Study course updated: %s'
342            % (parent.student_id, items_changed))
343        # Update the students_catalog
344        notify(grok.ObjectModifiedEvent(obj.__parent__))
345        return
346
347    def checkConversion(self, row, mode='ignore'):
348        """Validates all values in row.
349        """
350        errs, inv_errs, conv_dict = super(
351            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
352        # We have to check if current_level is in range of certificate.
353        if conv_dict.has_key('certificate'):
354          cert = conv_dict['certificate']
355          if conv_dict['current_level'] < cert.start_level or \
356              conv_dict['current_level'] > cert.end_level+120:
357              errs.append(('current_level','not in range'))
358        return errs, inv_errs, conv_dict
359
360class StudentStudyLevelProcessor(StudentProcessorBase):
361    """A batch processor for IStudentStudyLevel objects.
362    """
363    grok.implements(IBatchProcessor)
364    grok.provides(IBatchProcessor)
365    grok.context(Interface)
366    util_name = 'studylevelprocessor'
367    grok.name(util_name)
368
369    name = u'StudentStudyLevel Processor'
370    iface = IStudentStudyLevel
371    factory_name = 'waeup.StudentStudyLevel'
372
373    location_fields = []
374    additional_fields = ['level']
375    additional_headers = ['level']
376
377    def getParent(self, row, site):
378        student = self._getStudent(row, site)
379        if student is None:
380            return None
381        return student['studycourse']
382
383    def getEntry(self, row, site):
384        studycourse = self.getParent(row, site)
385        if studycourse is None:
386            return None
387        return studycourse.get(row['level'])
388
389    def addEntry(self, obj, row, site):
390        parent = self.getParent(row, site)
391        obj.level = int(row['level'])
392        parent[row['level']] = obj
393        return
394
395    def checkConversion(self, row, mode='ignore'):
396        """Validates all values in row.
397        """
398        errs, inv_errs, conv_dict = super(
399            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
400
401        # We have to check if level is a valid integer.
402        # This is not done by the converter.
403        try:
404            level = int(row['level'])
405            if level not in range(0,700,10):
406                errs.append(('level','no valid integer'))
407        except ValueError:
408            errs.append(('level','no integer'))
409        return errs, inv_errs, conv_dict
410
411class CourseTicketProcessor(StudentProcessorBase):
412    """A batch processor for ICourseTicket objects.
413    """
414    grok.implements(IBatchProcessor)
415    grok.provides(IBatchProcessor)
416    grok.context(Interface)
417    util_name = 'courseticketprocessor'
418    grok.name(util_name)
419
420    name = u'CourseTicket Processor'
421    iface = ICourseTicket
422    factory_name = 'waeup.CourseTicket'
423
424    location_fields = []
425    additional_fields = ['level', 'code']
426    additional_headers = ['level', 'code']
427
428    def getParent(self, row, site):
429        student = self._getStudent(row, site)
430        if student is None:
431            return None
432        return student['studycourse'].get(row['level'])
433
434    def getEntry(self, row, site):
435        level = self.getParent(row, site)
436        if level is None:
437            return None
438        return level.get(row['code'])
439
440    def addEntry(self, obj, row, site):
441        parent = self.getParent(row, site)
442        catalog = getUtility(ICatalog, name='courses_catalog')
443        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
444        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
445        obj.dcode = entries[0].__parent__.__parent__.code
446        obj.title = entries[0].title
447        obj.credits = entries[0].credits
448        obj.passmark = entries[0].passmark
449        obj.semester = entries[0].semester
450        parent[row['code']] = obj
451        return
452
453    def checkConversion(self, row, mode='ignore'):
454        """Validates all values in row.
455        """
456        errs, inv_errs, conv_dict = super(
457            CourseTicketProcessor, self).checkConversion(row, mode=mode)
458
459        # We have to check if course really exists.
460        # This is not done by the converter.
461        catalog = getUtility(ICatalog, name='courses_catalog')
462        entries = catalog.searchResults(code=(row['code'],row['code']))
463        if len(entries) == 0:
464            errs.append(('code','non-existent'))
465            return errs, inv_errs, conv_dict
466        return errs, inv_errs, conv_dict
467
468class StudentOnlinePaymentProcessor(StudentProcessorBase):
469    """A batch processor for IStudentOnlinePayment objects.
470    """
471    grok.implements(IBatchProcessor)
472    grok.provides(IBatchProcessor)
473    grok.context(Interface)
474    util_name = 'paymentprocessor'
475    grok.name(util_name)
476
477    name = u'Payment Processor'
478    iface = IStudentOnlinePayment
479    factory_name = 'waeup.StudentOnlinePayment'
480
481    location_fields = []
482    additional_fields = ['p_id']
483    additional_headers = ['p_id']
484
485    def parentsExist(self, row, site):
486        result = self.getParent(row, site) is not None
487        return self.getParent(row, site) is not None
488
489    def getParent(self, row, site):
490        student = self._getStudent(row, site)
491        if student is None:
492            return None
493        return student['payments']
494
495    def getEntry(self, row, site):
496        payments = self.getParent(row, site)
497        if payments is None:
498            return None
499        # We can use the hash symbol at the end of p_id in import files
500        # to avoid annoying automatic number transformation
501        # by Excel or Calc
502        p_id = row['p_id'].strip('#')
503        if p_id.startswith('p'):
504            entry = payments.get(p_id)
505        else:
506            # For data migration from old SRP
507            entry = payments.get('p' + p_id[6:])
508        return entry
509
510    def addEntry(self, obj, row, site):
511        parent = self.getParent(row, site)
512        p_id = row['p_id'].strip('#')
513        if not p_id.startswith('p'):
514            # For data migration from old SRP
515            obj.p_id = 'p' + p_id[6:]
516            parent[obj.p_id] = obj
517        else:
518            parent[p_id] = obj
519        return
520
521    def checkConversion(self, row, mode='ignore'):
522        """Validates all values in row.
523        """
524        errs, inv_errs, conv_dict = super(
525            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
526
527        # We have to check p_id.
528        p_id = row['p_id'].strip('#')
529        if p_id.startswith('p'):
530            if not len(p_id) == 14:
531                errs.append(('p_id','invalid length'))
532                return errs, inv_errs, conv_dict
533        else:
534            if not len(p_id) == 19:
535                errs.append(('p_id','invalid length'))
536                return errs, inv_errs, conv_dict
537        return errs, inv_errs, conv_dict
538
539class StudentVerdictProcessor(StudentStudyCourseProcessor):
540    """A batch processor for verdicts.
541
542    Import verdicts and perform workflow transitions.
543    """
544
545    util_name = 'verdictupdater'
546    grok.name(util_name)
547
548    name = u'Verdict Processor (update only)'
549    iface = IStudentVerdictUpdate
550    factory_name = 'waeup.StudentStudyCourse'
551
552    additional_fields = ['current_session', 'current_level']
553
554    def checkUpdateRequirements(self, obj, row, site):
555        """Checks requirements the studycourse and the student must fulfill
556        before being updated.
557        """
558        # Check if current_levels correspond
559        if obj.current_level != row['current_level']:
560            return 'Current level does not correspond.'
561        # Check if current_sessions correspond
562        if obj.current_session != row['current_session']:
563            return 'Current session does not correspond.'
564        # Check if student is in state REGISTERED
565        if obj.getStudent().state != VALIDATED:
566            return 'Student in wrong state.'
567        return None
568
569    def updateEntry(self, obj, row, site):
570        """Update obj to the values given in row.
571        """
572        # Don't set current_session, current_level
573        vals_to_set = dict((key, val) for key, val in row.items()
574                           if key not in ('current_session','current_level'))
575        items_changed = super(StudentVerdictProcessor, self).updateEntry(
576            obj, vals_to_set, site)
577        parent = self.getParent(row, site)
578        parent.__parent__.logger.info(
579            '%s - Verdict updated: %s'
580            % (parent.student_id, items_changed))
581        # Fire transition
582        IWorkflowInfo(obj.__parent__).fireTransition('return')
583        # Update the students_catalog
584        notify(grok.ObjectModifiedEvent(obj.__parent__))
585        return
Note: See TracBrowser for help on using the repository browser.