source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8300

Last change on this file since 8300 was 8300, checked in by Henrik Bettermann, 13 years ago

Remove unneeded imports.
  • Property svn:keywords set to Id
File size: 21.2 KB
Line 
1## $Id: batching.py 8300 2012-04-28 07:51:31Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  IMPORTABLE_STATES
44from waeup.kofa.utils.batching import BatchProcessor
45
46class StudentProcessor(BatchProcessor):
47    """A batch processor for IStudent objects.
48    """
49    grok.implements(IBatchProcessor)
50    grok.provides(IBatchProcessor)
51    grok.context(Interface)
52    util_name = 'studentprocessor'
53    grok.name(util_name)
54
55    name = u'Student Processor'
56    iface = IStudent
57
58    location_fields = []
59    factory_name = 'waeup.Student'
60
61    @property
62    def available_fields(self):
63        fields = getFields(self.iface)
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'state'] + fields.keys())))
67
68    def checkHeaders(self, headerfields, mode='create'):
69        if not 'reg_number' in headerfields and not 'student_id' \
70            in headerfields and not 'matric_number' in headerfields:
71            raise FatalCSVError(
72                "Need at least columns student_id or reg_number " +
73                "or matric_number for import!")
74        if mode == 'create':
75            for field in self.required_fields:
76                if not field in headerfields:
77                    raise FatalCSVError(
78                        "Need at least columns %s for import!" %
79                        ', '.join(["'%s'" % x for x in self.required_fields]))
80        # Check for fields to be ignored...
81        not_ignored_fields = [x for x in headerfields
82                              if not x.startswith('--')]
83        if len(set(not_ignored_fields)) < len(not_ignored_fields):
84            raise FatalCSVError(
85                "Double headers: each column name may only appear once.")
86        return True
87
88    def parentsExist(self, row, site):
89        return 'students' in site.keys()
90
91    def getLocator(self, row):
92        if row.get('student_id',None) not in (None, IGNORE_MARKER):
93            return 'student_id'
94        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
95            return 'reg_number'
96        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
97            return 'matric_number'
98        else:
99            return None
100
101    # The entry never exists in create mode.
102    def entryExists(self, row, site):
103        return self.getEntry(row, site) is not None
104
105    def getParent(self, row, site):
106        return site['students']
107
108    def getEntry(self, row, site):
109        if not 'students' in site.keys():
110            return None
111        if self.getLocator(row) == 'student_id':
112            if row['student_id'] in site['students']:
113                student = site['students'][row['student_id']]
114                return student
115        elif self.getLocator(row) == 'reg_number':
116            reg_number = row['reg_number']
117            cat = queryUtility(ICatalog, name='students_catalog')
118            results = list(
119                cat.searchResults(reg_number=(reg_number, reg_number)))
120            if results:
121                return results[0]
122        elif self.getLocator(row) == 'matric_number':
123            matric_number = row['matric_number']
124            cat = queryUtility(ICatalog, name='students_catalog')
125            results = list(
126                cat.searchResults(matric_number=(matric_number, matric_number)))
127            if results:
128                return results[0]
129        return None
130
131    def addEntry(self, obj, row, site):
132        parent = self.getParent(row, site)
133        parent.addStudent(obj)
134        # We have to log this if state is provided. If not,
135        # logging is done by the event handler handle_student_added
136        if row.has_key('state'):
137            parent.logger.info('%s - Student record created' % obj.student_id)
138        history = IObjectHistory(obj)
139        history.addMessage(_('Student record created'))
140        return
141
142    def delEntry(self, row, site):
143        student = self.getEntry(row, site)
144        if student is not None:
145            parent = self.getParent(row, site)
146            parent.logger.info('%s - Student removed' % student.student_id)
147            del parent[student.student_id]
148        pass
149
150    def updateEntry(self, obj, row, site):
151        """Update obj to the values given in row.
152        """
153        items_changed = ''
154
155        # Remove student_id from row if empty
156        if row.has_key('student_id') and row['student_id'] in (
157            None, IGNORE_MARKER):
158            row.pop('student_id')
159
160        # Update password
161        passwd = row.get('password', IGNORE_MARKER)
162        if passwd not in ('', IGNORE_MARKER):
163            IUserAccount(obj).setPassword(passwd)
164            row.pop('password')
165
166        # Update registration state
167        state = row.get('state', IGNORE_MARKER)
168        if state not in (IGNORE_MARKER, ''):
169            value = row['state']
170            IWorkflowState(obj).setState(value)
171            msg = _("State '${a}' set", mapping = {'a':value})
172            history = IObjectHistory(obj)
173            history.addMessage(msg)
174            row.pop('state')
175
176        # apply other values...
177        items_changed = super(StudentProcessor, self).updateEntry(
178            obj, row, site)
179
180        # Log actions...
181        parent = self.getParent(row, site)
182        if hasattr(obj,'student_id'):
183            # Update mode: the student exists and we can get the student_id
184            parent.logger.info(
185                '%s - Student record updated: %s'
186                % (obj.student_id, items_changed))
187        else:
188            # Create mode: the student does not yet exist
189            parent.logger.info('Student data imported: %s' % items_changed)
190        return items_changed
191
192    def getMapping(self, path, headerfields, mode):
193        """Get a mapping from CSV file headerfields to actually used fieldnames.
194        """
195        result = dict()
196        reader = csv.reader(open(path, 'rb'))
197        raw_header = reader.next()
198        for num, field in enumerate(headerfields):
199            if field not in ['student_id', 'reg_number', 'matric_number'
200                             ] and mode == 'remove':
201                continue
202            if field == u'--IGNORE--':
203                # Skip ignored columns in failed and finished data files.
204                continue
205            result[raw_header[num]] = field
206        return result
207
208    def checkConversion(self, row, mode='create'):
209        """Validates all values in row.
210        """
211        iface = self.iface
212        if mode in ['update', 'remove']:
213            if self.getLocator(row) == 'reg_number':
214                iface = IStudentUpdateByRegNo
215            elif self.getLocator(row) == 'matric_number':
216                iface = IStudentUpdateByMatricNo
217        converter = IObjectConverter(iface)
218        errs, inv_errs, conv_dict =  converter.fromStringDict(
219            row, self.factory_name, mode=mode)
220        if row.has_key('state') and \
221            not row['state'] in IMPORTABLE_STATES:
222            if row['state'] not in (IGNORE_MARKER, ''):
223                errs.append(('state','not allowed'))
224            else:
225                # state is an attribute of Student and must not
226                # be changed if empty
227                conv_dict['state'] = IGNORE_MARKER
228        return errs, inv_errs, conv_dict
229
230
231class StudentProcessorBase(BatchProcessor):
232    """A base for student subitem processor.
233
234    Helps reducing redundancy.
235    """
236    grok.baseclass()
237
238    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
239    additional_fields = []
240
241    #: header fields additional required
242    additional_headers = []
243
244    @property
245    def available_fields(self):
246        fields = ['student_id','reg_number','matric_number'
247                  ] + self.additional_fields
248        return sorted(list(set(fields + getFields(
249                self.iface).keys())))
250
251    def checkHeaders(self, headerfields, mode='ignore'):
252        if not 'reg_number' in headerfields and not 'student_id' \
253            in headerfields and not 'matric_number' in headerfields:
254            raise FatalCSVError(
255                "Need at least columns student_id " +
256                "or reg_number or matric_number for import!")
257        for name in self.additional_headers:
258            if not name in headerfields:
259                raise FatalCSVError(
260                    "Need %s for import!" % name)
261
262        # Check for fields to be ignored...
263        not_ignored_fields = [x for x in headerfields
264                              if not x.startswith('--')]
265        if len(set(not_ignored_fields)) < len(not_ignored_fields):
266            raise FatalCSVError(
267                "Double headers: each column name may only appear once.")
268        return True
269
270    def _getStudent(self, row, site):
271        NON_VALUES = ['', IGNORE_MARKER]
272        if not 'students' in site.keys():
273            return None
274        if row.get('student_id', '') not in NON_VALUES:
275            if row['student_id'] in site['students']:
276                student = site['students'][row['student_id']]
277                return student
278        elif row.get('reg_number', '') not in NON_VALUES:
279            reg_number = row['reg_number']
280            cat = queryUtility(ICatalog, name='students_catalog')
281            results = list(
282                cat.searchResults(reg_number=(reg_number, reg_number)))
283            if results:
284                return results[0]
285        elif row.get('matric_number', '') not in NON_VALUES:
286            matric_number = row['matric_number']
287            cat = queryUtility(ICatalog, name='students_catalog')
288            results = list(
289                cat.searchResults(matric_number=(matric_number, matric_number)))
290            if results:
291                return results[0]
292        return None
293
294    def parentsExist(self, row, site):
295        result = self.getParent(row, site) is not None
296        return self.getParent(row, site) is not None
297
298    def entryExists(self, row, site):
299        return self.getEntry(row, site) is not None
300
301    def checkConversion(self, row, mode='ignore'):
302        """Validates all values in row.
303        """
304        converter = IObjectConverter(self.iface)
305        errs, inv_errs, conv_dict =  converter.fromStringDict(
306            row, self.factory_name, mode=mode)
307        return errs, inv_errs, conv_dict
308
309
310class StudentStudyCourseProcessor(StudentProcessorBase):
311    """A batch processor for IStudentStudyCourse objects.
312    """
313    grok.implements(IBatchProcessor)
314    grok.provides(IBatchProcessor)
315    grok.context(Interface)
316    util_name = 'studycourseupdater'
317    grok.name(util_name)
318
319    name = u'StudentStudyCourse Processor (update only)'
320    iface = IStudentStudyCourse
321    factory_name = 'waeup.StudentStudyCourse'
322
323    location_fields = []
324    additional_fields = []
325
326    def getParent(self, row, site):
327        return self._getStudent(row, site)
328
329    def getEntry(self, row, site):
330        student = self.getParent(row, site)
331        if student is None:
332            return None
333        return student.get('studycourse')
334
335    def updateEntry(self, obj, row, site):
336        """Update obj to the values given in row.
337        """
338        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
339            obj, row, site)
340        parent = self.getParent(row, site)
341        parent.__parent__.logger.info(
342            '%s - Study course updated: %s'
343            % (parent.student_id, items_changed))
344        # Update the students_catalog
345        notify(grok.ObjectModifiedEvent(obj.__parent__))
346        return
347
348    def checkConversion(self, row, mode='ignore'):
349        """Validates all values in row.
350        """
351        errs, inv_errs, conv_dict = super(
352            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
353        # We have to check if current_level is in range of certificate.
354        if conv_dict.has_key('certificate'):
355          cert = conv_dict['certificate']
356          if conv_dict['current_level'] < cert.start_level or \
357              conv_dict['current_level'] > cert.end_level+120:
358              errs.append(('current_level','not in range'))
359        return errs, inv_errs, conv_dict
360
361class StudentStudyLevelProcessor(StudentProcessorBase):
362    """A batch processor for IStudentStudyLevel objects.
363    """
364    grok.implements(IBatchProcessor)
365    grok.provides(IBatchProcessor)
366    grok.context(Interface)
367    util_name = 'studylevelprocessor'
368    grok.name(util_name)
369
370    name = u'StudentStudyLevel Processor'
371    iface = IStudentStudyLevel
372    factory_name = 'waeup.StudentStudyLevel'
373
374    location_fields = []
375    additional_fields = ['level']
376    additional_headers = ['level']
377
378    def getParent(self, row, site):
379        student = self._getStudent(row, site)
380        if student is None:
381            return None
382        return student['studycourse']
383
384    def getEntry(self, row, site):
385        studycourse = self.getParent(row, site)
386        if studycourse is None:
387            return None
388        return studycourse.get(row['level'])
389
390    def addEntry(self, obj, row, site):
391        parent = self.getParent(row, site)
392        obj.level = int(row['level'])
393        parent[row['level']] = obj
394        return
395
396    def checkConversion(self, row, mode='ignore'):
397        """Validates all values in row.
398        """
399        errs, inv_errs, conv_dict = super(
400            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
401
402        # We have to check if level is a valid integer.
403        # This is not done by the converter.
404        try:
405            level = int(row['level'])
406            if level not in range(0,700,10):
407                errs.append(('level','no valid integer'))
408        except ValueError:
409            errs.append(('level','no integer'))
410        return errs, inv_errs, conv_dict
411
412class CourseTicketProcessor(StudentProcessorBase):
413    """A batch processor for ICourseTicket objects.
414    """
415    grok.implements(IBatchProcessor)
416    grok.provides(IBatchProcessor)
417    grok.context(Interface)
418    util_name = 'courseticketprocessor'
419    grok.name(util_name)
420
421    name = u'CourseTicket Processor'
422    iface = ICourseTicket
423    factory_name = 'waeup.CourseTicket'
424
425    location_fields = []
426    additional_fields = ['level', 'code']
427    additional_headers = ['level', 'code']
428
429    def getParent(self, row, site):
430        student = self._getStudent(row, site)
431        if student is None:
432            return None
433        return student['studycourse'].get(row['level'])
434
435    def getEntry(self, row, site):
436        level = self.getParent(row, site)
437        if level is None:
438            return None
439        return level.get(row['code'])
440
441    def addEntry(self, obj, row, site):
442        parent = self.getParent(row, site)
443        catalog = getUtility(ICatalog, name='courses_catalog')
444        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
445        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
446        obj.dcode = entries[0].__parent__.__parent__.code
447        obj.title = entries[0].title
448        obj.credits = entries[0].credits
449        obj.passmark = entries[0].passmark
450        obj.semester = entries[0].semester
451        parent[row['code']] = obj
452        return
453
454    def checkConversion(self, row, mode='ignore'):
455        """Validates all values in row.
456        """
457        errs, inv_errs, conv_dict = super(
458            CourseTicketProcessor, self).checkConversion(row, mode=mode)
459
460        # We have to check if course really exists.
461        # This is not done by the converter.
462        catalog = getUtility(ICatalog, name='courses_catalog')
463        entries = catalog.searchResults(code=(row['code'],row['code']))
464        if len(entries) == 0:
465            errs.append(('code','non-existent'))
466            return errs, inv_errs, conv_dict
467        return errs, inv_errs, conv_dict
468
469class StudentOnlinePaymentProcessor(StudentProcessorBase):
470    """A batch processor for IStudentOnlinePayment objects.
471    """
472    grok.implements(IBatchProcessor)
473    grok.provides(IBatchProcessor)
474    grok.context(Interface)
475    util_name = 'paymentprocessor'
476    grok.name(util_name)
477
478    name = u'Payment Processor'
479    iface = IStudentOnlinePayment
480    factory_name = 'waeup.StudentOnlinePayment'
481
482    location_fields = []
483    additional_fields = ['p_id']
484    additional_headers = ['p_id']
485
486    def parentsExist(self, row, site):
487        result = self.getParent(row, site) is not None
488        return self.getParent(row, site) is not None
489
490    def getParent(self, row, site):
491        student = self._getStudent(row, site)
492        if student is None:
493            return None
494        return student['payments']
495
496    def getEntry(self, row, site):
497        payments = self.getParent(row, site)
498        if payments is None:
499            return None
500        # We can use the hash symbol at the end of p_id in import files
501        # to avoid annoying automatic number transformation
502        # by Excel or Calc
503        p_id = row['p_id'].strip('#')
504        if p_id.startswith('p'):
505            entry = payments.get(p_id)
506        else:
507            # For data migration from old SRP
508            entry = payments.get('p' + p_id[6:])
509        return entry
510
511    def addEntry(self, obj, row, site):
512        parent = self.getParent(row, site)
513        p_id = row['p_id'].strip('#')
514        if not p_id.startswith('p'):
515            # For data migration from old SRP
516            obj.p_id = 'p' + p_id[6:]
517            parent[obj.p_id] = obj
518        else:
519            parent[p_id] = obj
520        return
521
522    def checkConversion(self, row, mode='ignore'):
523        """Validates all values in row.
524        """
525        errs, inv_errs, conv_dict = super(
526            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
527
528        # We have to check p_id.
529        p_id = row['p_id'].strip('#')
530        if p_id.startswith('p'):
531            if not len(p_id) == 14:
532                errs.append(('p_id','invalid length'))
533                return errs, inv_errs, conv_dict
534        else:
535            if not len(p_id) == 19:
536                errs.append(('p_id','invalid length'))
537                return errs, inv_errs, conv_dict
538        return errs, inv_errs, conv_dict
539
540class StudentVerdictProcessor(StudentStudyCourseProcessor):
541    """A batch processor for verdicts.
542
543    Import verdicts and perform workflow transitions.
544    """
545
546    util_name = 'verdictupdater'
547    grok.name(util_name)
548
549    name = u'Verdict Processor (update only)'
550    iface = IStudentVerdictUpdate
551    factory_name = 'waeup.StudentStudyCourse'
552
553    additional_fields = ['current_session', 'current_level']
554
555    def checkUpdateRequirements(self, obj, row, site):
556        """Checks requirements the studycourse and the student must fulfill
557        before being updated.
558        """
559        # Check if current_levels correspond
560        if obj.current_level != row['current_level']:
561            return 'Current level does not correspond.'
562        # Check if current_sessions correspond
563        if obj.current_session != row['current_session']:
564            return 'Current session does not correspond.'
565        # Check if student is in state REGISTERED
566        if obj.getStudent().state != VALIDATED:
567            return 'Student in wrong state.'
568        return None
569
570    def updateEntry(self, obj, row, site):
571        """Update obj to the values given in row.
572        """
573        # Don't set current_session, current_level
574        vals_to_set = dict((key, val) for key, val in row.items()
575                           if key not in ('current_session','current_level'))
576        items_changed = super(StudentVerdictProcessor, self).updateEntry(
577            obj, vals_to_set, site)
578        parent = self.getParent(row, site)
579        parent.__parent__.logger.info(
580            '%s - Verdict updated: %s'
581            % (parent.student_id, items_changed))
582        # Fire transition
583        IWorkflowInfo(obj.__parent__).fireTransition('return')
584        # Update the students_catalog
585        notify(grok.ObjectModifiedEvent(obj.__parent__))
586        return
Note: See TracBrowser for help on using the repository browser.