source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8494

Last change on this file since 8494 was 8491, checked in by Henrik Bettermann, 12 years ago

Reset _curr_stud_id if student_id has been imported.
  • Property svn:keywords set to Id
File size: 23.1 KB
Line 
1## $Id: batching.py 8491 2012-05-22 08:39:48Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58
59    location_fields = []
60    factory_name = 'waeup.Student'
61
62    @property
63    def available_fields(self):
64        fields = getFields(self.iface)
65        return sorted(list(set(
66            ['student_id','reg_number','matric_number',
67            'password', 'state', 'transition'] + fields.keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if 'state' in headerfields and 'transition' in headerfields:
71            raise FatalCSVError(
72                "State and transition can't be  imported at the same time!")
73        if not 'reg_number' in headerfields and not 'student_id' \
74            in headerfields and not 'matric_number' in headerfields:
75            raise FatalCSVError(
76                "Need at least columns student_id or reg_number " +
77                "or matric_number for import!")
78        if mode == 'create':
79            for field in self.required_fields:
80                if not field in headerfields:
81                    raise FatalCSVError(
82                        "Need at least columns %s for import!" %
83                        ', '.join(["'%s'" % x for x in self.required_fields]))
84        # Check for fields to be ignored...
85        not_ignored_fields = [x for x in headerfields
86                              if not x.startswith('--')]
87        if len(set(not_ignored_fields)) < len(not_ignored_fields):
88            raise FatalCSVError(
89                "Double headers: each column name may only appear once.")
90        return True
91
92    def parentsExist(self, row, site):
93        return 'students' in site.keys()
94
95    def getLocator(self, row):
96        if row.get('student_id',None) not in (None, IGNORE_MARKER):
97            return 'student_id'
98        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
99            return 'reg_number'
100        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
101            return 'matric_number'
102        else:
103            return None
104
105    # The entry never exists in create mode.
106    def entryExists(self, row, site):
107        return self.getEntry(row, site) is not None
108
109    def getParent(self, row, site):
110        return site['students']
111
112    def getEntry(self, row, site):
113        if not 'students' in site.keys():
114            return None
115        if self.getLocator(row) == 'student_id':
116            if row['student_id'] in site['students']:
117                student = site['students'][row['student_id']]
118                return student
119        elif self.getLocator(row) == 'reg_number':
120            reg_number = row['reg_number']
121            cat = queryUtility(ICatalog, name='students_catalog')
122            results = list(
123                cat.searchResults(reg_number=(reg_number, reg_number)))
124            if results:
125                return results[0]
126        elif self.getLocator(row) == 'matric_number':
127            matric_number = row['matric_number']
128            cat = queryUtility(ICatalog, name='students_catalog')
129            results = list(
130                cat.searchResults(matric_number=(matric_number, matric_number)))
131            if results:
132                return results[0]
133        return None
134
135    def addEntry(self, obj, row, site):
136        parent = self.getParent(row, site)
137        parent.addStudent(obj)
138        # Reset _curr_stud_id if student_id has been imported
139        if self.getLocator(row) == 'student_id':
140            parent._curr_stud_id -= 1
141        # We have to log this if state is provided. If not,
142        # logging is done by the event handler handle_student_added
143        if row.has_key('state'):
144            parent.logger.info('%s - Student record created' % obj.student_id)
145        history = IObjectHistory(obj)
146        history.addMessage(_('Student record created'))
147        return
148
149    def delEntry(self, row, site):
150        student = self.getEntry(row, site)
151        if student is not None:
152            parent = self.getParent(row, site)
153            parent.logger.info('%s - Student removed' % student.student_id)
154            del parent[student.student_id]
155        pass
156
157    def checkUpdateRequirements(self, obj, row, site):
158        """Checks requirements the object must fulfill when being updated.
159
160        This method is not used in case of deleting or adding objects.
161
162        Returns error messages as strings in case of requirement
163        problems.
164        """
165        transition = row.get('transition', IGNORE_MARKER)
166        if transition not in (IGNORE_MARKER, ''):
167            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
168            if transition not in allowed_transitions:
169                return 'Transition not allowed.'
170        return None
171
172    def updateEntry(self, obj, row, site):
173        """Update obj to the values given in row.
174        """
175        items_changed = ''
176
177        # Remove student_id from row if empty
178        if row.has_key('student_id') and row['student_id'] in (
179            None, IGNORE_MARKER):
180            row.pop('student_id')
181
182        # Update password
183        passwd = row.get('password', IGNORE_MARKER)
184        if passwd not in ('', IGNORE_MARKER):
185            if passwd.startswith('{SSHA}'):
186                # already encrypted password
187                obj.password = passwd
188            else:
189                # not yet encrypted password
190                IUserAccount(obj).setPassword(passwd)
191            row.pop('password')
192
193        # Update registration state
194        state = row.get('state', IGNORE_MARKER)
195        if state not in (IGNORE_MARKER, ''):
196            value = row['state']
197            IWorkflowState(obj).setState(value)
198            msg = _("State '${a}' set", mapping = {'a':value})
199            history = IObjectHistory(obj)
200            history.addMessage(msg)
201            row.pop('state')
202        transition = row.get('transition', IGNORE_MARKER)
203        if transition not in (IGNORE_MARKER, ''):
204            value = row['transition']
205            IWorkflowInfo(obj).fireTransition(value)
206            row.pop('transition')
207
208        # apply other values...
209        items_changed = super(StudentProcessor, self).updateEntry(
210            obj, row, site)
211
212        # Log actions...
213        parent = self.getParent(row, site)
214        if hasattr(obj,'student_id'):
215            # Update mode: the student exists and we can get the student_id
216            parent.logger.info(
217                '%s - Student record updated: %s'
218                % (obj.student_id, items_changed))
219        else:
220            # Create mode: the student does not yet exist
221            parent.logger.info('Student data imported: %s' % items_changed)
222        return items_changed
223
224    def getMapping(self, path, headerfields, mode):
225        """Get a mapping from CSV file headerfields to actually used fieldnames.
226        """
227        result = dict()
228        reader = csv.reader(open(path, 'rb'))
229        raw_header = reader.next()
230        for num, field in enumerate(headerfields):
231            if field not in ['student_id', 'reg_number', 'matric_number'
232                             ] and mode == 'remove':
233                continue
234            if field == u'--IGNORE--':
235                # Skip ignored columns in failed and finished data files.
236                continue
237            result[raw_header[num]] = field
238        return result
239
240    def checkConversion(self, row, mode='create'):
241        """Validates all values in row.
242        """
243        iface = self.iface
244        if mode in ['update', 'remove']:
245            if self.getLocator(row) == 'reg_number':
246                iface = IStudentUpdateByRegNo
247            elif self.getLocator(row) == 'matric_number':
248                iface = IStudentUpdateByMatricNo
249        converter = IObjectConverter(iface)
250        errs, inv_errs, conv_dict =  converter.fromStringDict(
251            row, self.factory_name, mode=mode)
252        if row.has_key('transition') and \
253            not row['transition'] in IMPORTABLE_TRANSITIONS:
254            if row['transition'] not in (IGNORE_MARKER, ''):
255                errs.append(('transition','not allowed'))
256        if row.has_key('state') and \
257            not row['state'] in IMPORTABLE_STATES:
258            if row['state'] not in (IGNORE_MARKER, ''):
259                errs.append(('state','not allowed'))
260            else:
261                # state is an attribute of Student and must not
262                # be changed if empty
263                conv_dict['state'] = IGNORE_MARKER
264
265        try:
266            # Correct stud_id counter. As the IConverter for students
267            # creates student objects that are not used afterwards, we
268            # have to fix the site-wide student_id counter.
269            site = grok.getSite()
270            students = site['students']
271            students._curr_stud_id -= 1
272        except (KeyError, TypeError, AttributeError):
273                pass
274        return errs, inv_errs, conv_dict
275
276
277class StudentProcessorBase(BatchProcessor):
278    """A base for student subitem processor.
279
280    Helps reducing redundancy.
281    """
282    grok.baseclass()
283
284    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
285    additional_fields = []
286
287    #: header fields additional required
288    additional_headers = []
289
290    @property
291    def available_fields(self):
292        fields = ['student_id','reg_number','matric_number'
293                  ] + self.additional_fields
294        return sorted(list(set(fields + getFields(
295                self.iface).keys())))
296
297    def checkHeaders(self, headerfields, mode='ignore'):
298        if not 'reg_number' in headerfields and not 'student_id' \
299            in headerfields and not 'matric_number' in headerfields:
300            raise FatalCSVError(
301                "Need at least columns student_id " +
302                "or reg_number or matric_number for import!")
303        for name in self.additional_headers:
304            if not name in headerfields:
305                raise FatalCSVError(
306                    "Need %s for import!" % name)
307
308        # Check for fields to be ignored...
309        not_ignored_fields = [x for x in headerfields
310                              if not x.startswith('--')]
311        if len(set(not_ignored_fields)) < len(not_ignored_fields):
312            raise FatalCSVError(
313                "Double headers: each column name may only appear once.")
314        return True
315
316    def _getStudent(self, row, site):
317        NON_VALUES = ['', IGNORE_MARKER]
318        if not 'students' in site.keys():
319            return None
320        if row.get('student_id', '') not in NON_VALUES:
321            if row['student_id'] in site['students']:
322                student = site['students'][row['student_id']]
323                return student
324        elif row.get('reg_number', '') not in NON_VALUES:
325            reg_number = row['reg_number']
326            cat = queryUtility(ICatalog, name='students_catalog')
327            results = list(
328                cat.searchResults(reg_number=(reg_number, reg_number)))
329            if results:
330                return results[0]
331        elif row.get('matric_number', '') not in NON_VALUES:
332            matric_number = row['matric_number']
333            cat = queryUtility(ICatalog, name='students_catalog')
334            results = list(
335                cat.searchResults(matric_number=(matric_number, matric_number)))
336            if results:
337                return results[0]
338        return None
339
340    def parentsExist(self, row, site):
341        return self.getParent(row, site) is not None
342
343    def entryExists(self, row, site):
344        return self.getEntry(row, site) is not None
345
346    def checkConversion(self, row, mode='ignore'):
347        """Validates all values in row.
348        """
349        converter = IObjectConverter(self.iface)
350        errs, inv_errs, conv_dict =  converter.fromStringDict(
351            row, self.factory_name, mode=mode)
352        return errs, inv_errs, conv_dict
353
354
355class StudentStudyCourseProcessor(StudentProcessorBase):
356    """A batch processor for IStudentStudyCourse objects.
357    """
358    grok.implements(IBatchProcessor)
359    grok.provides(IBatchProcessor)
360    grok.context(Interface)
361    util_name = 'studycourseupdater'
362    grok.name(util_name)
363
364    name = u'StudentStudyCourse Processor (update only)'
365    iface = IStudentStudyCourse
366    factory_name = 'waeup.StudentStudyCourse'
367
368    location_fields = []
369    additional_fields = []
370
371    def getParent(self, row, site):
372        return self._getStudent(row, site)
373
374    def getEntry(self, row, site):
375        student = self.getParent(row, site)
376        if student is None:
377            return None
378        return student.get('studycourse')
379
380    def updateEntry(self, obj, row, site):
381        """Update obj to the values given in row.
382        """
383        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
384            obj, row, site)
385        parent = self.getParent(row, site)
386        parent.__parent__.logger.info(
387            '%s - Study course updated: %s'
388            % (parent.student_id, items_changed))
389        # Update the students_catalog
390        notify(grok.ObjectModifiedEvent(obj.__parent__))
391        return
392
393    def checkConversion(self, row, mode='ignore'):
394        """Validates all values in row.
395        """
396        errs, inv_errs, conv_dict = super(
397            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
398        # We have to check if current_level is in range of certificate.
399        if conv_dict.has_key('certificate'):
400          cert = conv_dict['certificate']
401          if conv_dict['current_level'] < cert.start_level or \
402              conv_dict['current_level'] > cert.end_level+120:
403              errs.append(('current_level','not in range'))
404        return errs, inv_errs, conv_dict
405
406class StudentStudyLevelProcessor(StudentProcessorBase):
407    """A batch processor for IStudentStudyLevel objects.
408    """
409    grok.implements(IBatchProcessor)
410    grok.provides(IBatchProcessor)
411    grok.context(Interface)
412    util_name = 'studylevelprocessor'
413    grok.name(util_name)
414
415    name = u'StudentStudyLevel Processor'
416    iface = IStudentStudyLevel
417    factory_name = 'waeup.StudentStudyLevel'
418
419    location_fields = []
420    additional_fields = ['level']
421    additional_headers = ['level']
422
423    def getParent(self, row, site):
424        student = self._getStudent(row, site)
425        if student is None:
426            return None
427        return student['studycourse']
428
429    def getEntry(self, row, site):
430        studycourse = self.getParent(row, site)
431        if studycourse is None:
432            return None
433        return studycourse.get(row['level'])
434
435    def addEntry(self, obj, row, site):
436        parent = self.getParent(row, site)
437        obj.level = int(row['level'])
438        parent[row['level']] = obj
439        return
440
441    def checkConversion(self, row, mode='ignore'):
442        """Validates all values in row.
443        """
444        errs, inv_errs, conv_dict = super(
445            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
446
447        # We have to check if level is a valid integer.
448        # This is not done by the converter.
449        try:
450            level = int(row['level'])
451            if level not in range(0,700,10):
452                errs.append(('level','no valid integer'))
453        except ValueError:
454            errs.append(('level','no integer'))
455        return errs, inv_errs, conv_dict
456
457class CourseTicketProcessor(StudentProcessorBase):
458    """A batch processor for ICourseTicket objects.
459    """
460    grok.implements(IBatchProcessor)
461    grok.provides(IBatchProcessor)
462    grok.context(Interface)
463    util_name = 'courseticketprocessor'
464    grok.name(util_name)
465
466    name = u'CourseTicket Processor'
467    iface = ICourseTicket
468    factory_name = 'waeup.CourseTicket'
469
470    location_fields = []
471    additional_fields = ['level', 'code']
472    additional_headers = ['level', 'code']
473
474    def getParent(self, row, site):
475        student = self._getStudent(row, site)
476        if student is None:
477            return None
478        return student['studycourse'].get(row['level'])
479
480    def getEntry(self, row, site):
481        level = self.getParent(row, site)
482        if level is None:
483            return None
484        return level.get(row['code'])
485
486    def addEntry(self, obj, row, site):
487        parent = self.getParent(row, site)
488        catalog = getUtility(ICatalog, name='courses_catalog')
489        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
490        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
491        obj.dcode = entries[0].__parent__.__parent__.code
492        obj.title = entries[0].title
493        obj.credits = entries[0].credits
494        obj.passmark = entries[0].passmark
495        obj.semester = entries[0].semester
496        parent[row['code']] = obj
497        return
498
499    def checkConversion(self, row, mode='ignore'):
500        """Validates all values in row.
501        """
502        errs, inv_errs, conv_dict = super(
503            CourseTicketProcessor, self).checkConversion(row, mode=mode)
504
505        # We have to check if course really exists.
506        # This is not done by the converter.
507        catalog = getUtility(ICatalog, name='courses_catalog')
508        entries = catalog.searchResults(code=(row['code'],row['code']))
509        if len(entries) == 0:
510            errs.append(('code','non-existent'))
511            return errs, inv_errs, conv_dict
512        return errs, inv_errs, conv_dict
513
514class StudentOnlinePaymentProcessor(StudentProcessorBase):
515    """A batch processor for IStudentOnlinePayment objects.
516    """
517    grok.implements(IBatchProcessor)
518    grok.provides(IBatchProcessor)
519    grok.context(Interface)
520    util_name = 'paymentprocessor'
521    grok.name(util_name)
522
523    name = u'Payment Processor'
524    iface = IStudentOnlinePayment
525    factory_name = 'waeup.StudentOnlinePayment'
526
527    location_fields = []
528    additional_fields = ['p_id']
529    additional_headers = ['p_id']
530
531    def parentsExist(self, row, site):
532        return self.getParent(row, site) is not None
533
534    def getParent(self, row, site):
535        student = self._getStudent(row, site)
536        if student is None:
537            return None
538        return student['payments']
539
540    def getEntry(self, row, site):
541        payments = self.getParent(row, site)
542        if payments is None:
543            return None
544        # We can use the hash symbol at the end of p_id in import files
545        # to avoid annoying automatic number transformation
546        # by Excel or Calc
547        p_id = row['p_id'].strip('#')
548        if p_id.startswith('p'):
549            entry = payments.get(p_id)
550        else:
551            # For data migration from old SRP
552            entry = payments.get('p' + p_id[6:])
553        return entry
554
555    def addEntry(self, obj, row, site):
556        parent = self.getParent(row, site)
557        p_id = row['p_id'].strip('#')
558        if not p_id.startswith('p'):
559            # For data migration from old SRP
560            obj.p_id = 'p' + p_id[6:]
561            parent[obj.p_id] = obj
562        else:
563            parent[p_id] = obj
564        return
565
566    def checkConversion(self, row, mode='ignore'):
567        """Validates all values in row.
568        """
569        errs, inv_errs, conv_dict = super(
570            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
571
572        # We have to check p_id.
573        p_id = row['p_id'].strip('#')
574        if p_id.startswith('p'):
575            if not len(p_id) == 14:
576                errs.append(('p_id','invalid length'))
577                return errs, inv_errs, conv_dict
578        else:
579            if not len(p_id) == 19:
580                errs.append(('p_id','invalid length'))
581                return errs, inv_errs, conv_dict
582        return errs, inv_errs, conv_dict
583
584class StudentVerdictProcessor(StudentStudyCourseProcessor):
585    """A batch processor for verdicts.
586
587    Import verdicts and perform workflow transitions.
588    """
589
590    util_name = 'verdictupdater'
591    grok.name(util_name)
592
593    name = u'Verdict Processor (update only)'
594    iface = IStudentVerdictUpdate
595    factory_name = 'waeup.StudentStudyCourse'
596
597    additional_fields = ['current_session', 'current_level']
598
599    def checkUpdateRequirements(self, obj, row, site):
600        """Checks requirements the studycourse and the student must fulfill
601        before being updated.
602        """
603        # Check if current_levels correspond
604        if obj.current_level != row['current_level']:
605            return 'Current level does not correspond.'
606        # Check if current_sessions correspond
607        if obj.current_session != row['current_session']:
608            return 'Current session does not correspond.'
609        # Check if student is in state REGISTERED
610        if obj.getStudent().state != VALIDATED:
611            return 'Student in wrong state.'
612        return None
613
614    def updateEntry(self, obj, row, site):
615        """Update obj to the values given in row.
616        """
617        # Don't set current_session, current_level
618        vals_to_set = dict((key, val) for key, val in row.items()
619                           if key not in ('current_session','current_level'))
620        items_changed = super(StudentVerdictProcessor, self).updateEntry(
621            obj, vals_to_set, site)
622        parent = self.getParent(row, site)
623        parent.__parent__.logger.info(
624            '%s - Verdict updated: %s'
625            % (parent.student_id, items_changed))
626        # Fire transition
627        IWorkflowInfo(obj.__parent__).fireTransition('return')
628        # Update the students_catalog
629        notify(grok.ObjectModifiedEvent(obj.__parent__))
630        return
Note: See TracBrowser for help on using the repository browser.