source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8614

Last change on this file since 8614 was 8581, checked in by Henrik Bettermann, 12 years ago

We have to customize also those interfaces which are used for string conversion only. Prepare ApplicantProcessor? and StudentProcessor? for customization.
  • Property svn:keywords set to Id
File size: 23.6 KB
Line 
1## $Id: batching.py 8581 2012-05-31 20:19:58Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58    iface_byregnumber = IStudentUpdateByRegNo
59    iface_bymatricnumber = IStudentUpdateByMatricNo
60
61    location_fields = []
62    factory_name = 'waeup.Student'
63
64    @property
65    def available_fields(self):
66        fields = getFields(self.iface)
67        return sorted(list(set(
68            ['student_id','reg_number','matric_number',
69            'password', 'state', 'transition'] + fields.keys())))
70
71    def checkHeaders(self, headerfields, mode='create'):
72        if 'state' in headerfields and 'transition' in headerfields:
73            raise FatalCSVError(
74                "State and transition can't be  imported at the same time!")
75        if not 'reg_number' in headerfields and not 'student_id' \
76            in headerfields and not 'matric_number' in headerfields:
77            raise FatalCSVError(
78                "Need at least columns student_id or reg_number " +
79                "or matric_number for import!")
80        if mode == 'create':
81            for field in self.required_fields:
82                if not field in headerfields:
83                    raise FatalCSVError(
84                        "Need at least columns %s for import!" %
85                        ', '.join(["'%s'" % x for x in self.required_fields]))
86        # Check for fields to be ignored...
87        not_ignored_fields = [x for x in headerfields
88                              if not x.startswith('--')]
89        if len(set(not_ignored_fields)) < len(not_ignored_fields):
90            raise FatalCSVError(
91                "Double headers: each column name may only appear once.")
92        return True
93
94    def parentsExist(self, row, site):
95        return 'students' in site.keys()
96
97    def getLocator(self, row):
98        if row.get('student_id',None) not in (None, IGNORE_MARKER):
99            return 'student_id'
100        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
101            return 'reg_number'
102        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
103            return 'matric_number'
104        else:
105            return None
106
107    # The entry never exists in create mode.
108    def entryExists(self, row, site):
109        return self.getEntry(row, site) is not None
110
111    def getParent(self, row, site):
112        return site['students']
113
114    def getEntry(self, row, site):
115        if not 'students' in site.keys():
116            return None
117        if self.getLocator(row) == 'student_id':
118            if row['student_id'] in site['students']:
119                student = site['students'][row['student_id']]
120                return student
121        elif self.getLocator(row) == 'reg_number':
122            reg_number = row['reg_number']
123            cat = queryUtility(ICatalog, name='students_catalog')
124            results = list(
125                cat.searchResults(reg_number=(reg_number, reg_number)))
126            if results:
127                return results[0]
128        elif self.getLocator(row) == 'matric_number':
129            matric_number = row['matric_number']
130            cat = queryUtility(ICatalog, name='students_catalog')
131            results = list(
132                cat.searchResults(matric_number=(matric_number, matric_number)))
133            if results:
134                return results[0]
135        return None
136
137    def addEntry(self, obj, row, site):
138        parent = self.getParent(row, site)
139        parent.addStudent(obj)
140        # Reset _curr_stud_id if student_id has been imported
141        if self.getLocator(row) == 'student_id':
142            parent._curr_stud_id -= 1
143        # We have to log this if state is provided. If not,
144        # logging is done by the event handler handle_student_added
145        if row.has_key('state'):
146            parent.logger.info('%s - Student record created' % obj.student_id)
147        history = IObjectHistory(obj)
148        history.addMessage(_('Student record created'))
149        return
150
151    def delEntry(self, row, site):
152        student = self.getEntry(row, site)
153        if student is not None:
154            parent = self.getParent(row, site)
155            parent.logger.info('%s - Student removed' % student.student_id)
156            del parent[student.student_id]
157        pass
158
159    def checkUpdateRequirements(self, obj, row, site):
160        """Checks requirements the object must fulfill when being updated.
161
162        This method is not used in case of deleting or adding objects.
163
164        Returns error messages as strings in case of requirement
165        problems.
166        """
167        transition = row.get('transition', IGNORE_MARKER)
168        if transition not in (IGNORE_MARKER, ''):
169            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
170            if transition not in allowed_transitions:
171                return 'Transition not allowed.'
172        return None
173
174    def updateEntry(self, obj, row, site):
175        """Update obj to the values given in row.
176        """
177        items_changed = ''
178
179        # Remove student_id from row if empty
180        if row.has_key('student_id') and row['student_id'] in (
181            None, IGNORE_MARKER):
182            row.pop('student_id')
183
184        # Update password
185        if row.has_key('password'):
186            passwd = row.get('password', IGNORE_MARKER)
187            if passwd not in ('', IGNORE_MARKER):
188                if passwd.startswith('{SSHA}'):
189                    # already encrypted password
190                    obj.password = passwd
191                else:
192                    # not yet encrypted password
193                    IUserAccount(obj).setPassword(passwd)
194                items_changed += ('%s=%s, ' % ('password', passwd))
195            row.pop('password')
196
197        # Update registration state
198        if row.has_key('state'):
199            state = row.get('state', IGNORE_MARKER)
200            if state not in (IGNORE_MARKER, ''):
201                value = row['state']
202                IWorkflowState(obj).setState(value)
203                msg = _("State '${a}' set", mapping = {'a':value})
204                history = IObjectHistory(obj)
205                history.addMessage(msg)
206                items_changed += ('%s=%s, ' % ('state', state))
207            row.pop('state')
208
209        if row.has_key('transition'):
210            transition = row.get('transition', IGNORE_MARKER)
211            if transition not in (IGNORE_MARKER, ''):
212                value = row['transition']
213                IWorkflowInfo(obj).fireTransition(value)
214                items_changed += ('%s=%s, ' % ('transition', transition))
215            row.pop('transition')
216
217        # apply other values...
218        items_changed += super(StudentProcessor, self).updateEntry(
219            obj, row, site)
220
221        # Log actions...
222        parent = self.getParent(row, site)
223        if hasattr(obj,'student_id'):
224            # Update mode: the student exists and we can get the student_id
225            parent.logger.info(
226                '%s - Student record updated: %s'
227                % (obj.student_id, items_changed))
228        else:
229            # Create mode: the student does not yet exist
230            parent.logger.info('Student data imported: %s' % items_changed)
231        return items_changed
232
233    def getMapping(self, path, headerfields, mode):
234        """Get a mapping from CSV file headerfields to actually used fieldnames.
235        """
236        result = dict()
237        reader = csv.reader(open(path, 'rb'))
238        raw_header = reader.next()
239        for num, field in enumerate(headerfields):
240            if field not in ['student_id', 'reg_number', 'matric_number'
241                             ] and mode == 'remove':
242                continue
243            if field == u'--IGNORE--':
244                # Skip ignored columns in failed and finished data files.
245                continue
246            result[raw_header[num]] = field
247        return result
248
249    def checkConversion(self, row, mode='create'):
250        """Validates all values in row.
251        """
252        iface = self.iface
253        if mode in ['update', 'remove']:
254            if self.getLocator(row) == 'reg_number':
255                iface = self.iface_byregnumber
256            elif self.getLocator(row) == 'matric_number':
257                iface = self.iface_bymatricnumber
258        converter = IObjectConverter(iface)
259        errs, inv_errs, conv_dict =  converter.fromStringDict(
260            row, self.factory_name, mode=mode)
261        if row.has_key('transition') and \
262            not row['transition'] in IMPORTABLE_TRANSITIONS:
263            if row['transition'] not in (IGNORE_MARKER, ''):
264                errs.append(('transition','not allowed'))
265        if row.has_key('state') and \
266            not row['state'] in IMPORTABLE_STATES:
267            if row['state'] not in (IGNORE_MARKER, ''):
268                errs.append(('state','not allowed'))
269            else:
270                # state is an attribute of Student and must not
271                # be changed if empty
272                conv_dict['state'] = IGNORE_MARKER
273
274        try:
275            # Correct stud_id counter. As the IConverter for students
276            # creates student objects that are not used afterwards, we
277            # have to fix the site-wide student_id counter.
278            site = grok.getSite()
279            students = site['students']
280            students._curr_stud_id -= 1
281        except (KeyError, TypeError, AttributeError):
282                pass
283        return errs, inv_errs, conv_dict
284
285
286class StudentProcessorBase(BatchProcessor):
287    """A base for student subitem processor.
288
289    Helps reducing redundancy.
290    """
291    grok.baseclass()
292
293    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
294    additional_fields = []
295
296    #: header fields additional required
297    additional_headers = []
298
299    @property
300    def available_fields(self):
301        fields = ['student_id','reg_number','matric_number'
302                  ] + self.additional_fields
303        return sorted(list(set(fields + getFields(
304                self.iface).keys())))
305
306    def checkHeaders(self, headerfields, mode='ignore'):
307        if not 'reg_number' in headerfields and not 'student_id' \
308            in headerfields and not 'matric_number' in headerfields:
309            raise FatalCSVError(
310                "Need at least columns student_id " +
311                "or reg_number or matric_number for import!")
312        for name in self.additional_headers:
313            if not name in headerfields:
314                raise FatalCSVError(
315                    "Need %s for import!" % name)
316
317        # Check for fields to be ignored...
318        not_ignored_fields = [x for x in headerfields
319                              if not x.startswith('--')]
320        if len(set(not_ignored_fields)) < len(not_ignored_fields):
321            raise FatalCSVError(
322                "Double headers: each column name may only appear once.")
323        return True
324
325    def _getStudent(self, row, site):
326        NON_VALUES = ['', IGNORE_MARKER]
327        if not 'students' in site.keys():
328            return None
329        if row.get('student_id', '') not in NON_VALUES:
330            if row['student_id'] in site['students']:
331                student = site['students'][row['student_id']]
332                return student
333        elif row.get('reg_number', '') not in NON_VALUES:
334            reg_number = row['reg_number']
335            cat = queryUtility(ICatalog, name='students_catalog')
336            results = list(
337                cat.searchResults(reg_number=(reg_number, reg_number)))
338            if results:
339                return results[0]
340        elif row.get('matric_number', '') not in NON_VALUES:
341            matric_number = row['matric_number']
342            cat = queryUtility(ICatalog, name='students_catalog')
343            results = list(
344                cat.searchResults(matric_number=(matric_number, matric_number)))
345            if results:
346                return results[0]
347        return None
348
349    def parentsExist(self, row, site):
350        return self.getParent(row, site) is not None
351
352    def entryExists(self, row, site):
353        return self.getEntry(row, site) is not None
354
355    def checkConversion(self, row, mode='ignore'):
356        """Validates all values in row.
357        """
358        converter = IObjectConverter(self.iface)
359        errs, inv_errs, conv_dict =  converter.fromStringDict(
360            row, self.factory_name, mode=mode)
361        return errs, inv_errs, conv_dict
362
363
364class StudentStudyCourseProcessor(StudentProcessorBase):
365    """A batch processor for IStudentStudyCourse objects.
366    """
367    grok.implements(IBatchProcessor)
368    grok.provides(IBatchProcessor)
369    grok.context(Interface)
370    util_name = 'studycourseupdater'
371    grok.name(util_name)
372
373    name = u'StudentStudyCourse Processor (update only)'
374    iface = IStudentStudyCourse
375    factory_name = 'waeup.StudentStudyCourse'
376
377    location_fields = []
378    additional_fields = []
379
380    def getParent(self, row, site):
381        return self._getStudent(row, site)
382
383    def getEntry(self, row, site):
384        student = self.getParent(row, site)
385        if student is None:
386            return None
387        return student.get('studycourse')
388
389    def updateEntry(self, obj, row, site):
390        """Update obj to the values given in row.
391        """
392        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
393            obj, row, site)
394        parent = self.getParent(row, site)
395        parent.__parent__.logger.info(
396            '%s - Study course updated: %s'
397            % (parent.student_id, items_changed))
398        # Update the students_catalog
399        notify(grok.ObjectModifiedEvent(obj.__parent__))
400        return
401
402    def checkConversion(self, row, mode='ignore'):
403        """Validates all values in row.
404        """
405        errs, inv_errs, conv_dict = super(
406            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
407        # We have to check if current_level is in range of certificate.
408        if conv_dict.has_key('certificate'):
409          cert = conv_dict['certificate']
410          if conv_dict['current_level'] < cert.start_level or \
411              conv_dict['current_level'] > cert.end_level+120:
412              errs.append(('current_level','not in range'))
413        return errs, inv_errs, conv_dict
414
415class StudentStudyLevelProcessor(StudentProcessorBase):
416    """A batch processor for IStudentStudyLevel objects.
417    """
418    grok.implements(IBatchProcessor)
419    grok.provides(IBatchProcessor)
420    grok.context(Interface)
421    util_name = 'studylevelprocessor'
422    grok.name(util_name)
423
424    name = u'StudentStudyLevel Processor'
425    iface = IStudentStudyLevel
426    factory_name = 'waeup.StudentStudyLevel'
427
428    location_fields = []
429    additional_fields = ['level']
430    additional_headers = ['level']
431
432    def getParent(self, row, site):
433        student = self._getStudent(row, site)
434        if student is None:
435            return None
436        return student['studycourse']
437
438    def getEntry(self, row, site):
439        studycourse = self.getParent(row, site)
440        if studycourse is None:
441            return None
442        return studycourse.get(row['level'])
443
444    def addEntry(self, obj, row, site):
445        parent = self.getParent(row, site)
446        obj.level = int(row['level'])
447        parent[row['level']] = obj
448        return
449
450    def checkConversion(self, row, mode='ignore'):
451        """Validates all values in row.
452        """
453        errs, inv_errs, conv_dict = super(
454            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
455
456        # We have to check if level is a valid integer.
457        # This is not done by the converter.
458        try:
459            level = int(row['level'])
460            if level not in range(0,700,10):
461                errs.append(('level','no valid integer'))
462        except ValueError:
463            errs.append(('level','no integer'))
464        return errs, inv_errs, conv_dict
465
466class CourseTicketProcessor(StudentProcessorBase):
467    """A batch processor for ICourseTicket objects.
468    """
469    grok.implements(IBatchProcessor)
470    grok.provides(IBatchProcessor)
471    grok.context(Interface)
472    util_name = 'courseticketprocessor'
473    grok.name(util_name)
474
475    name = u'CourseTicket Processor'
476    iface = ICourseTicket
477    factory_name = 'waeup.CourseTicket'
478
479    location_fields = []
480    additional_fields = ['level', 'code']
481    additional_headers = ['level', 'code']
482
483    def getParent(self, row, site):
484        student = self._getStudent(row, site)
485        if student is None:
486            return None
487        return student['studycourse'].get(row['level'])
488
489    def getEntry(self, row, site):
490        level = self.getParent(row, site)
491        if level is None:
492            return None
493        return level.get(row['code'])
494
495    def addEntry(self, obj, row, site):
496        parent = self.getParent(row, site)
497        catalog = getUtility(ICatalog, name='courses_catalog')
498        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
499        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
500        obj.dcode = entries[0].__parent__.__parent__.code
501        obj.title = entries[0].title
502        obj.credits = entries[0].credits
503        obj.passmark = entries[0].passmark
504        obj.semester = entries[0].semester
505        parent[row['code']] = obj
506        return
507
508    def checkConversion(self, row, mode='ignore'):
509        """Validates all values in row.
510        """
511        errs, inv_errs, conv_dict = super(
512            CourseTicketProcessor, self).checkConversion(row, mode=mode)
513
514        # We have to check if course really exists.
515        # This is not done by the converter.
516        catalog = getUtility(ICatalog, name='courses_catalog')
517        entries = catalog.searchResults(code=(row['code'],row['code']))
518        if len(entries) == 0:
519            errs.append(('code','non-existent'))
520            return errs, inv_errs, conv_dict
521        return errs, inv_errs, conv_dict
522
523class StudentOnlinePaymentProcessor(StudentProcessorBase):
524    """A batch processor for IStudentOnlinePayment objects.
525    """
526    grok.implements(IBatchProcessor)
527    grok.provides(IBatchProcessor)
528    grok.context(Interface)
529    util_name = 'paymentprocessor'
530    grok.name(util_name)
531
532    name = u'Payment Processor'
533    iface = IStudentOnlinePayment
534    factory_name = 'waeup.StudentOnlinePayment'
535
536    location_fields = []
537    additional_fields = ['p_id']
538    additional_headers = ['p_id']
539
540    def parentsExist(self, row, site):
541        return self.getParent(row, site) is not None
542
543    def getParent(self, row, site):
544        student = self._getStudent(row, site)
545        if student is None:
546            return None
547        return student['payments']
548
549    def getEntry(self, row, site):
550        payments = self.getParent(row, site)
551        if payments is None:
552            return None
553        # We can use the hash symbol at the end of p_id in import files
554        # to avoid annoying automatic number transformation
555        # by Excel or Calc
556        p_id = row['p_id'].strip('#')
557        if p_id.startswith('p'):
558            entry = payments.get(p_id)
559        else:
560            # For data migration from old SRP
561            entry = payments.get('p' + p_id[6:])
562        return entry
563
564    def addEntry(self, obj, row, site):
565        parent = self.getParent(row, site)
566        p_id = row['p_id'].strip('#')
567        if not p_id.startswith('p'):
568            # For data migration from old SRP
569            obj.p_id = 'p' + p_id[6:]
570            parent[obj.p_id] = obj
571        else:
572            parent[p_id] = obj
573        return
574
575    def checkConversion(self, row, mode='ignore'):
576        """Validates all values in row.
577        """
578        errs, inv_errs, conv_dict = super(
579            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
580
581        # We have to check p_id.
582        p_id = row['p_id'].strip('#')
583        if p_id.startswith('p'):
584            if not len(p_id) == 14:
585                errs.append(('p_id','invalid length'))
586                return errs, inv_errs, conv_dict
587        else:
588            if not len(p_id) == 19:
589                errs.append(('p_id','invalid length'))
590                return errs, inv_errs, conv_dict
591        return errs, inv_errs, conv_dict
592
593class StudentVerdictProcessor(StudentStudyCourseProcessor):
594    """A batch processor for verdicts.
595
596    Import verdicts and perform workflow transitions.
597    """
598
599    util_name = 'verdictupdater'
600    grok.name(util_name)
601
602    name = u'Verdict Processor (update only)'
603    iface = IStudentVerdictUpdate
604    factory_name = 'waeup.StudentStudyCourse'
605
606    additional_fields = ['current_session', 'current_level']
607
608    def checkUpdateRequirements(self, obj, row, site):
609        """Checks requirements the studycourse and the student must fulfill
610        before being updated.
611        """
612        # Check if current_levels correspond
613        if obj.current_level != row['current_level']:
614            return 'Current level does not correspond.'
615        # Check if current_sessions correspond
616        if obj.current_session != row['current_session']:
617            return 'Current session does not correspond.'
618        # Check if student is in state REGISTERED
619        if obj.getStudent().state != VALIDATED:
620            return 'Student in wrong state.'
621        return None
622
623    def updateEntry(self, obj, row, site):
624        """Update obj to the values given in row.
625        """
626        # Don't set current_session, current_level
627        vals_to_set = dict((key, val) for key, val in row.items()
628                           if key not in ('current_session','current_level'))
629        items_changed = super(StudentVerdictProcessor, self).updateEntry(
630            obj, vals_to_set, site)
631        parent = self.getParent(row, site)
632        parent.__parent__.logger.info(
633            '%s - Verdict updated: %s'
634            % (parent.student_id, items_changed))
635        # Fire transition
636        IWorkflowInfo(obj.__parent__).fireTransition('return')
637        # Update the students_catalog
638        notify(grok.ObjectModifiedEvent(obj.__parent__))
639        return
Note: See TracBrowser for help on using the repository browser.