source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8490

Last change on this file since 8490 was 8490, checked in by uli, 12 years ago

Let conversion checks for student imports reset the student id counter.
  • Property svn:keywords set to Id
File size: 23.0 KB
Line 
1## $Id: batching.py 8490 2012-05-22 08:02:09Z uli $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58
59    location_fields = []
60    factory_name = 'waeup.Student'
61
62    @property
63    def available_fields(self):
64        fields = getFields(self.iface)
65        return sorted(list(set(
66            ['student_id','reg_number','matric_number',
67            'password', 'state', 'transition'] + fields.keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if 'state' in headerfields and 'transition' in headerfields:
71            raise FatalCSVError(
72                "State and transition can't be  imported at the same time!")
73        if not 'reg_number' in headerfields and not 'student_id' \
74            in headerfields and not 'matric_number' in headerfields:
75            raise FatalCSVError(
76                "Need at least columns student_id or reg_number " +
77                "or matric_number for import!")
78        if mode == 'create':
79            for field in self.required_fields:
80                if not field in headerfields:
81                    raise FatalCSVError(
82                        "Need at least columns %s for import!" %
83                        ', '.join(["'%s'" % x for x in self.required_fields]))
84        # Check for fields to be ignored...
85        not_ignored_fields = [x for x in headerfields
86                              if not x.startswith('--')]
87        if len(set(not_ignored_fields)) < len(not_ignored_fields):
88            raise FatalCSVError(
89                "Double headers: each column name may only appear once.")
90        return True
91
92    def parentsExist(self, row, site):
93        return 'students' in site.keys()
94
95    def getLocator(self, row):
96        if row.get('student_id',None) not in (None, IGNORE_MARKER):
97            return 'student_id'
98        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
99            return 'reg_number'
100        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
101            return 'matric_number'
102        else:
103            return None
104
105    # The entry never exists in create mode.
106    def entryExists(self, row, site):
107        return self.getEntry(row, site) is not None
108
109    def getParent(self, row, site):
110        return site['students']
111
112    def getEntry(self, row, site):
113        if not 'students' in site.keys():
114            return None
115        if self.getLocator(row) == 'student_id':
116            if row['student_id'] in site['students']:
117                student = site['students'][row['student_id']]
118                return student
119        elif self.getLocator(row) == 'reg_number':
120            reg_number = row['reg_number']
121            cat = queryUtility(ICatalog, name='students_catalog')
122            results = list(
123                cat.searchResults(reg_number=(reg_number, reg_number)))
124            if results:
125                return results[0]
126        elif self.getLocator(row) == 'matric_number':
127            matric_number = row['matric_number']
128            cat = queryUtility(ICatalog, name='students_catalog')
129            results = list(
130                cat.searchResults(matric_number=(matric_number, matric_number)))
131            if results:
132                return results[0]
133        return None
134
135    def addEntry(self, obj, row, site):
136        parent = self.getParent(row, site)
137        parent.addStudent(obj)
138        # We have to log this if state is provided. If not,
139        # logging is done by the event handler handle_student_added
140        if row.has_key('state'):
141            parent.logger.info('%s - Student record created' % obj.student_id)
142        history = IObjectHistory(obj)
143        history.addMessage(_('Student record created'))
144        return
145
146    def delEntry(self, row, site):
147        student = self.getEntry(row, site)
148        if student is not None:
149            parent = self.getParent(row, site)
150            parent.logger.info('%s - Student removed' % student.student_id)
151            del parent[student.student_id]
152        pass
153
154    def checkUpdateRequirements(self, obj, row, site):
155        """Checks requirements the object must fulfill when being updated.
156
157        This method is not used in case of deleting or adding objects.
158
159        Returns error messages as strings in case of requirement
160        problems.
161        """
162        transition = row.get('transition', IGNORE_MARKER)
163        if transition not in (IGNORE_MARKER, ''):
164            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
165            if transition not in allowed_transitions:
166                return 'Transition not allowed.'
167        return None
168
169    def updateEntry(self, obj, row, site):
170        """Update obj to the values given in row.
171        """
172        items_changed = ''
173
174        # Remove student_id from row if empty
175        if row.has_key('student_id') and row['student_id'] in (
176            None, IGNORE_MARKER):
177            row.pop('student_id')
178
179        # Update password
180        passwd = row.get('password', IGNORE_MARKER)
181        if passwd not in ('', IGNORE_MARKER):
182            if passwd.startswith('{SSHA}'):
183                # already encrypted password
184                obj.password = passwd
185            else:
186                # not yet encrypted password
187                IUserAccount(obj).setPassword(passwd)
188            row.pop('password')
189
190        # Update registration state
191        state = row.get('state', IGNORE_MARKER)
192        if state not in (IGNORE_MARKER, ''):
193            value = row['state']
194            IWorkflowState(obj).setState(value)
195            msg = _("State '${a}' set", mapping = {'a':value})
196            history = IObjectHistory(obj)
197            history.addMessage(msg)
198            row.pop('state')
199        transition = row.get('transition', IGNORE_MARKER)
200        if transition not in (IGNORE_MARKER, ''):
201            value = row['transition']
202            IWorkflowInfo(obj).fireTransition(value)
203            row.pop('transition')
204
205        # apply other values...
206        items_changed = super(StudentProcessor, self).updateEntry(
207            obj, row, site)
208
209        # Log actions...
210        parent = self.getParent(row, site)
211        if hasattr(obj,'student_id'):
212            # Update mode: the student exists and we can get the student_id
213            parent.logger.info(
214                '%s - Student record updated: %s'
215                % (obj.student_id, items_changed))
216        else:
217            # Create mode: the student does not yet exist
218            parent.logger.info('Student data imported: %s' % items_changed)
219        return items_changed
220
221    def getMapping(self, path, headerfields, mode):
222        """Get a mapping from CSV file headerfields to actually used fieldnames.
223        """
224        result = dict()
225        reader = csv.reader(open(path, 'rb'))
226        raw_header = reader.next()
227        for num, field in enumerate(headerfields):
228            if field not in ['student_id', 'reg_number', 'matric_number'
229                             ] and mode == 'remove':
230                continue
231            if field == u'--IGNORE--':
232                # Skip ignored columns in failed and finished data files.
233                continue
234            result[raw_header[num]] = field
235        return result
236
237    def checkConversion(self, row, mode='create'):
238        """Validates all values in row.
239        """
240        iface = self.iface
241        if mode in ['update', 'remove']:
242            if self.getLocator(row) == 'reg_number':
243                iface = IStudentUpdateByRegNo
244            elif self.getLocator(row) == 'matric_number':
245                iface = IStudentUpdateByMatricNo
246        converter = IObjectConverter(iface)
247        errs, inv_errs, conv_dict =  converter.fromStringDict(
248            row, self.factory_name, mode=mode)
249        if row.has_key('transition') and \
250            not row['transition'] in IMPORTABLE_TRANSITIONS:
251            if row['transition'] not in (IGNORE_MARKER, ''):
252                errs.append(('transition','not allowed'))
253        if row.has_key('state') and \
254            not row['state'] in IMPORTABLE_STATES:
255            if row['state'] not in (IGNORE_MARKER, ''):
256                errs.append(('state','not allowed'))
257            else:
258                # state is an attribute of Student and must not
259                # be changed if empty
260                conv_dict['state'] = IGNORE_MARKER
261
262        try:
263            # Correct stud_id counter. As the IConverter for students
264            # creates student objects that are not used afterwards, we
265            # have to fix the site-wide student_id counter.
266            site = grok.getSite()
267            students = site['students']
268            students._curr_stud_id -= 1
269        except (KeyError, TypeError, AttributeError):
270                pass
271        return errs, inv_errs, conv_dict
272
273
274class StudentProcessorBase(BatchProcessor):
275    """A base for student subitem processor.
276
277    Helps reducing redundancy.
278    """
279    grok.baseclass()
280
281    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
282    additional_fields = []
283
284    #: header fields additional required
285    additional_headers = []
286
287    @property
288    def available_fields(self):
289        fields = ['student_id','reg_number','matric_number'
290                  ] + self.additional_fields
291        return sorted(list(set(fields + getFields(
292                self.iface).keys())))
293
294    def checkHeaders(self, headerfields, mode='ignore'):
295        if not 'reg_number' in headerfields and not 'student_id' \
296            in headerfields and not 'matric_number' in headerfields:
297            raise FatalCSVError(
298                "Need at least columns student_id " +
299                "or reg_number or matric_number for import!")
300        for name in self.additional_headers:
301            if not name in headerfields:
302                raise FatalCSVError(
303                    "Need %s for import!" % name)
304
305        # Check for fields to be ignored...
306        not_ignored_fields = [x for x in headerfields
307                              if not x.startswith('--')]
308        if len(set(not_ignored_fields)) < len(not_ignored_fields):
309            raise FatalCSVError(
310                "Double headers: each column name may only appear once.")
311        return True
312
313    def _getStudent(self, row, site):
314        NON_VALUES = ['', IGNORE_MARKER]
315        if not 'students' in site.keys():
316            return None
317        if row.get('student_id', '') not in NON_VALUES:
318            if row['student_id'] in site['students']:
319                student = site['students'][row['student_id']]
320                return student
321        elif row.get('reg_number', '') not in NON_VALUES:
322            reg_number = row['reg_number']
323            cat = queryUtility(ICatalog, name='students_catalog')
324            results = list(
325                cat.searchResults(reg_number=(reg_number, reg_number)))
326            if results:
327                return results[0]
328        elif row.get('matric_number', '') not in NON_VALUES:
329            matric_number = row['matric_number']
330            cat = queryUtility(ICatalog, name='students_catalog')
331            results = list(
332                cat.searchResults(matric_number=(matric_number, matric_number)))
333            if results:
334                return results[0]
335        return None
336
337    def parentsExist(self, row, site):
338        return self.getParent(row, site) is not None
339
340    def entryExists(self, row, site):
341        return self.getEntry(row, site) is not None
342
343    def checkConversion(self, row, mode='ignore'):
344        """Validates all values in row.
345        """
346        converter = IObjectConverter(self.iface)
347        errs, inv_errs, conv_dict =  converter.fromStringDict(
348            row, self.factory_name, mode=mode)
349        return errs, inv_errs, conv_dict
350
351
352class StudentStudyCourseProcessor(StudentProcessorBase):
353    """A batch processor for IStudentStudyCourse objects.
354    """
355    grok.implements(IBatchProcessor)
356    grok.provides(IBatchProcessor)
357    grok.context(Interface)
358    util_name = 'studycourseupdater'
359    grok.name(util_name)
360
361    name = u'StudentStudyCourse Processor (update only)'
362    iface = IStudentStudyCourse
363    factory_name = 'waeup.StudentStudyCourse'
364
365    location_fields = []
366    additional_fields = []
367
368    def getParent(self, row, site):
369        return self._getStudent(row, site)
370
371    def getEntry(self, row, site):
372        student = self.getParent(row, site)
373        if student is None:
374            return None
375        return student.get('studycourse')
376
377    def updateEntry(self, obj, row, site):
378        """Update obj to the values given in row.
379        """
380        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
381            obj, row, site)
382        parent = self.getParent(row, site)
383        parent.__parent__.logger.info(
384            '%s - Study course updated: %s'
385            % (parent.student_id, items_changed))
386        # Update the students_catalog
387        notify(grok.ObjectModifiedEvent(obj.__parent__))
388        return
389
390    def checkConversion(self, row, mode='ignore'):
391        """Validates all values in row.
392        """
393        errs, inv_errs, conv_dict = super(
394            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
395        # We have to check if current_level is in range of certificate.
396        if conv_dict.has_key('certificate'):
397          cert = conv_dict['certificate']
398          if conv_dict['current_level'] < cert.start_level or \
399              conv_dict['current_level'] > cert.end_level+120:
400              errs.append(('current_level','not in range'))
401        return errs, inv_errs, conv_dict
402
403class StudentStudyLevelProcessor(StudentProcessorBase):
404    """A batch processor for IStudentStudyLevel objects.
405    """
406    grok.implements(IBatchProcessor)
407    grok.provides(IBatchProcessor)
408    grok.context(Interface)
409    util_name = 'studylevelprocessor'
410    grok.name(util_name)
411
412    name = u'StudentStudyLevel Processor'
413    iface = IStudentStudyLevel
414    factory_name = 'waeup.StudentStudyLevel'
415
416    location_fields = []
417    additional_fields = ['level']
418    additional_headers = ['level']
419
420    def getParent(self, row, site):
421        student = self._getStudent(row, site)
422        if student is None:
423            return None
424        return student['studycourse']
425
426    def getEntry(self, row, site):
427        studycourse = self.getParent(row, site)
428        if studycourse is None:
429            return None
430        return studycourse.get(row['level'])
431
432    def addEntry(self, obj, row, site):
433        parent = self.getParent(row, site)
434        obj.level = int(row['level'])
435        parent[row['level']] = obj
436        return
437
438    def checkConversion(self, row, mode='ignore'):
439        """Validates all values in row.
440        """
441        errs, inv_errs, conv_dict = super(
442            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
443
444        # We have to check if level is a valid integer.
445        # This is not done by the converter.
446        try:
447            level = int(row['level'])
448            if level not in range(0,700,10):
449                errs.append(('level','no valid integer'))
450        except ValueError:
451            errs.append(('level','no integer'))
452        return errs, inv_errs, conv_dict
453
454class CourseTicketProcessor(StudentProcessorBase):
455    """A batch processor for ICourseTicket objects.
456    """
457    grok.implements(IBatchProcessor)
458    grok.provides(IBatchProcessor)
459    grok.context(Interface)
460    util_name = 'courseticketprocessor'
461    grok.name(util_name)
462
463    name = u'CourseTicket Processor'
464    iface = ICourseTicket
465    factory_name = 'waeup.CourseTicket'
466
467    location_fields = []
468    additional_fields = ['level', 'code']
469    additional_headers = ['level', 'code']
470
471    def getParent(self, row, site):
472        student = self._getStudent(row, site)
473        if student is None:
474            return None
475        return student['studycourse'].get(row['level'])
476
477    def getEntry(self, row, site):
478        level = self.getParent(row, site)
479        if level is None:
480            return None
481        return level.get(row['code'])
482
483    def addEntry(self, obj, row, site):
484        parent = self.getParent(row, site)
485        catalog = getUtility(ICatalog, name='courses_catalog')
486        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
487        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
488        obj.dcode = entries[0].__parent__.__parent__.code
489        obj.title = entries[0].title
490        obj.credits = entries[0].credits
491        obj.passmark = entries[0].passmark
492        obj.semester = entries[0].semester
493        parent[row['code']] = obj
494        return
495
496    def checkConversion(self, row, mode='ignore'):
497        """Validates all values in row.
498        """
499        errs, inv_errs, conv_dict = super(
500            CourseTicketProcessor, self).checkConversion(row, mode=mode)
501
502        # We have to check if course really exists.
503        # This is not done by the converter.
504        catalog = getUtility(ICatalog, name='courses_catalog')
505        entries = catalog.searchResults(code=(row['code'],row['code']))
506        if len(entries) == 0:
507            errs.append(('code','non-existent'))
508            return errs, inv_errs, conv_dict
509        return errs, inv_errs, conv_dict
510
511class StudentOnlinePaymentProcessor(StudentProcessorBase):
512    """A batch processor for IStudentOnlinePayment objects.
513    """
514    grok.implements(IBatchProcessor)
515    grok.provides(IBatchProcessor)
516    grok.context(Interface)
517    util_name = 'paymentprocessor'
518    grok.name(util_name)
519
520    name = u'Payment Processor'
521    iface = IStudentOnlinePayment
522    factory_name = 'waeup.StudentOnlinePayment'
523
524    location_fields = []
525    additional_fields = ['p_id']
526    additional_headers = ['p_id']
527
528    def parentsExist(self, row, site):
529        return self.getParent(row, site) is not None
530
531    def getParent(self, row, site):
532        student = self._getStudent(row, site)
533        if student is None:
534            return None
535        return student['payments']
536
537    def getEntry(self, row, site):
538        payments = self.getParent(row, site)
539        if payments is None:
540            return None
541        # We can use the hash symbol at the end of p_id in import files
542        # to avoid annoying automatic number transformation
543        # by Excel or Calc
544        p_id = row['p_id'].strip('#')
545        if p_id.startswith('p'):
546            entry = payments.get(p_id)
547        else:
548            # For data migration from old SRP
549            entry = payments.get('p' + p_id[6:])
550        return entry
551
552    def addEntry(self, obj, row, site):
553        parent = self.getParent(row, site)
554        p_id = row['p_id'].strip('#')
555        if not p_id.startswith('p'):
556            # For data migration from old SRP
557            obj.p_id = 'p' + p_id[6:]
558            parent[obj.p_id] = obj
559        else:
560            parent[p_id] = obj
561        return
562
563    def checkConversion(self, row, mode='ignore'):
564        """Validates all values in row.
565        """
566        errs, inv_errs, conv_dict = super(
567            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
568
569        # We have to check p_id.
570        p_id = row['p_id'].strip('#')
571        if p_id.startswith('p'):
572            if not len(p_id) == 14:
573                errs.append(('p_id','invalid length'))
574                return errs, inv_errs, conv_dict
575        else:
576            if not len(p_id) == 19:
577                errs.append(('p_id','invalid length'))
578                return errs, inv_errs, conv_dict
579        return errs, inv_errs, conv_dict
580
581class StudentVerdictProcessor(StudentStudyCourseProcessor):
582    """A batch processor for verdicts.
583
584    Import verdicts and perform workflow transitions.
585    """
586
587    util_name = 'verdictupdater'
588    grok.name(util_name)
589
590    name = u'Verdict Processor (update only)'
591    iface = IStudentVerdictUpdate
592    factory_name = 'waeup.StudentStudyCourse'
593
594    additional_fields = ['current_session', 'current_level']
595
596    def checkUpdateRequirements(self, obj, row, site):
597        """Checks requirements the studycourse and the student must fulfill
598        before being updated.
599        """
600        # Check if current_levels correspond
601        if obj.current_level != row['current_level']:
602            return 'Current level does not correspond.'
603        # Check if current_sessions correspond
604        if obj.current_session != row['current_session']:
605            return 'Current session does not correspond.'
606        # Check if student is in state REGISTERED
607        if obj.getStudent().state != VALIDATED:
608            return 'Student in wrong state.'
609        return None
610
611    def updateEntry(self, obj, row, site):
612        """Update obj to the values given in row.
613        """
614        # Don't set current_session, current_level
615        vals_to_set = dict((key, val) for key, val in row.items()
616                           if key not in ('current_session','current_level'))
617        items_changed = super(StudentVerdictProcessor, self).updateEntry(
618            obj, vals_to_set, site)
619        parent = self.getParent(row, site)
620        parent.__parent__.logger.info(
621            '%s - Verdict updated: %s'
622            % (parent.student_id, items_changed))
623        # Fire transition
624        IWorkflowInfo(obj.__parent__).fireTransition('return')
625        # Update the students_catalog
626        notify(grok.ObjectModifiedEvent(obj.__parent__))
627        return
Note: See TracBrowser for help on using the repository browser.