source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8291

Last change on this file since 8291 was 8287, checked in by Henrik Bettermann, 13 years ago

Rename reg_state to state to be in accordance with applicants.

  • Property svn:keywords set to Id
File size: 21.3 KB
Line 
1## $Id: batching.py 8287 2012-04-26 14:13:51Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, DELETION_MARKER, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  IMPORTABLE_STATES
44from waeup.kofa.utils.batching import BatchProcessor
45
46class StudentProcessor(BatchProcessor):
47    """A batch processor for IStudent objects.
48    """
49    grok.implements(IBatchProcessor)
50    grok.provides(IBatchProcessor)
51    grok.context(Interface)
52    util_name = 'studentprocessor'
53    grok.name(util_name)
54
55    name = u'Student Processor'
56    iface = IStudent
57
58    location_fields = []
59    factory_name = 'waeup.Student'
60
61    @property
62    def available_fields(self):
63        fields = getFields(self.iface)
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'state'] + getFields(
67                self.iface).keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if not 'reg_number' in headerfields and not 'student_id' \
71            in headerfields and not 'matric_number' in headerfields:
72            raise FatalCSVError(
73                "Need at least columns student_id or reg_number " +
74                "or matric_number for import!")
75        if mode == 'create':
76            for field in self.required_fields:
77                if not field in headerfields:
78                    raise FatalCSVError(
79                        "Need at least columns %s for import!" %
80                        ', '.join(["'%s'" % x for x in self.required_fields]))
81        # Check for fields to be ignored...
82        not_ignored_fields = [x for x in headerfields
83                              if not x.startswith('--')]
84        if len(set(not_ignored_fields)) < len(not_ignored_fields):
85            raise FatalCSVError(
86                "Double headers: each column name may only appear once.")
87        return True
88
89    def parentsExist(self, row, site):
90        return 'students' in site.keys()
91
92    def getLocator(self, row):
93        if row.get('student_id',None) not in (None, IGNORE_MARKER):
94            return 'student_id'
95        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
96            return 'reg_number'
97        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
98            return 'matric_number'
99        else:
100            return None
101
102    # The entry never exists in create mode.
103    def entryExists(self, row, site):
104        return self.getEntry(row, site) is not None
105
106    def getParent(self, row, site):
107        return site['students']
108
109    def getEntry(self, row, site):
110        if not 'students' in site.keys():
111            return None
112        if self.getLocator(row) == 'student_id':
113            if row['student_id'] in site['students']:
114                student = site['students'][row['student_id']]
115                return student
116        elif self.getLocator(row) == 'reg_number':
117            reg_number = row['reg_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(reg_number=(reg_number, reg_number)))
121            if results:
122                return results[0]
123        elif self.getLocator(row) == 'matric_number':
124            matric_number = row['matric_number']
125            cat = queryUtility(ICatalog, name='students_catalog')
126            results = list(
127                cat.searchResults(matric_number=(matric_number, matric_number)))
128            if results:
129                return results[0]
130        return None
131
132    def addEntry(self, obj, row, site):
133        parent = self.getParent(row, site)
134        parent.addStudent(obj)
135        # We have to log this if state is provided. If not,
136        # logging is done by the event handler handle_student_added
137        if row.has_key('state'):
138            parent.logger.info('%s - Student record created' % obj.student_id)
139        history = IObjectHistory(obj)
140        history.addMessage(_('Student record created'))
141        return
142
143    def delEntry(self, row, site):
144        student = self.getEntry(row, site)
145        if student is not None:
146            parent = self.getParent(row, site)
147            parent.logger.info('%s - Student removed' % student.student_id)
148            del parent[student.student_id]
149        pass
150
151    def updateEntry(self, obj, row, site):
152        """Update obj to the values given in row.
153        """
154        items_changed = ''
155
156        # Remove student_id from row if empty
157        if row.has_key('student_id') and row['student_id'] in (
158            None, IGNORE_MARKER):
159            row.pop('student_id')
160
161        # Update password
162        passwd = row.get('password', IGNORE_MARKER)
163        if passwd not in ('', IGNORE_MARKER):
164            IUserAccount(obj).setPassword(passwd)
165            row.pop('password')
166
167        # Update registration state
168        state = row.get('state', IGNORE_MARKER)
169        if state not in (IGNORE_MARKER, ''):
170            value = row['state']
171            IWorkflowState(obj).setState(value)
172            msg = _("State '${a}' set", mapping = {'a':value})
173            history = IObjectHistory(obj)
174            history.addMessage(msg)
175            row.pop('state')
176
177        # apply other values...
178        items_changed = super(StudentProcessor, self).updateEntry(
179            obj, row, site)
180
181        # Log actions...
182        parent = self.getParent(row, site)
183        if hasattr(obj,'student_id'):
184            # Update mode: the student exists and we can get the student_id
185            parent.logger.info(
186                '%s - Student record updated: %s'
187                % (obj.student_id, items_changed))
188        else:
189            # Create mode: the student does not yet exist
190            parent.logger.info('Student data imported: %s' % items_changed)
191        return items_changed
192
193    def getMapping(self, path, headerfields, mode):
194        """Get a mapping from CSV file headerfields to actually used fieldnames.
195        """
196        result = dict()
197        reader = csv.reader(open(path, 'rb'))
198        raw_header = reader.next()
199        for num, field in enumerate(headerfields):
200            if field not in ['student_id', 'reg_number', 'matric_number'
201                             ] and mode == 'remove':
202                continue
203            if field == u'--IGNORE--':
204                # Skip ignored columns in failed and finished data files.
205                continue
206            result[raw_header[num]] = field
207        return result
208
209    def checkConversion(self, row, mode='create'):
210        """Validates all values in row.
211        """
212        iface = self.iface
213        if mode in ['update', 'remove']:
214            if self.getLocator(row) == 'reg_number':
215                iface = IStudentUpdateByRegNo
216            elif self.getLocator(row) == 'matric_number':
217                iface = IStudentUpdateByMatricNo
218        converter = IObjectConverter(iface)
219        errs, inv_errs, conv_dict =  converter.fromStringDict(
220            row, self.factory_name, mode=mode)
221        if row.has_key('state') and \
222            not row['state'] in IMPORTABLE_STATES:
223            if row['state'] not in (IGNORE_MARKER, ''):
224                errs.append(('state','not allowed'))
225            else:
226                # state is an attribute of Student and must not
227                # be changed if empty
228                conv_dict['state'] = IGNORE_MARKER
229        return errs, inv_errs, conv_dict
230
231
232class StudentProcessorBase(BatchProcessor):
233    """A base for student subitem processor.
234
235    Helps reducing redundancy.
236    """
237    grok.baseclass()
238
239    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
240    additional_fields = []
241
242    #: header fields additional required
243    additional_headers = []
244
245    @property
246    def available_fields(self):
247        fields = ['student_id','reg_number','matric_number'
248                  ] + self.additional_fields
249        return sorted(list(set(fields + getFields(
250                self.iface).keys())))
251
252    def checkHeaders(self, headerfields, mode='ignore'):
253        if not 'reg_number' in headerfields and not 'student_id' \
254            in headerfields and not 'matric_number' in headerfields:
255            raise FatalCSVError(
256                "Need at least columns student_id " +
257                "or reg_number or matric_number for import!")
258        for name in self.additional_headers:
259            if not name in headerfields:
260                raise FatalCSVError(
261                    "Need %s for import!" % name)
262
263        # Check for fields to be ignored...
264        not_ignored_fields = [x for x in headerfields
265                              if not x.startswith('--')]
266        if len(set(not_ignored_fields)) < len(not_ignored_fields):
267            raise FatalCSVError(
268                "Double headers: each column name may only appear once.")
269        return True
270
271    def _getStudent(self, row, site):
272        NON_VALUES = ['', IGNORE_MARKER]
273        if not 'students' in site.keys():
274            return None
275        if row.get('student_id', '') not in NON_VALUES:
276            if row['student_id'] in site['students']:
277                student = site['students'][row['student_id']]
278                return student
279        elif row.get('reg_number', '') not in NON_VALUES:
280            reg_number = row['reg_number']
281            cat = queryUtility(ICatalog, name='students_catalog')
282            results = list(
283                cat.searchResults(reg_number=(reg_number, reg_number)))
284            if results:
285                return results[0]
286        elif row.get('matric_number', '') not in NON_VALUES:
287            matric_number = row['matric_number']
288            cat = queryUtility(ICatalog, name='students_catalog')
289            results = list(
290                cat.searchResults(matric_number=(matric_number, matric_number)))
291            if results:
292                return results[0]
293        return None
294
295    def parentsExist(self, row, site):
296        result = self.getParent(row, site) is not None
297        return self.getParent(row, site) is not None
298
299    def entryExists(self, row, site):
300        return self.getEntry(row, site) is not None
301
302    def checkConversion(self, row, mode='ignore'):
303        """Validates all values in row.
304        """
305        converter = IObjectConverter(self.iface)
306        errs, inv_errs, conv_dict =  converter.fromStringDict(
307            row, self.factory_name, mode=mode)
308        return errs, inv_errs, conv_dict
309
310
311class StudentStudyCourseProcessor(StudentProcessorBase):
312    """A batch processor for IStudentStudyCourse objects.
313    """
314    grok.implements(IBatchProcessor)
315    grok.provides(IBatchProcessor)
316    grok.context(Interface)
317    util_name = 'studycourseupdater'
318    grok.name(util_name)
319
320    name = u'StudentStudyCourse Processor (update only)'
321    iface = IStudentStudyCourse
322    factory_name = 'waeup.StudentStudyCourse'
323
324    location_fields = []
325    additional_fields = []
326
327    def getParent(self, row, site):
328        return self._getStudent(row, site)
329
330    def getEntry(self, row, site):
331        student = self.getParent(row, site)
332        if student is None:
333            return None
334        return student.get('studycourse')
335
336    def updateEntry(self, obj, row, site):
337        """Update obj to the values given in row.
338        """
339        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
340            obj, row, site)
341        parent = self.getParent(row, site)
342        parent.__parent__.logger.info(
343            '%s - Study course updated: %s'
344            % (parent.student_id, items_changed))
345        # Update the students_catalog
346        notify(grok.ObjectModifiedEvent(obj.__parent__))
347        return
348
349    def checkConversion(self, row, mode='ignore'):
350        """Validates all values in row.
351        """
352        errs, inv_errs, conv_dict = super(
353            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
354        # We have to check if current_level is in range of certificate.
355        if conv_dict.has_key('certificate'):
356          cert = conv_dict['certificate']
357          if conv_dict['current_level'] < cert.start_level or \
358              conv_dict['current_level'] > cert.end_level+120:
359              errs.append(('current_level','not in range'))
360        return errs, inv_errs, conv_dict
361
362class StudentStudyLevelProcessor(StudentProcessorBase):
363    """A batch processor for IStudentStudyLevel objects.
364    """
365    grok.implements(IBatchProcessor)
366    grok.provides(IBatchProcessor)
367    grok.context(Interface)
368    util_name = 'studylevelprocessor'
369    grok.name(util_name)
370
371    name = u'StudentStudyLevel Processor'
372    iface = IStudentStudyLevel
373    factory_name = 'waeup.StudentStudyLevel'
374
375    location_fields = []
376    additional_fields = ['level']
377    additional_headers = ['level']
378
379    def getParent(self, row, site):
380        student = self._getStudent(row, site)
381        if student is None:
382            return None
383        return student['studycourse']
384
385    def getEntry(self, row, site):
386        studycourse = self.getParent(row, site)
387        if studycourse is None:
388            return None
389        return studycourse.get(row['level'])
390
391    def addEntry(self, obj, row, site):
392        parent = self.getParent(row, site)
393        obj.level = int(row['level'])
394        parent[row['level']] = obj
395        return
396
397    def checkConversion(self, row, mode='ignore'):
398        """Validates all values in row.
399        """
400        errs, inv_errs, conv_dict = super(
401            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
402
403        # We have to check if level is a valid integer.
404        # This is not done by the converter.
405        try:
406            level = int(row['level'])
407            if level not in range(0,700,10):
408                errs.append(('level','no valid integer'))
409        except ValueError:
410            errs.append(('level','no integer'))
411        return errs, inv_errs, conv_dict
412
413class CourseTicketProcessor(StudentProcessorBase):
414    """A batch processor for ICourseTicket objects.
415    """
416    grok.implements(IBatchProcessor)
417    grok.provides(IBatchProcessor)
418    grok.context(Interface)
419    util_name = 'courseticketprocessor'
420    grok.name(util_name)
421
422    name = u'CourseTicket Processor'
423    iface = ICourseTicket
424    factory_name = 'waeup.CourseTicket'
425
426    location_fields = []
427    additional_fields = ['level', 'code']
428    additional_headers = ['level', 'code']
429
430    def getParent(self, row, site):
431        student = self._getStudent(row, site)
432        if student is None:
433            return None
434        return student['studycourse'].get(row['level'])
435
436    def getEntry(self, row, site):
437        level = self.getParent(row, site)
438        if level is None:
439            return None
440        return level.get(row['code'])
441
442    def addEntry(self, obj, row, site):
443        parent = self.getParent(row, site)
444        catalog = getUtility(ICatalog, name='courses_catalog')
445        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
446        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
447        obj.dcode = entries[0].__parent__.__parent__.code
448        obj.title = entries[0].title
449        obj.credits = entries[0].credits
450        obj.passmark = entries[0].passmark
451        obj.semester = entries[0].semester
452        parent[row['code']] = obj
453        return
454
455    def checkConversion(self, row, mode='ignore'):
456        """Validates all values in row.
457        """
458        errs, inv_errs, conv_dict = super(
459            CourseTicketProcessor, self).checkConversion(row, mode=mode)
460
461        # We have to check if course really exists.
462        # This is not done by the converter.
463        catalog = getUtility(ICatalog, name='courses_catalog')
464        entries = catalog.searchResults(code=(row['code'],row['code']))
465        if len(entries) == 0:
466            errs.append(('code','non-existent'))
467            return errs, inv_errs, conv_dict
468        return errs, inv_errs, conv_dict
469
470class StudentOnlinePaymentProcessor(StudentProcessorBase):
471    """A batch processor for IStudentOnlinePayment objects.
472    """
473    grok.implements(IBatchProcessor)
474    grok.provides(IBatchProcessor)
475    grok.context(Interface)
476    util_name = 'paymentprocessor'
477    grok.name(util_name)
478
479    name = u'Payment Processor'
480    iface = IStudentOnlinePayment
481    factory_name = 'waeup.StudentOnlinePayment'
482
483    location_fields = []
484    additional_fields = ['p_id']
485    additional_headers = ['p_id']
486
487    def parentsExist(self, row, site):
488        result = self.getParent(row, site) is not None
489        return self.getParent(row, site) is not None
490
491    def getParent(self, row, site):
492        student = self._getStudent(row, site)
493        if student is None:
494            return None
495        return student['payments']
496
497    def getEntry(self, row, site):
498        payments = self.getParent(row, site)
499        if payments is None:
500            return None
501        # We can use the hash symbol at the end of p_id in import files
502        # to avoid annoying automatic number transformation
503        # by Excel or Calc
504        p_id = row['p_id'].strip('#')
505        if p_id.startswith('p'):
506            entry = payments.get(p_id)
507        else:
508            # For data migration from old SRP
509            entry = payments.get('p' + p_id[6:])
510        return entry
511
512    def addEntry(self, obj, row, site):
513        parent = self.getParent(row, site)
514        p_id = row['p_id'].strip('#')
515        if not p_id.startswith('p'):
516            # For data migration from old SRP
517            obj.p_id = 'p' + p_id[6:]
518            parent[obj.p_id] = obj
519        else:
520            parent[p_id] = obj
521        return
522
523    def checkConversion(self, row, mode='ignore'):
524        """Validates all values in row.
525        """
526        errs, inv_errs, conv_dict = super(
527            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
528
529        # We have to check p_id.
530        p_id = row['p_id'].strip('#')
531        if p_id.startswith('p'):
532            if not len(p_id) == 14:
533                errs.append(('p_id','invalid length'))
534                return errs, inv_errs, conv_dict
535        else:
536            if not len(p_id) == 19:
537                errs.append(('p_id','invalid length'))
538                return errs, inv_errs, conv_dict
539        return errs, inv_errs, conv_dict
540
541class StudentVerdictProcessor(StudentStudyCourseProcessor):
542    """A batch processor for verdicts.
543
544    Import verdicts and perform workflow transitions.
545    """
546
547    util_name = 'verdictupdater'
548    grok.name(util_name)
549
550    name = u'Verdict Processor (update only)'
551    iface = IStudentVerdictUpdate
552    factory_name = 'waeup.StudentStudyCourse'
553
554    additional_fields = ['current_session', 'current_level']
555
556    def checkUpdateRequirements(self, obj, row, site):
557        """Checks requirements the studycourse and the student must fulfill
558        before being updated.
559        """
560        # Check if current_levels correspond
561        if obj.current_level != row['current_level']:
562            return 'Current level does not correspond.'
563        # Check if current_sessions correspond
564        if obj.current_session != row['current_session']:
565            return 'Current session does not correspond.'
566        # Check if student is in state REGISTERED
567        if obj.getStudent().state != VALIDATED:
568            return 'Student in wrong state.'
569        return None
570
571    def updateEntry(self, obj, row, site):
572        """Update obj to the values given in row.
573        """
574        # Don't set current_session, current_level
575        vals_to_set = dict((key, val) for key, val in row.items()
576                           if key not in ('current_session','current_level'))
577        items_changed = super(StudentVerdictProcessor, self).updateEntry(
578            obj, vals_to_set, site)
579        parent = self.getParent(row, site)
580        parent.__parent__.logger.info(
581            '%s - Verdict updated: %s'
582            % (parent.student_id, items_changed))
583        # Fire transition
584        IWorkflowInfo(obj.__parent__).fireTransition('return')
585        # Update the students_catalog
586        notify(grok.ObjectModifiedEvent(obj.__parent__))
587        return
Note: See TracBrowser for help on using the repository browser.