source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8540

Last change on this file since 8540 was 8498, checked in by Henrik Bettermann, 13 years ago

items_changes must include state, password and transition (like in applicants).
  • Property svn:keywords set to Id
File size: 23.5 KB
Line 
1## $Id: batching.py 8498 2012-05-23 07:44:04Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58
59    location_fields = []
60    factory_name = 'waeup.Student'
61
62    @property
63    def available_fields(self):
64        fields = getFields(self.iface)
65        return sorted(list(set(
66            ['student_id','reg_number','matric_number',
67            'password', 'state', 'transition'] + fields.keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if 'state' in headerfields and 'transition' in headerfields:
71            raise FatalCSVError(
72                "State and transition can't be  imported at the same time!")
73        if not 'reg_number' in headerfields and not 'student_id' \
74            in headerfields and not 'matric_number' in headerfields:
75            raise FatalCSVError(
76                "Need at least columns student_id or reg_number " +
77                "or matric_number for import!")
78        if mode == 'create':
79            for field in self.required_fields:
80                if not field in headerfields:
81                    raise FatalCSVError(
82                        "Need at least columns %s for import!" %
83                        ', '.join(["'%s'" % x for x in self.required_fields]))
84        # Check for fields to be ignored...
85        not_ignored_fields = [x for x in headerfields
86                              if not x.startswith('--')]
87        if len(set(not_ignored_fields)) < len(not_ignored_fields):
88            raise FatalCSVError(
89                "Double headers: each column name may only appear once.")
90        return True
91
92    def parentsExist(self, row, site):
93        return 'students' in site.keys()
94
95    def getLocator(self, row):
96        if row.get('student_id',None) not in (None, IGNORE_MARKER):
97            return 'student_id'
98        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
99            return 'reg_number'
100        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
101            return 'matric_number'
102        else:
103            return None
104
105    # The entry never exists in create mode.
106    def entryExists(self, row, site):
107        return self.getEntry(row, site) is not None
108
109    def getParent(self, row, site):
110        return site['students']
111
112    def getEntry(self, row, site):
113        if not 'students' in site.keys():
114            return None
115        if self.getLocator(row) == 'student_id':
116            if row['student_id'] in site['students']:
117                student = site['students'][row['student_id']]
118                return student
119        elif self.getLocator(row) == 'reg_number':
120            reg_number = row['reg_number']
121            cat = queryUtility(ICatalog, name='students_catalog')
122            results = list(
123                cat.searchResults(reg_number=(reg_number, reg_number)))
124            if results:
125                return results[0]
126        elif self.getLocator(row) == 'matric_number':
127            matric_number = row['matric_number']
128            cat = queryUtility(ICatalog, name='students_catalog')
129            results = list(
130                cat.searchResults(matric_number=(matric_number, matric_number)))
131            if results:
132                return results[0]
133        return None
134
135    def addEntry(self, obj, row, site):
136        parent = self.getParent(row, site)
137        parent.addStudent(obj)
138        # Reset _curr_stud_id if student_id has been imported
139        if self.getLocator(row) == 'student_id':
140            parent._curr_stud_id -= 1
141        # We have to log this if state is provided. If not,
142        # logging is done by the event handler handle_student_added
143        if row.has_key('state'):
144            parent.logger.info('%s - Student record created' % obj.student_id)
145        history = IObjectHistory(obj)
146        history.addMessage(_('Student record created'))
147        return
148
149    def delEntry(self, row, site):
150        student = self.getEntry(row, site)
151        if student is not None:
152            parent = self.getParent(row, site)
153            parent.logger.info('%s - Student removed' % student.student_id)
154            del parent[student.student_id]
155        pass
156
157    def checkUpdateRequirements(self, obj, row, site):
158        """Checks requirements the object must fulfill when being updated.
159
160        This method is not used in case of deleting or adding objects.
161
162        Returns error messages as strings in case of requirement
163        problems.
164        """
165        transition = row.get('transition', IGNORE_MARKER)
166        if transition not in (IGNORE_MARKER, ''):
167            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
168            if transition not in allowed_transitions:
169                return 'Transition not allowed.'
170        return None
171
172    def updateEntry(self, obj, row, site):
173        """Update obj to the values given in row.
174        """
175        items_changed = ''
176
177        # Remove student_id from row if empty
178        if row.has_key('student_id') and row['student_id'] in (
179            None, IGNORE_MARKER):
180            row.pop('student_id')
181
182        # Update password
183        if row.has_key('password'):
184            passwd = row.get('password', IGNORE_MARKER)
185            if passwd not in ('', IGNORE_MARKER):
186                if passwd.startswith('{SSHA}'):
187                    # already encrypted password
188                    obj.password = passwd
189                else:
190                    # not yet encrypted password
191                    IUserAccount(obj).setPassword(passwd)
192                items_changed += ('%s=%s, ' % ('password', passwd))
193            row.pop('password')
194
195        # Update registration state
196        if row.has_key('state'):
197            state = row.get('state', IGNORE_MARKER)
198            if state not in (IGNORE_MARKER, ''):
199                value = row['state']
200                IWorkflowState(obj).setState(value)
201                msg = _("State '${a}' set", mapping = {'a':value})
202                history = IObjectHistory(obj)
203                history.addMessage(msg)
204                items_changed += ('%s=%s, ' % ('state', state))
205            row.pop('state')
206
207        if row.has_key('transition'):
208            transition = row.get('transition', IGNORE_MARKER)
209            if transition not in (IGNORE_MARKER, ''):
210                value = row['transition']
211                IWorkflowInfo(obj).fireTransition(value)
212                items_changed += ('%s=%s, ' % ('transition', transition))
213            row.pop('transition')
214
215        # apply other values...
216        items_changed += super(StudentProcessor, self).updateEntry(
217            obj, row, site)
218
219        # Log actions...
220        parent = self.getParent(row, site)
221        if hasattr(obj,'student_id'):
222            # Update mode: the student exists and we can get the student_id
223            parent.logger.info(
224                '%s - Student record updated: %s'
225                % (obj.student_id, items_changed))
226        else:
227            # Create mode: the student does not yet exist
228            parent.logger.info('Student data imported: %s' % items_changed)
229        return items_changed
230
231    def getMapping(self, path, headerfields, mode):
232        """Get a mapping from CSV file headerfields to actually used fieldnames.
233        """
234        result = dict()
235        reader = csv.reader(open(path, 'rb'))
236        raw_header = reader.next()
237        for num, field in enumerate(headerfields):
238            if field not in ['student_id', 'reg_number', 'matric_number'
239                             ] and mode == 'remove':
240                continue
241            if field == u'--IGNORE--':
242                # Skip ignored columns in failed and finished data files.
243                continue
244            result[raw_header[num]] = field
245        return result
246
247    def checkConversion(self, row, mode='create'):
248        """Validates all values in row.
249        """
250        iface = self.iface
251        if mode in ['update', 'remove']:
252            if self.getLocator(row) == 'reg_number':
253                iface = IStudentUpdateByRegNo
254            elif self.getLocator(row) == 'matric_number':
255                iface = IStudentUpdateByMatricNo
256        converter = IObjectConverter(iface)
257        errs, inv_errs, conv_dict =  converter.fromStringDict(
258            row, self.factory_name, mode=mode)
259        if row.has_key('transition') and \
260            not row['transition'] in IMPORTABLE_TRANSITIONS:
261            if row['transition'] not in (IGNORE_MARKER, ''):
262                errs.append(('transition','not allowed'))
263        if row.has_key('state') and \
264            not row['state'] in IMPORTABLE_STATES:
265            if row['state'] not in (IGNORE_MARKER, ''):
266                errs.append(('state','not allowed'))
267            else:
268                # state is an attribute of Student and must not
269                # be changed if empty
270                conv_dict['state'] = IGNORE_MARKER
271
272        try:
273            # Correct stud_id counter. As the IConverter for students
274            # creates student objects that are not used afterwards, we
275            # have to fix the site-wide student_id counter.
276            site = grok.getSite()
277            students = site['students']
278            students._curr_stud_id -= 1
279        except (KeyError, TypeError, AttributeError):
280                pass
281        return errs, inv_errs, conv_dict
282
283
284class StudentProcessorBase(BatchProcessor):
285    """A base for student subitem processor.
286
287    Helps reducing redundancy.
288    """
289    grok.baseclass()
290
291    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
292    additional_fields = []
293
294    #: header fields additional required
295    additional_headers = []
296
297    @property
298    def available_fields(self):
299        fields = ['student_id','reg_number','matric_number'
300                  ] + self.additional_fields
301        return sorted(list(set(fields + getFields(
302                self.iface).keys())))
303
304    def checkHeaders(self, headerfields, mode='ignore'):
305        if not 'reg_number' in headerfields and not 'student_id' \
306            in headerfields and not 'matric_number' in headerfields:
307            raise FatalCSVError(
308                "Need at least columns student_id " +
309                "or reg_number or matric_number for import!")
310        for name in self.additional_headers:
311            if not name in headerfields:
312                raise FatalCSVError(
313                    "Need %s for import!" % name)
314
315        # Check for fields to be ignored...
316        not_ignored_fields = [x for x in headerfields
317                              if not x.startswith('--')]
318        if len(set(not_ignored_fields)) < len(not_ignored_fields):
319            raise FatalCSVError(
320                "Double headers: each column name may only appear once.")
321        return True
322
323    def _getStudent(self, row, site):
324        NON_VALUES = ['', IGNORE_MARKER]
325        if not 'students' in site.keys():
326            return None
327        if row.get('student_id', '') not in NON_VALUES:
328            if row['student_id'] in site['students']:
329                student = site['students'][row['student_id']]
330                return student
331        elif row.get('reg_number', '') not in NON_VALUES:
332            reg_number = row['reg_number']
333            cat = queryUtility(ICatalog, name='students_catalog')
334            results = list(
335                cat.searchResults(reg_number=(reg_number, reg_number)))
336            if results:
337                return results[0]
338        elif row.get('matric_number', '') not in NON_VALUES:
339            matric_number = row['matric_number']
340            cat = queryUtility(ICatalog, name='students_catalog')
341            results = list(
342                cat.searchResults(matric_number=(matric_number, matric_number)))
343            if results:
344                return results[0]
345        return None
346
347    def parentsExist(self, row, site):
348        return self.getParent(row, site) is not None
349
350    def entryExists(self, row, site):
351        return self.getEntry(row, site) is not None
352
353    def checkConversion(self, row, mode='ignore'):
354        """Validates all values in row.
355        """
356        converter = IObjectConverter(self.iface)
357        errs, inv_errs, conv_dict =  converter.fromStringDict(
358            row, self.factory_name, mode=mode)
359        return errs, inv_errs, conv_dict
360
361
362class StudentStudyCourseProcessor(StudentProcessorBase):
363    """A batch processor for IStudentStudyCourse objects.
364    """
365    grok.implements(IBatchProcessor)
366    grok.provides(IBatchProcessor)
367    grok.context(Interface)
368    util_name = 'studycourseupdater'
369    grok.name(util_name)
370
371    name = u'StudentStudyCourse Processor (update only)'
372    iface = IStudentStudyCourse
373    factory_name = 'waeup.StudentStudyCourse'
374
375    location_fields = []
376    additional_fields = []
377
378    def getParent(self, row, site):
379        return self._getStudent(row, site)
380
381    def getEntry(self, row, site):
382        student = self.getParent(row, site)
383        if student is None:
384            return None
385        return student.get('studycourse')
386
387    def updateEntry(self, obj, row, site):
388        """Update obj to the values given in row.
389        """
390        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
391            obj, row, site)
392        parent = self.getParent(row, site)
393        parent.__parent__.logger.info(
394            '%s - Study course updated: %s'
395            % (parent.student_id, items_changed))
396        # Update the students_catalog
397        notify(grok.ObjectModifiedEvent(obj.__parent__))
398        return
399
400    def checkConversion(self, row, mode='ignore'):
401        """Validates all values in row.
402        """
403        errs, inv_errs, conv_dict = super(
404            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
405        # We have to check if current_level is in range of certificate.
406        if conv_dict.has_key('certificate'):
407          cert = conv_dict['certificate']
408          if conv_dict['current_level'] < cert.start_level or \
409              conv_dict['current_level'] > cert.end_level+120:
410              errs.append(('current_level','not in range'))
411        return errs, inv_errs, conv_dict
412
413class StudentStudyLevelProcessor(StudentProcessorBase):
414    """A batch processor for IStudentStudyLevel objects.
415    """
416    grok.implements(IBatchProcessor)
417    grok.provides(IBatchProcessor)
418    grok.context(Interface)
419    util_name = 'studylevelprocessor'
420    grok.name(util_name)
421
422    name = u'StudentStudyLevel Processor'
423    iface = IStudentStudyLevel
424    factory_name = 'waeup.StudentStudyLevel'
425
426    location_fields = []
427    additional_fields = ['level']
428    additional_headers = ['level']
429
430    def getParent(self, row, site):
431        student = self._getStudent(row, site)
432        if student is None:
433            return None
434        return student['studycourse']
435
436    def getEntry(self, row, site):
437        studycourse = self.getParent(row, site)
438        if studycourse is None:
439            return None
440        return studycourse.get(row['level'])
441
442    def addEntry(self, obj, row, site):
443        parent = self.getParent(row, site)
444        obj.level = int(row['level'])
445        parent[row['level']] = obj
446        return
447
448    def checkConversion(self, row, mode='ignore'):
449        """Validates all values in row.
450        """
451        errs, inv_errs, conv_dict = super(
452            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
453
454        # We have to check if level is a valid integer.
455        # This is not done by the converter.
456        try:
457            level = int(row['level'])
458            if level not in range(0,700,10):
459                errs.append(('level','no valid integer'))
460        except ValueError:
461            errs.append(('level','no integer'))
462        return errs, inv_errs, conv_dict
463
464class CourseTicketProcessor(StudentProcessorBase):
465    """A batch processor for ICourseTicket objects.
466    """
467    grok.implements(IBatchProcessor)
468    grok.provides(IBatchProcessor)
469    grok.context(Interface)
470    util_name = 'courseticketprocessor'
471    grok.name(util_name)
472
473    name = u'CourseTicket Processor'
474    iface = ICourseTicket
475    factory_name = 'waeup.CourseTicket'
476
477    location_fields = []
478    additional_fields = ['level', 'code']
479    additional_headers = ['level', 'code']
480
481    def getParent(self, row, site):
482        student = self._getStudent(row, site)
483        if student is None:
484            return None
485        return student['studycourse'].get(row['level'])
486
487    def getEntry(self, row, site):
488        level = self.getParent(row, site)
489        if level is None:
490            return None
491        return level.get(row['code'])
492
493    def addEntry(self, obj, row, site):
494        parent = self.getParent(row, site)
495        catalog = getUtility(ICatalog, name='courses_catalog')
496        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
497        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
498        obj.dcode = entries[0].__parent__.__parent__.code
499        obj.title = entries[0].title
500        obj.credits = entries[0].credits
501        obj.passmark = entries[0].passmark
502        obj.semester = entries[0].semester
503        parent[row['code']] = obj
504        return
505
506    def checkConversion(self, row, mode='ignore'):
507        """Validates all values in row.
508        """
509        errs, inv_errs, conv_dict = super(
510            CourseTicketProcessor, self).checkConversion(row, mode=mode)
511
512        # We have to check if course really exists.
513        # This is not done by the converter.
514        catalog = getUtility(ICatalog, name='courses_catalog')
515        entries = catalog.searchResults(code=(row['code'],row['code']))
516        if len(entries) == 0:
517            errs.append(('code','non-existent'))
518            return errs, inv_errs, conv_dict
519        return errs, inv_errs, conv_dict
520
521class StudentOnlinePaymentProcessor(StudentProcessorBase):
522    """A batch processor for IStudentOnlinePayment objects.
523    """
524    grok.implements(IBatchProcessor)
525    grok.provides(IBatchProcessor)
526    grok.context(Interface)
527    util_name = 'paymentprocessor'
528    grok.name(util_name)
529
530    name = u'Payment Processor'
531    iface = IStudentOnlinePayment
532    factory_name = 'waeup.StudentOnlinePayment'
533
534    location_fields = []
535    additional_fields = ['p_id']
536    additional_headers = ['p_id']
537
538    def parentsExist(self, row, site):
539        return self.getParent(row, site) is not None
540
541    def getParent(self, row, site):
542        student = self._getStudent(row, site)
543        if student is None:
544            return None
545        return student['payments']
546
547    def getEntry(self, row, site):
548        payments = self.getParent(row, site)
549        if payments is None:
550            return None
551        # We can use the hash symbol at the end of p_id in import files
552        # to avoid annoying automatic number transformation
553        # by Excel or Calc
554        p_id = row['p_id'].strip('#')
555        if p_id.startswith('p'):
556            entry = payments.get(p_id)
557        else:
558            # For data migration from old SRP
559            entry = payments.get('p' + p_id[6:])
560        return entry
561
562    def addEntry(self, obj, row, site):
563        parent = self.getParent(row, site)
564        p_id = row['p_id'].strip('#')
565        if not p_id.startswith('p'):
566            # For data migration from old SRP
567            obj.p_id = 'p' + p_id[6:]
568            parent[obj.p_id] = obj
569        else:
570            parent[p_id] = obj
571        return
572
573    def checkConversion(self, row, mode='ignore'):
574        """Validates all values in row.
575        """
576        errs, inv_errs, conv_dict = super(
577            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
578
579        # We have to check p_id.
580        p_id = row['p_id'].strip('#')
581        if p_id.startswith('p'):
582            if not len(p_id) == 14:
583                errs.append(('p_id','invalid length'))
584                return errs, inv_errs, conv_dict
585        else:
586            if not len(p_id) == 19:
587                errs.append(('p_id','invalid length'))
588                return errs, inv_errs, conv_dict
589        return errs, inv_errs, conv_dict
590
591class StudentVerdictProcessor(StudentStudyCourseProcessor):
592    """A batch processor for verdicts.
593
594    Import verdicts and perform workflow transitions.
595    """
596
597    util_name = 'verdictupdater'
598    grok.name(util_name)
599
600    name = u'Verdict Processor (update only)'
601    iface = IStudentVerdictUpdate
602    factory_name = 'waeup.StudentStudyCourse'
603
604    additional_fields = ['current_session', 'current_level']
605
606    def checkUpdateRequirements(self, obj, row, site):
607        """Checks requirements the studycourse and the student must fulfill
608        before being updated.
609        """
610        # Check if current_levels correspond
611        if obj.current_level != row['current_level']:
612            return 'Current level does not correspond.'
613        # Check if current_sessions correspond
614        if obj.current_session != row['current_session']:
615            return 'Current session does not correspond.'
616        # Check if student is in state REGISTERED
617        if obj.getStudent().state != VALIDATED:
618            return 'Student in wrong state.'
619        return None
620
621    def updateEntry(self, obj, row, site):
622        """Update obj to the values given in row.
623        """
624        # Don't set current_session, current_level
625        vals_to_set = dict((key, val) for key, val in row.items()
626                           if key not in ('current_session','current_level'))
627        items_changed = super(StudentVerdictProcessor, self).updateEntry(
628            obj, vals_to_set, site)
629        parent = self.getParent(row, site)
630        parent.__parent__.logger.info(
631            '%s - Verdict updated: %s'
632            % (parent.student_id, items_changed))
633        # Fire transition
634        IWorkflowInfo(obj.__parent__).fireTransition('return')
635        # Update the students_catalog
636        notify(grok.ObjectModifiedEvent(obj.__parent__))
637        return
Note: See TracBrowser for help on using the repository browser.