source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8488

Last change on this file since 8488 was 8349, checked in by uli, 13 years ago

pyflakes.
  • Property svn:keywords set to Id
File size: 22.6 KB
Line 
1## $Id: batching.py 8349 2012-05-04 22:49:57Z uli $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
34from waeup.kofa.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory, VALIDATED, IGNORE_MARKER)
37from waeup.kofa.interfaces import MessageFactory as _
38from waeup.kofa.students.interfaces import (
39    IStudent, IStudentStudyCourse,
40    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
41    IStudentStudyLevel, ICourseTicket,
42    IStudentOnlinePayment, IStudentVerdictUpdate)
43from waeup.kofa.students.workflow import  (
44    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
45from waeup.kofa.utils.batching import BatchProcessor
46
47class StudentProcessor(BatchProcessor):
48    """A batch processor for IStudent objects.
49    """
50    grok.implements(IBatchProcessor)
51    grok.provides(IBatchProcessor)
52    grok.context(Interface)
53    util_name = 'studentprocessor'
54    grok.name(util_name)
55
56    name = u'Student Processor'
57    iface = IStudent
58
59    location_fields = []
60    factory_name = 'waeup.Student'
61
62    @property
63    def available_fields(self):
64        fields = getFields(self.iface)
65        return sorted(list(set(
66            ['student_id','reg_number','matric_number',
67            'password', 'state', 'transition'] + fields.keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if 'state' in headerfields and 'transition' in headerfields:
71            raise FatalCSVError(
72                "State and transition can't be  imported at the same time!")
73        if not 'reg_number' in headerfields and not 'student_id' \
74            in headerfields and not 'matric_number' in headerfields:
75            raise FatalCSVError(
76                "Need at least columns student_id or reg_number " +
77                "or matric_number for import!")
78        if mode == 'create':
79            for field in self.required_fields:
80                if not field in headerfields:
81                    raise FatalCSVError(
82                        "Need at least columns %s for import!" %
83                        ', '.join(["'%s'" % x for x in self.required_fields]))
84        # Check for fields to be ignored...
85        not_ignored_fields = [x for x in headerfields
86                              if not x.startswith('--')]
87        if len(set(not_ignored_fields)) < len(not_ignored_fields):
88            raise FatalCSVError(
89                "Double headers: each column name may only appear once.")
90        return True
91
92    def parentsExist(self, row, site):
93        return 'students' in site.keys()
94
95    def getLocator(self, row):
96        if row.get('student_id',None) not in (None, IGNORE_MARKER):
97            return 'student_id'
98        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
99            return 'reg_number'
100        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
101            return 'matric_number'
102        else:
103            return None
104
105    # The entry never exists in create mode.
106    def entryExists(self, row, site):
107        return self.getEntry(row, site) is not None
108
109    def getParent(self, row, site):
110        return site['students']
111
112    def getEntry(self, row, site):
113        if not 'students' in site.keys():
114            return None
115        if self.getLocator(row) == 'student_id':
116            if row['student_id'] in site['students']:
117                student = site['students'][row['student_id']]
118                return student
119        elif self.getLocator(row) == 'reg_number':
120            reg_number = row['reg_number']
121            cat = queryUtility(ICatalog, name='students_catalog')
122            results = list(
123                cat.searchResults(reg_number=(reg_number, reg_number)))
124            if results:
125                return results[0]
126        elif self.getLocator(row) == 'matric_number':
127            matric_number = row['matric_number']
128            cat = queryUtility(ICatalog, name='students_catalog')
129            results = list(
130                cat.searchResults(matric_number=(matric_number, matric_number)))
131            if results:
132                return results[0]
133        return None
134
135    def addEntry(self, obj, row, site):
136        parent = self.getParent(row, site)
137        parent.addStudent(obj)
138        # We have to log this if state is provided. If not,
139        # logging is done by the event handler handle_student_added
140        if row.has_key('state'):
141            parent.logger.info('%s - Student record created' % obj.student_id)
142        history = IObjectHistory(obj)
143        history.addMessage(_('Student record created'))
144        return
145
146    def delEntry(self, row, site):
147        student = self.getEntry(row, site)
148        if student is not None:
149            parent = self.getParent(row, site)
150            parent.logger.info('%s - Student removed' % student.student_id)
151            del parent[student.student_id]
152        pass
153
154    def checkUpdateRequirements(self, obj, row, site):
155        """Checks requirements the object must fulfill when being updated.
156
157        This method is not used in case of deleting or adding objects.
158
159        Returns error messages as strings in case of requirement
160        problems.
161        """
162        transition = row.get('transition', IGNORE_MARKER)
163        if transition not in (IGNORE_MARKER, ''):
164            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
165            if transition not in allowed_transitions:
166                return 'Transition not allowed.'
167        return None
168
169    def updateEntry(self, obj, row, site):
170        """Update obj to the values given in row.
171        """
172        items_changed = ''
173
174        # Remove student_id from row if empty
175        if row.has_key('student_id') and row['student_id'] in (
176            None, IGNORE_MARKER):
177            row.pop('student_id')
178
179        # Update password
180        passwd = row.get('password', IGNORE_MARKER)
181        if passwd not in ('', IGNORE_MARKER):
182            if passwd.startswith('{SSHA}'):
183                # already encrypted password
184                obj.password = passwd
185            else:
186                # not yet encrypted password
187                IUserAccount(obj).setPassword(passwd)
188            row.pop('password')
189
190        # Update registration state
191        state = row.get('state', IGNORE_MARKER)
192        if state not in (IGNORE_MARKER, ''):
193            value = row['state']
194            IWorkflowState(obj).setState(value)
195            msg = _("State '${a}' set", mapping = {'a':value})
196            history = IObjectHistory(obj)
197            history.addMessage(msg)
198            row.pop('state')
199        transition = row.get('transition', IGNORE_MARKER)
200        if transition not in (IGNORE_MARKER, ''):
201            value = row['transition']
202            IWorkflowInfo(obj).fireTransition(value)
203            row.pop('transition')
204
205        # apply other values...
206        items_changed = super(StudentProcessor, self).updateEntry(
207            obj, row, site)
208
209        # Log actions...
210        parent = self.getParent(row, site)
211        if hasattr(obj,'student_id'):
212            # Update mode: the student exists and we can get the student_id
213            parent.logger.info(
214                '%s - Student record updated: %s'
215                % (obj.student_id, items_changed))
216        else:
217            # Create mode: the student does not yet exist
218            parent.logger.info('Student data imported: %s' % items_changed)
219        return items_changed
220
221    def getMapping(self, path, headerfields, mode):
222        """Get a mapping from CSV file headerfields to actually used fieldnames.
223        """
224        result = dict()
225        reader = csv.reader(open(path, 'rb'))
226        raw_header = reader.next()
227        for num, field in enumerate(headerfields):
228            if field not in ['student_id', 'reg_number', 'matric_number'
229                             ] and mode == 'remove':
230                continue
231            if field == u'--IGNORE--':
232                # Skip ignored columns in failed and finished data files.
233                continue
234            result[raw_header[num]] = field
235        return result
236
237    def checkConversion(self, row, mode='create'):
238        """Validates all values in row.
239        """
240        iface = self.iface
241        if mode in ['update', 'remove']:
242            if self.getLocator(row) == 'reg_number':
243                iface = IStudentUpdateByRegNo
244            elif self.getLocator(row) == 'matric_number':
245                iface = IStudentUpdateByMatricNo
246        converter = IObjectConverter(iface)
247        errs, inv_errs, conv_dict =  converter.fromStringDict(
248            row, self.factory_name, mode=mode)
249        if row.has_key('transition') and \
250            not row['transition'] in IMPORTABLE_TRANSITIONS:
251            if row['transition'] not in (IGNORE_MARKER, ''):
252                errs.append(('transition','not allowed'))
253        if row.has_key('state') and \
254            not row['state'] in IMPORTABLE_STATES:
255            if row['state'] not in (IGNORE_MARKER, ''):
256                errs.append(('state','not allowed'))
257            else:
258                # state is an attribute of Student and must not
259                # be changed if empty
260                conv_dict['state'] = IGNORE_MARKER
261        return errs, inv_errs, conv_dict
262
263
264class StudentProcessorBase(BatchProcessor):
265    """A base for student subitem processor.
266
267    Helps reducing redundancy.
268    """
269    grok.baseclass()
270
271    #: required fields beside 'student_id', 'reg_number' and 'matric_number'
272    additional_fields = []
273
274    #: header fields additional required
275    additional_headers = []
276
277    @property
278    def available_fields(self):
279        fields = ['student_id','reg_number','matric_number'
280                  ] + self.additional_fields
281        return sorted(list(set(fields + getFields(
282                self.iface).keys())))
283
284    def checkHeaders(self, headerfields, mode='ignore'):
285        if not 'reg_number' in headerfields and not 'student_id' \
286            in headerfields and not 'matric_number' in headerfields:
287            raise FatalCSVError(
288                "Need at least columns student_id " +
289                "or reg_number or matric_number for import!")
290        for name in self.additional_headers:
291            if not name in headerfields:
292                raise FatalCSVError(
293                    "Need %s for import!" % name)
294
295        # Check for fields to be ignored...
296        not_ignored_fields = [x for x in headerfields
297                              if not x.startswith('--')]
298        if len(set(not_ignored_fields)) < len(not_ignored_fields):
299            raise FatalCSVError(
300                "Double headers: each column name may only appear once.")
301        return True
302
303    def _getStudent(self, row, site):
304        NON_VALUES = ['', IGNORE_MARKER]
305        if not 'students' in site.keys():
306            return None
307        if row.get('student_id', '') not in NON_VALUES:
308            if row['student_id'] in site['students']:
309                student = site['students'][row['student_id']]
310                return student
311        elif row.get('reg_number', '') not in NON_VALUES:
312            reg_number = row['reg_number']
313            cat = queryUtility(ICatalog, name='students_catalog')
314            results = list(
315                cat.searchResults(reg_number=(reg_number, reg_number)))
316            if results:
317                return results[0]
318        elif row.get('matric_number', '') not in NON_VALUES:
319            matric_number = row['matric_number']
320            cat = queryUtility(ICatalog, name='students_catalog')
321            results = list(
322                cat.searchResults(matric_number=(matric_number, matric_number)))
323            if results:
324                return results[0]
325        return None
326
327    def parentsExist(self, row, site):
328        return self.getParent(row, site) is not None
329
330    def entryExists(self, row, site):
331        return self.getEntry(row, site) is not None
332
333    def checkConversion(self, row, mode='ignore'):
334        """Validates all values in row.
335        """
336        converter = IObjectConverter(self.iface)
337        errs, inv_errs, conv_dict =  converter.fromStringDict(
338            row, self.factory_name, mode=mode)
339        return errs, inv_errs, conv_dict
340
341
342class StudentStudyCourseProcessor(StudentProcessorBase):
343    """A batch processor for IStudentStudyCourse objects.
344    """
345    grok.implements(IBatchProcessor)
346    grok.provides(IBatchProcessor)
347    grok.context(Interface)
348    util_name = 'studycourseupdater'
349    grok.name(util_name)
350
351    name = u'StudentStudyCourse Processor (update only)'
352    iface = IStudentStudyCourse
353    factory_name = 'waeup.StudentStudyCourse'
354
355    location_fields = []
356    additional_fields = []
357
358    def getParent(self, row, site):
359        return self._getStudent(row, site)
360
361    def getEntry(self, row, site):
362        student = self.getParent(row, site)
363        if student is None:
364            return None
365        return student.get('studycourse')
366
367    def updateEntry(self, obj, row, site):
368        """Update obj to the values given in row.
369        """
370        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
371            obj, row, site)
372        parent = self.getParent(row, site)
373        parent.__parent__.logger.info(
374            '%s - Study course updated: %s'
375            % (parent.student_id, items_changed))
376        # Update the students_catalog
377        notify(grok.ObjectModifiedEvent(obj.__parent__))
378        return
379
380    def checkConversion(self, row, mode='ignore'):
381        """Validates all values in row.
382        """
383        errs, inv_errs, conv_dict = super(
384            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
385        # We have to check if current_level is in range of certificate.
386        if conv_dict.has_key('certificate'):
387          cert = conv_dict['certificate']
388          if conv_dict['current_level'] < cert.start_level or \
389              conv_dict['current_level'] > cert.end_level+120:
390              errs.append(('current_level','not in range'))
391        return errs, inv_errs, conv_dict
392
393class StudentStudyLevelProcessor(StudentProcessorBase):
394    """A batch processor for IStudentStudyLevel objects.
395    """
396    grok.implements(IBatchProcessor)
397    grok.provides(IBatchProcessor)
398    grok.context(Interface)
399    util_name = 'studylevelprocessor'
400    grok.name(util_name)
401
402    name = u'StudentStudyLevel Processor'
403    iface = IStudentStudyLevel
404    factory_name = 'waeup.StudentStudyLevel'
405
406    location_fields = []
407    additional_fields = ['level']
408    additional_headers = ['level']
409
410    def getParent(self, row, site):
411        student = self._getStudent(row, site)
412        if student is None:
413            return None
414        return student['studycourse']
415
416    def getEntry(self, row, site):
417        studycourse = self.getParent(row, site)
418        if studycourse is None:
419            return None
420        return studycourse.get(row['level'])
421
422    def addEntry(self, obj, row, site):
423        parent = self.getParent(row, site)
424        obj.level = int(row['level'])
425        parent[row['level']] = obj
426        return
427
428    def checkConversion(self, row, mode='ignore'):
429        """Validates all values in row.
430        """
431        errs, inv_errs, conv_dict = super(
432            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
433
434        # We have to check if level is a valid integer.
435        # This is not done by the converter.
436        try:
437            level = int(row['level'])
438            if level not in range(0,700,10):
439                errs.append(('level','no valid integer'))
440        except ValueError:
441            errs.append(('level','no integer'))
442        return errs, inv_errs, conv_dict
443
444class CourseTicketProcessor(StudentProcessorBase):
445    """A batch processor for ICourseTicket objects.
446    """
447    grok.implements(IBatchProcessor)
448    grok.provides(IBatchProcessor)
449    grok.context(Interface)
450    util_name = 'courseticketprocessor'
451    grok.name(util_name)
452
453    name = u'CourseTicket Processor'
454    iface = ICourseTicket
455    factory_name = 'waeup.CourseTicket'
456
457    location_fields = []
458    additional_fields = ['level', 'code']
459    additional_headers = ['level', 'code']
460
461    def getParent(self, row, site):
462        student = self._getStudent(row, site)
463        if student is None:
464            return None
465        return student['studycourse'].get(row['level'])
466
467    def getEntry(self, row, site):
468        level = self.getParent(row, site)
469        if level is None:
470            return None
471        return level.get(row['code'])
472
473    def addEntry(self, obj, row, site):
474        parent = self.getParent(row, site)
475        catalog = getUtility(ICatalog, name='courses_catalog')
476        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
477        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
478        obj.dcode = entries[0].__parent__.__parent__.code
479        obj.title = entries[0].title
480        obj.credits = entries[0].credits
481        obj.passmark = entries[0].passmark
482        obj.semester = entries[0].semester
483        parent[row['code']] = obj
484        return
485
486    def checkConversion(self, row, mode='ignore'):
487        """Validates all values in row.
488        """
489        errs, inv_errs, conv_dict = super(
490            CourseTicketProcessor, self).checkConversion(row, mode=mode)
491
492        # We have to check if course really exists.
493        # This is not done by the converter.
494        catalog = getUtility(ICatalog, name='courses_catalog')
495        entries = catalog.searchResults(code=(row['code'],row['code']))
496        if len(entries) == 0:
497            errs.append(('code','non-existent'))
498            return errs, inv_errs, conv_dict
499        return errs, inv_errs, conv_dict
500
501class StudentOnlinePaymentProcessor(StudentProcessorBase):
502    """A batch processor for IStudentOnlinePayment objects.
503    """
504    grok.implements(IBatchProcessor)
505    grok.provides(IBatchProcessor)
506    grok.context(Interface)
507    util_name = 'paymentprocessor'
508    grok.name(util_name)
509
510    name = u'Payment Processor'
511    iface = IStudentOnlinePayment
512    factory_name = 'waeup.StudentOnlinePayment'
513
514    location_fields = []
515    additional_fields = ['p_id']
516    additional_headers = ['p_id']
517
518    def parentsExist(self, row, site):
519        return self.getParent(row, site) is not None
520
521    def getParent(self, row, site):
522        student = self._getStudent(row, site)
523        if student is None:
524            return None
525        return student['payments']
526
527    def getEntry(self, row, site):
528        payments = self.getParent(row, site)
529        if payments is None:
530            return None
531        # We can use the hash symbol at the end of p_id in import files
532        # to avoid annoying automatic number transformation
533        # by Excel or Calc
534        p_id = row['p_id'].strip('#')
535        if p_id.startswith('p'):
536            entry = payments.get(p_id)
537        else:
538            # For data migration from old SRP
539            entry = payments.get('p' + p_id[6:])
540        return entry
541
542    def addEntry(self, obj, row, site):
543        parent = self.getParent(row, site)
544        p_id = row['p_id'].strip('#')
545        if not p_id.startswith('p'):
546            # For data migration from old SRP
547            obj.p_id = 'p' + p_id[6:]
548            parent[obj.p_id] = obj
549        else:
550            parent[p_id] = obj
551        return
552
553    def checkConversion(self, row, mode='ignore'):
554        """Validates all values in row.
555        """
556        errs, inv_errs, conv_dict = super(
557            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
558
559        # We have to check p_id.
560        p_id = row['p_id'].strip('#')
561        if p_id.startswith('p'):
562            if not len(p_id) == 14:
563                errs.append(('p_id','invalid length'))
564                return errs, inv_errs, conv_dict
565        else:
566            if not len(p_id) == 19:
567                errs.append(('p_id','invalid length'))
568                return errs, inv_errs, conv_dict
569        return errs, inv_errs, conv_dict
570
571class StudentVerdictProcessor(StudentStudyCourseProcessor):
572    """A batch processor for verdicts.
573
574    Import verdicts and perform workflow transitions.
575    """
576
577    util_name = 'verdictupdater'
578    grok.name(util_name)
579
580    name = u'Verdict Processor (update only)'
581    iface = IStudentVerdictUpdate
582    factory_name = 'waeup.StudentStudyCourse'
583
584    additional_fields = ['current_session', 'current_level']
585
586    def checkUpdateRequirements(self, obj, row, site):
587        """Checks requirements the studycourse and the student must fulfill
588        before being updated.
589        """
590        # Check if current_levels correspond
591        if obj.current_level != row['current_level']:
592            return 'Current level does not correspond.'
593        # Check if current_sessions correspond
594        if obj.current_session != row['current_session']:
595            return 'Current session does not correspond.'
596        # Check if student is in state REGISTERED
597        if obj.getStudent().state != VALIDATED:
598            return 'Student in wrong state.'
599        return None
600
601    def updateEntry(self, obj, row, site):
602        """Update obj to the values given in row.
603        """
604        # Don't set current_session, current_level
605        vals_to_set = dict((key, val) for key, val in row.items()
606                           if key not in ('current_session','current_level'))
607        items_changed = super(StudentVerdictProcessor, self).updateEntry(
608            obj, vals_to_set, site)
609        parent = self.getParent(row, site)
610        parent.__parent__.logger.info(
611            '%s - Verdict updated: %s'
612            % (parent.student_id, items_changed))
613        # Fire transition
614        IWorkflowInfo(obj.__parent__).fireTransition('return')
615        # Update the students_catalog
616        notify(grok.ObjectModifiedEvent(obj.__parent__))
617        return
Note: See TracBrowser for help on using the repository browser.