source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 8884

Last change on this file since 8884 was 8884, checked in by Henrik Bettermann, 13 years ago

Import payment tickets without p_id and generate p_id in checkConversion.

Increase resolution of timestamp.
  • Property svn:keywords set to Id
File size: 25.4 KB
Line 
1## $Id: batching.py 8884 2012-07-03 07:13:59Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from time import time
29from zope.interface import Interface
30from zope.schema import getFields
31from zope.component import queryUtility, getUtility
32from zope.event import notify
33from zope.catalog.interfaces import ICatalog
34from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
35from waeup.kofa.interfaces import (
36    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
37    IObjectHistory, VALIDATED, IGNORE_MARKER)
38from waeup.kofa.interfaces import MessageFactory as _
39from waeup.kofa.students.interfaces import (
40    IStudent, IStudentStudyCourse,
41    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
42    IStudentStudyLevel, ICourseTicket,
43    IStudentOnlinePayment, IStudentVerdictUpdate)
44from waeup.kofa.students.workflow import  (
45    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS)
46from waeup.kofa.utils.batching import BatchProcessor
47
48class StudentProcessor(BatchProcessor):
49    """A batch processor for IStudent objects.
50    """
51    grok.implements(IBatchProcessor)
52    grok.provides(IBatchProcessor)
53    grok.context(Interface)
54    util_name = 'studentprocessor'
55    grok.name(util_name)
56
57    name = u'Student Processor'
58    iface = IStudent
59    iface_byregnumber = IStudentUpdateByRegNo
60    iface_bymatricnumber = IStudentUpdateByMatricNo
61
62    location_fields = []
63    factory_name = 'waeup.Student'
64
65    @property
66    def available_fields(self):
67        fields = getFields(self.iface)
68        return sorted(list(set(
69            ['student_id','reg_number','matric_number',
70            'password', 'state', 'transition'] + fields.keys())))
71
72    def checkHeaders(self, headerfields, mode='create'):
73        if 'state' in headerfields and 'transition' in headerfields:
74            raise FatalCSVError(
75                "State and transition can't be  imported at the same time!")
76        if not 'reg_number' in headerfields and not 'student_id' \
77            in headerfields and not 'matric_number' in headerfields:
78            raise FatalCSVError(
79                "Need at least columns student_id or reg_number " +
80                "or matric_number for import!")
81        if mode == 'create':
82            for field in self.required_fields:
83                if not field in headerfields:
84                    raise FatalCSVError(
85                        "Need at least columns %s for import!" %
86                        ', '.join(["'%s'" % x for x in self.required_fields]))
87        # Check for fields to be ignored...
88        not_ignored_fields = [x for x in headerfields
89                              if not x.startswith('--')]
90        if len(set(not_ignored_fields)) < len(not_ignored_fields):
91            raise FatalCSVError(
92                "Double headers: each column name may only appear once.")
93        return True
94
95    def parentsExist(self, row, site):
96        return 'students' in site.keys()
97
98    def getLocator(self, row):
99        if row.get('student_id',None) not in (None, IGNORE_MARKER):
100            return 'student_id'
101        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
102            return 'reg_number'
103        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
104            return 'matric_number'
105        else:
106            return None
107
108    # The entry never exists in create mode.
109    def entryExists(self, row, site):
110        return self.getEntry(row, site) is not None
111
112    def getParent(self, row, site):
113        return site['students']
114
115    def getEntry(self, row, site):
116        if not 'students' in site.keys():
117            return None
118        if self.getLocator(row) == 'student_id':
119            if row['student_id'] in site['students']:
120                student = site['students'][row['student_id']]
121                return student
122        elif self.getLocator(row) == 'reg_number':
123            reg_number = row['reg_number']
124            cat = queryUtility(ICatalog, name='students_catalog')
125            results = list(
126                cat.searchResults(reg_number=(reg_number, reg_number)))
127            if results:
128                return results[0]
129        elif self.getLocator(row) == 'matric_number':
130            matric_number = row['matric_number']
131            cat = queryUtility(ICatalog, name='students_catalog')
132            results = list(
133                cat.searchResults(matric_number=(matric_number, matric_number)))
134            if results:
135                return results[0]
136        return None
137
138    def addEntry(self, obj, row, site):
139        parent = self.getParent(row, site)
140        parent.addStudent(obj)
141        # Reset _curr_stud_id if student_id has been imported
142        if self.getLocator(row) == 'student_id':
143            parent._curr_stud_id -= 1
144        # We have to log this if state is provided. If not,
145        # logging is done by the event handler handle_student_added
146        if row.has_key('state'):
147            parent.logger.info('%s - Student record created' % obj.student_id)
148        history = IObjectHistory(obj)
149        history.addMessage(_('Student record created'))
150        return
151
152    def delEntry(self, row, site):
153        student = self.getEntry(row, site)
154        if student is not None:
155            parent = self.getParent(row, site)
156            parent.logger.info('%s - Student removed' % student.student_id)
157            del parent[student.student_id]
158        pass
159
160    def checkUpdateRequirements(self, obj, row, site):
161        """Checks requirements the object must fulfill when being updated.
162
163        This method is not used in case of deleting or adding objects.
164
165        Returns error messages as strings in case of requirement
166        problems.
167        """
168        transition = row.get('transition', IGNORE_MARKER)
169        if transition not in (IGNORE_MARKER, ''):
170            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
171            if transition not in allowed_transitions:
172                return 'Transition not allowed.'
173        return None
174
175    def updateEntry(self, obj, row, site):
176        """Update obj to the values given in row.
177        """
178        items_changed = ''
179
180        # Remove student_id from row if empty
181        if row.has_key('student_id') and row['student_id'] in (
182            None, IGNORE_MARKER):
183            row.pop('student_id')
184
185        # Update password
186        if row.has_key('password'):
187            passwd = row.get('password', IGNORE_MARKER)
188            if passwd not in ('', IGNORE_MARKER):
189                if passwd.startswith('{SSHA}'):
190                    # already encrypted password
191                    obj.password = passwd
192                else:
193                    # not yet encrypted password
194                    IUserAccount(obj).setPassword(passwd)
195                items_changed += ('%s=%s, ' % ('password', passwd))
196            row.pop('password')
197
198        # Update registration state
199        if row.has_key('state'):
200            state = row.get('state', IGNORE_MARKER)
201            if state not in (IGNORE_MARKER, ''):
202                value = row['state']
203                IWorkflowState(obj).setState(value)
204                msg = _("State '${a}' set", mapping = {'a':value})
205                history = IObjectHistory(obj)
206                history.addMessage(msg)
207                items_changed += ('%s=%s, ' % ('state', state))
208            row.pop('state')
209
210        if row.has_key('transition'):
211            transition = row.get('transition', IGNORE_MARKER)
212            if transition not in (IGNORE_MARKER, ''):
213                value = row['transition']
214                IWorkflowInfo(obj).fireTransition(value)
215                items_changed += ('%s=%s, ' % ('transition', transition))
216            row.pop('transition')
217
218        # apply other values...
219        items_changed += super(StudentProcessor, self).updateEntry(
220            obj, row, site)
221
222        # Log actions...
223        parent = self.getParent(row, site)
224        if hasattr(obj,'student_id'):
225            # Update mode: the student exists and we can get the student_id
226            parent.logger.info(
227                '%s - Student record updated: %s'
228                % (obj.student_id, items_changed))
229        else:
230            # Create mode: the student does not yet exist
231            parent.logger.info('Student data imported: %s' % items_changed)
232        return items_changed
233
234    def getMapping(self, path, headerfields, mode):
235        """Get a mapping from CSV file headerfields to actually used fieldnames.
236        """
237        result = dict()
238        reader = csv.reader(open(path, 'rb'))
239        raw_header = reader.next()
240        for num, field in enumerate(headerfields):
241            if field not in ['student_id', 'reg_number', 'matric_number'
242                             ] and mode == 'remove':
243                continue
244            if field == u'--IGNORE--':
245                # Skip ignored columns in failed and finished data files.
246                continue
247            result[raw_header[num]] = field
248        return result
249
250    def checkConversion(self, row, mode='create'):
251        """Validates all values in row.
252        """
253        iface = self.iface
254        if mode in ['update', 'remove']:
255            if self.getLocator(row) == 'reg_number':
256                iface = self.iface_byregnumber
257            elif self.getLocator(row) == 'matric_number':
258                iface = self.iface_bymatricnumber
259        converter = IObjectConverter(iface)
260        errs, inv_errs, conv_dict =  converter.fromStringDict(
261            row, self.factory_name, mode=mode)
262        if row.has_key('transition') and \
263            not row['transition'] in IMPORTABLE_TRANSITIONS:
264            if row['transition'] not in (IGNORE_MARKER, ''):
265                errs.append(('transition','not allowed'))
266        if row.has_key('state') and \
267            not row['state'] in IMPORTABLE_STATES:
268            if row['state'] not in (IGNORE_MARKER, ''):
269                errs.append(('state','not allowed'))
270            else:
271                # state is an attribute of Student and must not
272                # be changed if empty
273                conv_dict['state'] = IGNORE_MARKER
274
275        try:
276            # Correct stud_id counter. As the IConverter for students
277            # creates student objects that are not used afterwards, we
278            # have to fix the site-wide student_id counter.
279            site = grok.getSite()
280            students = site['students']
281            students._curr_stud_id -= 1
282        except (KeyError, TypeError, AttributeError):
283                pass
284        return errs, inv_errs, conv_dict
285
286
287class StudentProcessorBase(BatchProcessor):
288    """A base for student subitem processor.
289
290    Helps reducing redundancy.
291    """
292    grok.baseclass()
293
294    # additional available  fields
295    # beside 'student_id', 'reg_number' and 'matric_number'
296    additional_fields = []
297
298    #: header fields additionally required
299    additional_headers = []
300
301    @property
302    def available_fields(self):
303        fields = ['student_id','reg_number','matric_number'
304                  ] + self.additional_fields
305        return sorted(list(set(fields + getFields(
306                self.iface).keys())))
307
308    def checkHeaders(self, headerfields, mode='ignore'):
309        if not 'reg_number' in headerfields and not 'student_id' \
310            in headerfields and not 'matric_number' in headerfields:
311            raise FatalCSVError(
312                "Need at least columns student_id " +
313                "or reg_number or matric_number for import!")
314        for name in self.additional_headers:
315            if not name in headerfields:
316                raise FatalCSVError(
317                    "Need %s for import!" % name)
318
319        # Check for fields to be ignored...
320        not_ignored_fields = [x for x in headerfields
321                              if not x.startswith('--')]
322        if len(set(not_ignored_fields)) < len(not_ignored_fields):
323            raise FatalCSVError(
324                "Double headers: each column name may only appear once.")
325        return True
326
327    def _getStudent(self, row, site):
328        NON_VALUES = ['', IGNORE_MARKER]
329        if not 'students' in site.keys():
330            return None
331        if row.get('student_id', '') not in NON_VALUES:
332            if row['student_id'] in site['students']:
333                student = site['students'][row['student_id']]
334                return student
335        elif row.get('reg_number', '') not in NON_VALUES:
336            reg_number = row['reg_number']
337            cat = queryUtility(ICatalog, name='students_catalog')
338            results = list(
339                cat.searchResults(reg_number=(reg_number, reg_number)))
340            if results:
341                return results[0]
342        elif row.get('matric_number', '') not in NON_VALUES:
343            matric_number = row['matric_number']
344            cat = queryUtility(ICatalog, name='students_catalog')
345            results = list(
346                cat.searchResults(matric_number=(matric_number, matric_number)))
347            if results:
348                return results[0]
349        return None
350
351    def parentsExist(self, row, site):
352        return self.getParent(row, site) is not None
353
354    def entryExists(self, row, site):
355        return self.getEntry(row, site) is not None
356
357    def checkConversion(self, row, mode='ignore'):
358        """Validates all values in row.
359        """
360        converter = IObjectConverter(self.iface)
361        errs, inv_errs, conv_dict =  converter.fromStringDict(
362            row, self.factory_name, mode=mode)
363        return errs, inv_errs, conv_dict
364
365
366class StudentStudyCourseProcessor(StudentProcessorBase):
367    """A batch processor for IStudentStudyCourse objects.
368    """
369    grok.implements(IBatchProcessor)
370    grok.provides(IBatchProcessor)
371    grok.context(Interface)
372    util_name = 'studycourseupdater'
373    grok.name(util_name)
374
375    name = u'StudentStudyCourse Processor (update only)'
376    iface = IStudentStudyCourse
377    factory_name = 'waeup.StudentStudyCourse'
378
379    location_fields = []
380    additional_fields = []
381
382    def getParent(self, row, site):
383        return self._getStudent(row, site)
384
385    def getEntry(self, row, site):
386        student = self.getParent(row, site)
387        if student is None:
388            return None
389        return student.get('studycourse')
390
391    def updateEntry(self, obj, row, site):
392        """Update obj to the values given in row.
393        """
394        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
395            obj, row, site)
396        parent = self.getParent(row, site)
397        parent.__parent__.logger.info(
398            '%s - Study course updated: %s'
399            % (parent.student_id, items_changed))
400        # Update the students_catalog
401        notify(grok.ObjectModifiedEvent(obj.__parent__))
402        return
403
404    def checkConversion(self, row, mode='ignore'):
405        """Validates all values in row.
406        """
407        errs, inv_errs, conv_dict = super(
408            StudentStudyCourseProcessor, self).checkConversion(row, mode=mode)
409        # We have to check if current_level is in range of certificate.
410        if conv_dict.has_key('certificate'):
411          cert = conv_dict['certificate']
412          if conv_dict['current_level'] < cert.start_level or \
413              conv_dict['current_level'] > cert.end_level+120:
414              errs.append(('current_level','not in range'))
415        return errs, inv_errs, conv_dict
416
417class StudentStudyLevelProcessor(StudentProcessorBase):
418    """A batch processor for IStudentStudyLevel objects.
419    """
420    grok.implements(IBatchProcessor)
421    grok.provides(IBatchProcessor)
422    grok.context(Interface)
423    util_name = 'studylevelprocessor'
424    grok.name(util_name)
425
426    name = u'StudentStudyLevel Processor'
427    iface = IStudentStudyLevel
428    factory_name = 'waeup.StudentStudyLevel'
429
430    location_fields = []
431    additional_fields = ['level']
432    additional_headers = ['level']
433
434    def getParent(self, row, site):
435        student = self._getStudent(row, site)
436        if student is None:
437            return None
438        return student['studycourse']
439
440    def getEntry(self, row, site):
441        studycourse = self.getParent(row, site)
442        if studycourse is None:
443            return None
444        return studycourse.get(row['level'])
445
446    def updateEntry(self, obj, row, site):
447        """Update obj to the values given in row.
448        """
449        items_changed = super(StudentStudyLevelProcessor, self).updateEntry(
450            obj, row, site)
451        student = self.getParent(row, site).__parent__
452        student.__parent__.logger.info(
453            '%s - Study level updated: %s'
454            % (student.student_id, items_changed))
455        return
456
457    def addEntry(self, obj, row, site):
458        parent = self.getParent(row, site)
459        obj.level = int(row['level'])
460        parent[row['level']] = obj
461        return
462
463    def checkConversion(self, row, mode='ignore'):
464        """Validates all values in row.
465        """
466        errs, inv_errs, conv_dict = super(
467            StudentStudyLevelProcessor, self).checkConversion(row, mode=mode)
468
469        # We have to check if level is a valid integer.
470        # This is not done by the converter.
471        try:
472            level = int(row['level'])
473            if level not in range(0,700,10):
474                errs.append(('level','no valid integer'))
475        except ValueError:
476            errs.append(('level','no integer'))
477        return errs, inv_errs, conv_dict
478
479class CourseTicketProcessor(StudentProcessorBase):
480    """A batch processor for ICourseTicket objects.
481    """
482    grok.implements(IBatchProcessor)
483    grok.provides(IBatchProcessor)
484    grok.context(Interface)
485    util_name = 'courseticketprocessor'
486    grok.name(util_name)
487
488    name = u'CourseTicket Processor'
489    iface = ICourseTicket
490    factory_name = 'waeup.CourseTicket'
491
492    location_fields = []
493    additional_fields = ['level', 'code']
494    additional_headers = ['level', 'code']
495
496    def getParent(self, row, site):
497        student = self._getStudent(row, site)
498        if student is None:
499            return None
500        return student['studycourse'].get(row['level'])
501
502    def getEntry(self, row, site):
503        level = self.getParent(row, site)
504        if level is None:
505            return None
506        return level.get(row['code'])
507
508    def updateEntry(self, obj, row, site):
509        """Update obj to the values given in row.
510        """
511        items_changed = super(CourseTicketProcessor, self).updateEntry(
512            obj, row, site)
513        student = self.getParent(row, site).__parent__.__parent__
514        student.__parent__.logger.info(
515            '%s - Course ticket updated: %s'
516            % (student.student_id, items_changed))
517        return
518
519    def addEntry(self, obj, row, site):
520        parent = self.getParent(row, site)
521        catalog = getUtility(ICatalog, name='courses_catalog')
522        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
523        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
524        obj.dcode = entries[0].__parent__.__parent__.code
525        obj.title = entries[0].title
526        obj.credits = entries[0].credits
527        obj.passmark = entries[0].passmark
528        obj.semester = entries[0].semester
529        parent[row['code']] = obj
530        return
531
532    def checkConversion(self, row, mode='ignore'):
533        """Validates all values in row.
534        """
535        errs, inv_errs, conv_dict = super(
536            CourseTicketProcessor, self).checkConversion(row, mode=mode)
537
538        # We have to check if course really exists.
539        # This is not done by the converter.
540        catalog = getUtility(ICatalog, name='courses_catalog')
541        entries = catalog.searchResults(code=(row['code'],row['code']))
542        if len(entries) == 0:
543            errs.append(('code','non-existent'))
544            return errs, inv_errs, conv_dict
545        return errs, inv_errs, conv_dict
546
547class StudentOnlinePaymentProcessor(StudentProcessorBase):
548    """A batch processor for IStudentOnlinePayment objects.
549    """
550    grok.implements(IBatchProcessor)
551    grok.provides(IBatchProcessor)
552    grok.context(Interface)
553    util_name = 'paymentprocessor'
554    grok.name(util_name)
555
556    name = u'Payment Processor'
557    iface = IStudentOnlinePayment
558    factory_name = 'waeup.StudentOnlinePayment'
559
560    location_fields = []
561    additional_fields = ['p_id']
562    additional_headers = []
563
564    def checkHeaders(self, headerfields, mode='ignore'):
565        super(StudentOnlinePaymentProcessor, self).checkHeaders(headerfields)
566        if mode == 'update' and not 'p_id' in headerfields:
567            raise FatalCSVError(
568                "Need p_id for import in update mode!")
569        return True
570
571    def parentsExist(self, row, site):
572        return self.getParent(row, site) is not None
573
574    def getParent(self, row, site):
575        student = self._getStudent(row, site)
576        if student is None:
577            return None
578        return student['payments']
579
580    def getEntry(self, row, site):
581        payments = self.getParent(row, site)
582        if payments is None:
583            return None
584        p_id = row.get('p_id', None)
585        if p_id is None:
586            return None
587        # We can use the hash symbol at the end of p_id in import files
588        # to avoid annoying automatic number transformation
589        # by Excel or Calc
590        p_id = p_id.strip('#')
591        if not p_id.startswith('p'):
592            # For data migration from old SRP only
593            p_id = 'p' + p_id[7:] + '0'
594        entry = payments.get(p_id)
595        return entry
596
597    def updateEntry(self, obj, row, site):
598        """Update obj to the values given in row.
599        """
600        items_changed = super(StudentOnlinePaymentProcessor, self).updateEntry(
601            obj, row, site)
602        student = self.getParent(row, site).__parent__
603        student.__parent__.logger.info(
604            '%s - Payment ticket updated: %s'
605            % (student.student_id, items_changed))
606        return
607
608    def addEntry(self, obj, row, site):
609        parent = self.getParent(row, site)
610        p_id = row['p_id'].strip('#')
611        if not p_id.startswith('p'):
612            # For data migration from old SRP
613            obj.p_id = 'p' + p_id[7:] + '0'
614            parent[obj.p_id] = obj
615        else:
616            parent[p_id] = obj
617        return
618
619    def checkConversion(self, row, mode='ignore'):
620        """Validates all values in row.
621        """
622        errs, inv_errs, conv_dict = super(
623            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
624
625        # We have to check p_id.
626        p_id = row.get('p_id', None)
627        if p_id is None:
628            timestamp = ("%d" % int(time()*10000))[1:]
629            p_id = "p%s" % timestamp
630            conv_dict['p_id'] = p_id
631            return errs, inv_errs, conv_dict
632        else:
633            p_id = p_id.strip('#')
634        if p_id.startswith('p'):
635            if not len(p_id) == 14:
636                errs.append(('p_id','invalid length'))
637                return errs, inv_errs, conv_dict
638        else:
639            if not len(p_id) == 19:
640                errs.append(('p_id','invalid length'))
641                return errs, inv_errs, conv_dict
642        return errs, inv_errs, conv_dict
643
644class StudentVerdictProcessor(StudentStudyCourseProcessor):
645    """A batch processor for verdicts.
646
647    Import verdicts and perform workflow transitions.
648    """
649
650    util_name = 'verdictupdater'
651    grok.name(util_name)
652
653    name = u'Verdict Processor (update only)'
654    iface = IStudentVerdictUpdate
655    factory_name = 'waeup.StudentStudyCourse'
656
657    additional_fields = ['current_session', 'current_level']
658
659    def checkUpdateRequirements(self, obj, row, site):
660        """Checks requirements the studycourse and the student must fulfill
661        before being updated.
662        """
663        # Check if current_levels correspond
664        if obj.current_level != row['current_level']:
665            return 'Current level does not correspond.'
666        # Check if current_sessions correspond
667        if obj.current_session != row['current_session']:
668            return 'Current session does not correspond.'
669        # Check if student is in state REGISTERED
670        if obj.student.state != VALIDATED:
671            return 'Student in wrong state.'
672        return None
673
674    def updateEntry(self, obj, row, site):
675        """Update obj to the values given in row.
676        """
677        # Don't set current_session, current_level
678        vals_to_set = dict((key, val) for key, val in row.items()
679                           if key not in ('current_session','current_level'))
680        items_changed = super(StudentVerdictProcessor, self).updateEntry(
681            obj, vals_to_set, site)
682        parent = self.getParent(row, site)
683        parent.__parent__.logger.info(
684            '%s - Verdict updated: %s'
685            % (parent.student_id, items_changed))
686        # Fire transition
687        IWorkflowInfo(obj.__parent__).fireTransition('return')
688        # Update the students_catalog
689        notify(grok.ObjectModifiedEvent(obj.__parent__))
690        return
Note: See TracBrowser for help on using the repository browser.