source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 15505

Last change on this file since 15505 was 15446, checked in by Henrik Bettermann, 5 years ago

Add level 0 (None) to course levels.

  • Property svn:keywords set to Id
File size: 42.6 KB
Line 
1## $Id: batching.py 15446 2019-06-03 10:32:23Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import unicodecsv as csv # XXX: csv ops should move to dedicated module.
28from time import time
29from datetime import datetime
30from zope.i18n import translate
31from zope.interface import Interface
32from zope.schema import getFields
33from zope.component import queryUtility, getUtility, createObject
34from zope.event import notify
35from zope.catalog.interfaces import ICatalog
36from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
37from waeup.kofa.interfaces import (
38    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
39    IObjectHistory, VALIDATED, REGISTERED, IGNORE_MARKER)
40from waeup.kofa.interfaces import IKofaUtils, DuplicationError
41from waeup.kofa.interfaces import MessageFactory as _
42from waeup.kofa.students.interfaces import (
43    IStudent, IStudentStudyCourse, IStudentStudyCourseTransfer,
44    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
45    IStudentStudyLevel, ICourseTicketImport,
46    IStudentOnlinePayment, IStudentVerdictUpdate)
47from waeup.kofa.payments.interfaces import IPayer
48from waeup.kofa.students.workflow import  (
49    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS,
50    FORBIDDEN_POSTGRAD_TRANS, FORBIDDEN_POSTGRAD_STATES)
51from waeup.kofa.utils.batching import BatchProcessor
52
53class StudentProcessor(BatchProcessor):
54    """The Student Processor imports student base data.
55
56    In create mode no locator is required. If no `student_id` is given,
57    the portal automatically assigns a new student id.
58
59    In update or remove mode the processor uses
60    either the `student_id`, `reg_number` or `matric_number` to localize the
61    student object, exactly in this order. If `student_id` is given and an
62    object can be found, `reg_number` and `matric_number` will be overwritten
63    by the values provided in the import file. If `student_id` is missing,
64    `reg_number` is used to localize the object and only `matric_number`
65    will be overwritten. `matric_number` is used as locator only if both
66    `student_id` and `reg_number` are missing. `student_id` can't be changed
67    by the batch processor.
68
69    There are two ways to change the workflow state of the student,
70    an unsafe and a safe way. The safe way makes use of workflow transitions.
71    Transitions are only possible between allowed workflow states. Only
72    transitions ensure that the registration workflow is maintained.
73
74    **Always prefer the safe way!**
75    """
76    grok.implements(IBatchProcessor)
77    grok.provides(IBatchProcessor)
78    grok.context(Interface)
79    util_name = 'studentprocessor'
80    grok.name(util_name)
81
82    name = _('Student Processor')
83    iface = IStudent
84    iface_byregnumber = IStudentUpdateByRegNo
85    iface_bymatricnumber = IStudentUpdateByMatricNo
86
87    factory_name = 'waeup.Student'
88
89    @property
90    def available_fields(self):
91        fields = getFields(self.iface)
92        return sorted(list(set(
93            ['student_id','reg_number','matric_number',
94            'password', 'state', 'transition'] + fields.keys())))
95
96    def checkHeaders(self, headerfields, mode='create'):
97        if 'state' in headerfields and 'transition' in headerfields:
98            raise FatalCSVError(
99                "State and transition can't be imported at the same time!")
100        if not 'reg_number' in headerfields and not 'student_id' \
101            in headerfields and not 'matric_number' in headerfields:
102            raise FatalCSVError(
103                "Need at least columns student_id or reg_number " +
104                "or matric_number for import!")
105        if mode == 'create':
106            for field in self.required_fields:
107                if not field in headerfields:
108                    raise FatalCSVError(
109                        "Need at least columns %s for import!" %
110                        ', '.join(["'%s'" % x for x in self.required_fields]))
111        # Check for fields to be ignored...
112        not_ignored_fields = [x for x in headerfields
113                              if not x.startswith('--')]
114        if len(set(not_ignored_fields)) < len(not_ignored_fields):
115            raise FatalCSVError(
116                "Double headers: each column name may only appear once.")
117        return True
118
119    def parentsExist(self, row, site):
120        return 'students' in site.keys()
121
122    def getLocator(self, row):
123        if row.get('student_id',None) not in (None, IGNORE_MARKER):
124            return 'student_id'
125        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
126            return 'reg_number'
127        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
128            return 'matric_number'
129        else:
130            return None
131
132    # The entry never exists in create mode.
133    def entryExists(self, row, site):
134        return self.getEntry(row, site) is not None
135
136    def getParent(self, row, site):
137        return site['students']
138
139    def getEntry(self, row, site):
140        if not 'students' in site.keys():
141            return None
142        if self.getLocator(row) == 'student_id':
143            if row['student_id'] in site['students']:
144                student = site['students'][row['student_id']]
145                return student
146        elif self.getLocator(row) == 'reg_number':
147            reg_number = row['reg_number']
148            cat = queryUtility(ICatalog, name='students_catalog')
149            results = list(
150                cat.searchResults(reg_number=(reg_number, reg_number)))
151            if results:
152                return results[0]
153        elif self.getLocator(row) == 'matric_number':
154            matric_number = row['matric_number']
155            cat = queryUtility(ICatalog, name='students_catalog')
156            results = list(
157                cat.searchResults(matric_number=(matric_number, matric_number)))
158            if results:
159                return results[0]
160        return None
161
162    def addEntry(self, obj, row, site):
163        parent = self.getParent(row, site)
164        parent.addStudent(obj)
165        # Reset _curr_stud_id if student_id has been imported
166        if self.getLocator(row) == 'student_id':
167            parent._curr_stud_id -= 1
168        # We have to log this if state is provided. If not,
169        # logging is done by the event handler handle_student_added
170        if 'state' in row:
171            parent.logger.info('%s - Student record created' % obj.student_id)
172        return
173
174    def delEntry(self, row, site):
175        student = self.getEntry(row, site)
176        if student is not None:
177            parent = self.getParent(row, site)
178            parent.logger.info('%s - Student removed' % student.student_id)
179            del parent[student.student_id]
180        pass
181
182    def checkUpdateRequirements(self, obj, row, site):
183        """Checks requirements the object must fulfill when being updated.
184
185        This method is not used in case of deleting or adding objects.
186
187        Returns error messages as strings in case of requirement
188        problems.
189        """
190        transition = row.get('transition', IGNORE_MARKER)
191        if transition not in (IGNORE_MARKER, ''):
192            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
193            if transition not in allowed_transitions:
194                return 'Transition not allowed.'
195            if transition in FORBIDDEN_POSTGRAD_TRANS and \
196                obj.is_postgrad:
197                return 'Transition not allowed (pg student).'
198        state = row.get('state', IGNORE_MARKER)
199        if state not in (IGNORE_MARKER, ''):
200            if state in FORBIDDEN_POSTGRAD_STATES and \
201                obj.is_postgrad:
202                return 'State not allowed (pg student).'
203        return None
204
205    def updateEntry(self, obj, row, site, filename):
206        """Update obj to the values given in row.
207        """
208        items_changed = ''
209
210        # Remove student_id from row if empty
211        if 'student_id' in row and row['student_id'] in (None, IGNORE_MARKER):
212            row.pop('student_id')
213
214        # Update password
215        # XXX: Take DELETION_MARKER into consideration
216        if 'password' in row:
217            passwd = row.get('password', IGNORE_MARKER)
218            if passwd not in ('', IGNORE_MARKER):
219                if passwd.startswith('{SSHA}'):
220                    # already encrypted password
221                    obj.password = passwd
222                else:
223                    # not yet encrypted password
224                    IUserAccount(obj).setPassword(passwd)
225                items_changed += ('%s=%s, ' % ('password', passwd))
226            row.pop('password')
227
228        # Update registration state
229        if 'state' in row:
230            state = row.get('state', IGNORE_MARKER)
231            if state not in (IGNORE_MARKER, ''):
232                value = row['state']
233                IWorkflowState(obj).setState(value)
234                msg = _("State '${a}' set", mapping = {'a':value})
235                history = IObjectHistory(obj)
236                history.addMessage(msg)
237                items_changed += ('%s=%s, ' % ('state', state))
238            row.pop('state')
239
240        if 'transition' in row:
241            transition = row.get('transition', IGNORE_MARKER)
242            if transition not in (IGNORE_MARKER, ''):
243                value = row['transition']
244                IWorkflowInfo(obj).fireTransition(value)
245                items_changed += ('%s=%s, ' % ('transition', transition))
246            row.pop('transition')
247
248        # apply other values...
249        items_changed += super(StudentProcessor, self).updateEntry(
250            obj, row, site, filename)
251
252        # Log actions...
253        parent = self.getParent(row, site)
254        if hasattr(obj,'student_id'):
255            # Update mode: the student exists and we can get the student_id.
256            # Create mode: the record contains the student_id
257            parent.logger.info(
258                '%s - %s - %s - updated: %s'
259                % (self.name, filename, obj.student_id, items_changed))
260        else:
261            # Create mode: the student does not yet exist
262            # XXX: It seems that this never happens because student_id
263            # is always set.
264            parent.logger.info(
265                '%s - %s - %s - imported: %s'
266                % (self.name, filename, obj.student_id, items_changed))
267        notify(grok.ObjectModifiedEvent(obj))
268        return items_changed
269
270    def getMapping(self, path, headerfields, mode):
271        """Get a mapping from CSV file headerfields to actually used fieldnames.
272        """
273        result = dict()
274        reader = csv.reader(open(path, 'rb'))
275        raw_header = reader.next()
276        for num, field in enumerate(headerfields):
277            if field not in ['student_id', 'reg_number', 'matric_number'
278                             ] and mode == 'remove':
279                continue
280            if field == u'--IGNORE--':
281                # Skip ignored columns in failed and finished data files.
282                continue
283            result[raw_header[num]] = field
284        return result
285
286    def checkConversion(self, row, mode='create'):
287        """Validates all values in row.
288        """
289        iface = self.iface
290        if mode in ['update', 'remove']:
291            if self.getLocator(row) == 'reg_number':
292                iface = self.iface_byregnumber
293            elif self.getLocator(row) == 'matric_number':
294                iface = self.iface_bymatricnumber
295        converter = IObjectConverter(iface)
296        errs, inv_errs, conv_dict =  converter.fromStringDict(
297            row, self.factory_name, mode=mode)
298        # We cannot import both state and transition.
299        if 'transition' in row and 'state' in row:
300            if row['transition'] not in (IGNORE_MARKER, '') and \
301                row['state'] not in (IGNORE_MARKER, ''):
302                errs.append(('workflow','not allowed'))
303                return errs, inv_errs, conv_dict
304        if 'transition' in row:
305            if row['transition'] not in IMPORTABLE_TRANSITIONS:
306                if row['transition'] not in (IGNORE_MARKER, ''):
307                    errs.append(('transition','not allowed'))
308        if 'state' in row:
309            if row['state'] not in IMPORTABLE_STATES:
310                if row['state'] not in (IGNORE_MARKER, ''):
311                    errs.append(('state','not allowed'))
312                else:
313                    # State is an attribute of Student and must not
314                    # be changed if empty.
315                    conv_dict['state'] = IGNORE_MARKER
316        try:
317            # Correct stud_id counter. As the IConverter for students
318            # creates student objects that are not used afterwards, we
319            # have to fix the site-wide student_id counter.
320            site = grok.getSite()
321            students = site['students']
322            students._curr_stud_id -= 1
323        except (KeyError, TypeError, AttributeError):
324                pass
325        return errs, inv_errs, conv_dict
326
327
328class StudentProcessorBase(BatchProcessor):
329    """A base for student subitem processor.
330
331    Helps reducing redundancy.
332    """
333    grok.baseclass()
334
335    # additional available fields
336    # beside 'student_id', 'reg_number' and 'matric_number'
337    additional_fields = []
338
339    # additional required fields (subset of additional_fields)
340    additional_fields_required = []
341
342    @property
343    def available_fields(self):
344        fields = ['student_id','reg_number','matric_number'
345                  ] + self.additional_fields
346        return sorted(list(set(fields + getFields(
347                self.iface).keys())))
348
349    def checkHeaders(self, headerfields, mode='ignore'):
350        if not 'reg_number' in headerfields and not 'student_id' \
351            in headerfields and not 'matric_number' in headerfields:
352            raise FatalCSVError(
353                "Need at least columns student_id " +
354                "or reg_number or matric_number for import!")
355        for name in self.additional_fields_required:
356            if not name in headerfields:
357                raise FatalCSVError(
358                    "Need %s for import!" % name)
359
360        # Check for fields to be ignored...
361        not_ignored_fields = [x for x in headerfields
362                              if not x.startswith('--')]
363        if len(set(not_ignored_fields)) < len(not_ignored_fields):
364            raise FatalCSVError(
365                "Double headers: each column name may only appear once.")
366        return True
367
368    def _getStudent(self, row, site):
369        NON_VALUES = ['', IGNORE_MARKER]
370        if not 'students' in site.keys():
371            return None
372        if row.get('student_id', '') not in NON_VALUES:
373            if row['student_id'] in site['students']:
374                student = site['students'][row['student_id']]
375                return student
376        elif row.get('reg_number', '') not in NON_VALUES:
377            reg_number = row['reg_number']
378            cat = queryUtility(ICatalog, name='students_catalog')
379            results = list(
380                cat.searchResults(reg_number=(reg_number, reg_number)))
381            if results:
382                return results[0]
383        elif row.get('matric_number', '') not in NON_VALUES:
384            matric_number = row['matric_number']
385            cat = queryUtility(ICatalog, name='students_catalog')
386            results = list(
387                cat.searchResults(matric_number=(matric_number, matric_number)))
388            if results:
389                return results[0]
390        return None
391
392    def parentsExist(self, row, site):
393        return self.getParent(row, site) is not None
394
395    def entryExists(self, row, site):
396        return self.getEntry(row, site) is not None
397
398    def checkConversion(self, row, mode='ignore'):
399        """Validates all values in row.
400        """
401        converter = IObjectConverter(self.iface)
402        errs, inv_errs, conv_dict =  converter.fromStringDict(
403            row, self.factory_name, mode=mode)
404        return errs, inv_errs, conv_dict
405
406    def getMapping(self, path, headerfields, mode):
407        """Get a mapping from CSV file headerfields to actually used fieldnames.
408        """
409        result = dict()
410        reader = csv.reader(open(path, 'rb'))
411        raw_header = reader.next()
412        for num, field in enumerate(headerfields):
413            if field not in ['student_id', 'reg_number', 'matric_number',
414                             'p_id', 'code', 'level'
415                             ] and mode == 'remove':
416                continue
417            if field == u'--IGNORE--':
418                # Skip ignored columns in failed and finished data files.
419                continue
420            result[raw_header[num]] = field
421        return result
422
423
424class StudentStudyCourseProcessor(StudentProcessorBase):
425    """The Student Study Course Processor imports data which refer
426    to the student's course of study. The study course container data
427    describe the current state of the course of study and it stores the
428    entry conditions, i.e. when the student started the course.
429
430    Most important is the `certificate` attribute which tells us which course
431    the student is studying. The terms 'study course' and 'course of study'
432    are used synonymously. The 'certificate' is the study programme described
433    in the acadmic section. The study course object stores a referrer to a
434    certificate in the acadmic section.
435
436    When importing a new certificate code, `checkConversion` does not only
437    check whether a certificate with the same code exists, it also
438    proves if `current_level` is inside the level range of the certificate.
439    For example, some study programmes start at level 200. The imported
440    current level must thus be 200 or higher.
441
442    `checkUpdateRequirements` looks up if the imported values match the
443    certificate already stored with the study course object. The imported
444    `current_level` must be in the range of the certificate already
445    stored.
446
447    .. note::
448
449      The processor does only offer an update mode. An 'empty' study course
450      object is automatically created when the student object is added. So this
451      object always exists. It can neither be added a second time nor
452      be removed.
453
454    Students can be transferred by import. A transfer is initialized if the
455    `entry_mode` value is ``transfer``. In this case `checkConversion` uses a
456    different interface for data validation and `checkUpdateRequirements`
457    ensures that a student can only be transferred twice. The student transfer
458    process is described elsewhere.
459    """
460    grok.implements(IBatchProcessor)
461    grok.provides(IBatchProcessor)
462    grok.context(Interface)
463    util_name = 'studycourseupdater'
464    grok.name(util_name)
465
466    name = _('StudentStudyCourse Processor (update only)')
467    iface = IStudentStudyCourse
468    iface_transfer = IStudentStudyCourseTransfer
469    factory_name = 'waeup.StudentStudyCourse'
470
471    def getParent(self, row, site):
472        return self._getStudent(row, site)
473
474    def getEntry(self, row, site):
475        student = self.getParent(row, site)
476        if student is None:
477            return None
478        return student.get('studycourse')
479
480    def updateEntry(self, obj, row, site, filename):
481        """Update obj to the values given in row.
482        """
483        entry_mode = row.get('entry_mode', None)
484        certificate = row.get('certificate', None)
485        current_session = row.get('current_session', None)
486        student = self.getParent(row, site)
487        if entry_mode == 'transfer':
488            # We do not expect any error here since we
489            # checked all constraints in checkConversion and
490            # in checkUpdateRequirements
491            student.transfer(
492                certificate=certificate, current_session=current_session)
493            obj = student['studycourse']
494        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
495            obj, row, site, filename)
496        student.__parent__.logger.info(
497            '%s - %s - %s - updated: %s'
498            % (self.name, filename, student.student_id, items_changed))
499        # Update the students_catalog
500        notify(grok.ObjectModifiedEvent(student))
501        return
502
503    def checkConversion(self, row, mode='ignore'):
504        """Validates all values in row.
505        """
506        # We have to use the correct interface. Transfer
507        # updates have different constraints.
508        entry_mode = row.get('entry_mode', None)
509        if entry_mode == 'transfer':
510            converter = IObjectConverter(self.iface_transfer)
511        else:
512            converter = IObjectConverter(self.iface)
513        errs, inv_errs, conv_dict =  converter.fromStringDict(
514            row, self.factory_name, mode=mode)
515
516        # We have to check if current_level is in range of certificate.
517        if 'certificate' in conv_dict and 'current_level' in conv_dict:
518            cert = conv_dict['certificate']
519            level = conv_dict['current_level']
520            if level < cert.start_level or level > cert.end_level+120:
521                errs.append(('current_level','not in range'))
522        return errs, inv_errs, conv_dict
523
524    def checkUpdateRequirements(self, obj, row, site):
525        """Checks requirements the object must fulfill when being updated.
526        Returns error messages as strings in case of requirement
527        problems.
528        """
529        if obj.student.studycourse_locked:
530            return 'Studycourse is locked.'
531        certificate = getattr(obj, 'certificate', None)
532        entry_session = getattr(obj, 'entry_session', None)
533        current_level = row.get('current_level', None)
534        entry_mode = row.get('entry_mode', None)
535        # We have to ensure that the student can be transferred.
536        if entry_mode == 'transfer':
537            if certificate is None or entry_session is None:
538                return 'Former study course record incomplete.'
539            if 'studycourse_1' in obj.__parent__.keys() and \
540                'studycourse_2' in obj.__parent__.keys():
541                return 'Maximum number of transfers exceeded.'
542        if current_level:
543            if current_level == 999 and \
544                obj.__parent__.state in FORBIDDEN_POSTGRAD_STATES:
545                return 'Not a pg student.'
546            cert = row.get('certificate', None)
547            if certificate is None and cert is None:
548                return 'No certificate to check level.'
549            if certificate is not None and cert is None and (
550                current_level < certificate.start_level or \
551                current_level > certificate.end_level+120):
552                return 'current_level not in range.'
553        return None
554
555class StudentStudyLevelProcessor(StudentProcessorBase):
556    """The Student Study Level Processor imports study level data.
557    It overwrites the container attributes but not the content of the container,
558    i.e. the course tickets stored inside the container. There is nothing
559    special about this processor.
560    """
561    grok.implements(IBatchProcessor)
562    grok.provides(IBatchProcessor)
563    grok.context(Interface)
564    util_name = 'studylevelprocessor'
565    grok.name(util_name)
566
567    name = _('StudentStudyLevel Processor')
568    iface = IStudentStudyLevel
569    factory_name = 'waeup.StudentStudyLevel'
570
571    @property
572    def available_fields(self):
573        fields = super(StudentStudyLevelProcessor, self).available_fields
574        fields.remove('total_credits')
575        fields.remove('gpa')
576        return  fields
577
578    def getParent(self, row, site):
579        student = self._getStudent(row, site)
580        if student is None:
581            return None
582        return student['studycourse']
583
584    def getEntry(self, row, site):
585        studycourse = self.getParent(row, site)
586        if studycourse is None:
587            return None
588        try:
589            entry = studycourse.get(str(row['level']))
590        except KeyError:
591            return None
592        return entry
593
594    def delEntry(self, row, site):
595        studylevel = self.getEntry(row, site)
596        parent = self.getParent(row, site)
597        if studylevel is not None:
598            student = self._getStudent(row, site)
599            student.__parent__.logger.info('%s - Level removed: %s'
600                % (student.student_id, studylevel.__name__))
601            del parent[studylevel.__name__]
602        return
603
604    def updateEntry(self, obj, row, site, filename):
605        """Update obj to the values given in row.
606        """
607        items_changed = super(StudentStudyLevelProcessor, self).updateEntry(
608            obj, row, site, filename)
609        student = self.getParent(row, site).__parent__
610        student.__parent__.logger.info(
611            '%s - %s - %s - updated: %s'
612            % (self.name, filename, student.student_id, items_changed))
613        return
614
615    def addEntry(self, obj, row, site):
616        if IGNORE_MARKER == str(row['level']):
617            raise FatalCSVError("level: Invalid value")
618        parent = self.getParent(row, site)
619        parent[str(row['level'])] = obj
620        return
621
622    def checkUpdateRequirements(self, obj, row, site):
623        """
624        """
625        if obj.student.studycourse_locked:
626            return 'Studylevel is locked.'
627        return None
628
629class CourseTicketProcessor(StudentProcessorBase):
630    """The Course Ticket Processor imports course tickets, the subobjects
631    of student study levels (= course lists).
632
633    An imported course ticket contains a copy of the original course data.
634    During import only a few attributes can be set/overwritten.
635
636    Like all other student data importers, this processor also requires
637    either `student_id`, `reg_number` or `matric_number` to find the student.
638    Then  it needs `level` and `code` to localize the course ticket.
639
640    `checkConversion` first searches the courses catalog for the imported
641    `code` and ensures that a course with such a code really exists
642    in the academic section. It furthermore checks if `level_session` in
643    the row corresponds with the session of the parent student
644    study level object. It fails if one of the conditions is not met.
645
646    In create mode `fcode`, `dcode`, `title`, `credits`, `passmark` and
647    `semester` are taken from the course found in the academic section.
648    `fcode` and `dcode` can nevermore be changed, neither via the user interface
649    nor by import. Other values can be overwritten by import.
650
651    `ticket_session` is an additional attribute which can be used to store the
652    session of the course when it was taken. Usually this information is
653    redundant because the parent study level object already contains this
654    information, except for the study level zero container which can be used to
655    store 'orphaned' course tickets.
656    """
657    grok.implements(IBatchProcessor)
658    grok.provides(IBatchProcessor)
659    grok.context(Interface)
660    util_name = 'courseticketprocessor'
661    grok.name(util_name)
662
663    name = _('CourseTicket Processor')
664    iface = ICourseTicketImport
665    factory_name = 'waeup.CourseTicket'
666
667    additional_fields = ['level', 'code']
668    additional_fields_required = additional_fields
669
670    @property
671    def available_fields(self):
672        fields = [
673            'student_id','reg_number','matric_number',
674            'mandatory', 'score', 'carry_over', 'automatic',
675            'outstanding', 'course_category', 'level_session',
676            'title', 'credits', 'passmark', 'semester', 'ticket_session'
677            ] + self.additional_fields
678        return sorted(fields)
679
680    def getParent(self, row, site):
681        student = self._getStudent(row, site)
682        if student is None:
683            return None
684        return student['studycourse'].get(str(row['level']))
685
686    def getEntry(self, row, site):
687        level = self.getParent(row, site)
688        if level is None:
689            return None
690        return level.get(row['code'])
691
692    def updateEntry(self, obj, row, site, filename):
693        """Update obj to the values given in row.
694        """
695        items_changed = super(CourseTicketProcessor, self).updateEntry(
696            obj, row, site, filename)
697        parent = self.getParent(row, site)
698        student = self.getParent(row, site).__parent__.__parent__
699        student.__parent__.logger.info(
700            '%s - %s - %s - %s - updated: %s'
701            % (self.name, filename, student.student_id, parent.level, items_changed))
702        return
703
704    def addEntry(self, obj, row, site):
705        parent = self.getParent(row, site)
706        catalog = getUtility(ICatalog, name='courses_catalog')
707        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
708        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
709        obj.dcode = entries[0].__parent__.__parent__.code
710        if getattr(obj, 'title', None) is None:
711            obj.title = entries[0].title
712        if getattr(obj, 'credits', None) is None:
713            obj.credits = entries[0].credits
714        if getattr(obj, 'passmark', None) is None:
715            obj.passmark = entries[0].passmark
716        if getattr(obj, 'semester', None) is None:
717            obj.semester = entries[0].semester
718        parent[row['code']] = obj
719        return
720
721    def delEntry(self, row, site):
722        ticket = self.getEntry(row, site)
723        parent = self.getParent(row, site)
724        if ticket is not None:
725            student = self._getStudent(row, site)
726            student.__parent__.logger.info('%s - Course ticket in %s removed: %s'
727                % (student.student_id, parent.level, ticket.code))
728            del parent[ticket.code]
729        return
730
731    def checkUpdateRequirements(self, obj, row, site):
732        """
733        """
734        if obj.student.studycourse_locked:
735            return 'Studycourse is locked.'
736        return None
737
738    def checkConversion(self, row, mode='ignore'):
739        """Validates all values in row.
740        """
741        errs, inv_errs, conv_dict = super(
742            CourseTicketProcessor, self).checkConversion(row, mode=mode)
743        if mode == 'remove':
744            return errs, inv_errs, conv_dict
745        # In update and create mode we have to check if course really exists.
746        # This is not done by the converter.
747        catalog = getUtility(ICatalog, name='courses_catalog')
748        entries = catalog.searchResults(code=(row['code'],row['code']))
749        if len(entries) == 0:
750            errs.append(('code','non-existent'))
751            return errs, inv_errs, conv_dict
752        # If level_session is provided in row we have to check if
753        # the parent studylevel exists and if its level_session
754        # attribute corresponds with the expected value in row.
755        level_session = conv_dict.get('level_session', IGNORE_MARKER)
756        if level_session not in (IGNORE_MARKER, None):
757            site = grok.getSite()
758            studylevel = self.getParent(row, site)
759            if studylevel is not None:
760                if studylevel.level_session != level_session:
761                    errs.append(('level_session','does not match %s'
762                        % studylevel.level_session))
763            else:
764                errs.append(('level object','does not exist'))
765        return errs, inv_errs, conv_dict
766
767class StudentOnlinePaymentProcessor(StudentProcessorBase):
768    """The Student Online Payment Processor imports student payment tickets.
769    The tickets are located in the ``payments`` subfolder of the student
770    container. The only additional locator is `p_id`, the object id.
771
772    The `checkConversion` method checks the format of the payment identifier.
773    In create mode it does also ensures that same `p_id` does not exist
774    elsewhere. It must be portal-wide unique.
775
776    When adding a payment ticket, the `addEntry` method checks if the same
777    payment has already been made. It compares `p_category` and `p_session`
778    in the row with the corresponding attributes  of existing payment
779    tickets in state ``paid``. If they match, a `DuplicationError` is raised.
780    """
781    grok.implements(IBatchProcessor)
782    grok.provides(IBatchProcessor)
783    grok.context(Interface)
784    util_name = 'paymentprocessor'
785    grok.name(util_name)
786
787    name = _('StudentOnlinePayment Processor')
788    iface = IStudentOnlinePayment
789    factory_name = 'waeup.StudentOnlinePayment'
790
791    additional_fields = ['p_id']
792
793    @property
794    def available_fields(self):
795        af = super(
796            StudentOnlinePaymentProcessor, self).available_fields
797        af.remove('display_item')
798        return af
799
800    def checkHeaders(self, headerfields, mode='ignore'):
801        super(StudentOnlinePaymentProcessor, self).checkHeaders(headerfields)
802        if mode in ('update', 'remove') and not 'p_id' in headerfields:
803            raise FatalCSVError(
804                "Need p_id for import in update and remove modes!")
805        return True
806
807    def parentsExist(self, row, site):
808        return self.getParent(row, site) is not None
809
810    def getParent(self, row, site):
811        student = self._getStudent(row, site)
812        if student is None:
813            return None
814        return student['payments']
815
816    def getEntry(self, row, site):
817        payments = self.getParent(row, site)
818        if payments is None:
819            return None
820        p_id = row.get('p_id', None)
821        if p_id in (None, IGNORE_MARKER):
822            return None
823        # We can use the hash symbol at the end of p_id in import files
824        # to avoid annoying automatic number transformation
825        # by Excel or Calc
826        p_id = p_id.strip('#')
827        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
828            # For data migration from old SRP only
829            p_id = 'p' + p_id[7:] + '0'
830        entry = payments.get(p_id)
831        return entry
832
833    def updateEntry(self, obj, row, site, filename):
834        """Update obj to the values given in row.
835        """
836        items_changed = super(StudentOnlinePaymentProcessor, self).updateEntry(
837            obj, row, site, filename)
838        student = self.getParent(row, site).__parent__
839        student.__parent__.logger.info(
840            '%s - %s - %s - updated: %s'
841            % (self.name, filename, student.student_id, items_changed))
842        return
843
844    def samePaymentMade(self, student, category, p_session):
845        for key in student['payments'].keys():
846            ticket = student['payments'][key]
847            if ticket.p_state == 'paid' and\
848               ticket.p_category == category and \
849               ticket.p_session == p_session and \
850               ticket.p_item != 'Balance':
851                  return True
852        return False
853
854    def addEntry(self, obj, row, site):
855        parent = self.getParent(row, site)
856        student = parent.student
857        p_id = row['p_id'].strip('#')
858        # Requirement added on 19/02/2015: same payment must not exist.
859        if obj.p_item != 'Balance' and self.samePaymentMade(
860            student, obj.p_category, obj.p_session):
861            student.__parent__.logger.info(
862                '%s - %s - previous update cancelled'
863                % (self.name, student.student_id))
864            raise DuplicationError('Same payment has already been made.')
865        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
866            # For data migration from old SRP
867            obj.p_id = 'p' + p_id[7:] + '0'
868            parent[obj.p_id] = obj
869        else:
870            parent[p_id] = obj
871        return
872
873    def delEntry(self, row, site):
874        payment = self.getEntry(row, site)
875        parent = self.getParent(row, site)
876        if payment is not None:
877            student = self._getStudent(row, site)
878            student.__parent__.logger.info('%s - Payment ticket removed: %s'
879                % (student.student_id, payment.p_id))
880            del parent[payment.p_id]
881        return
882
883    def checkConversion(self, row, mode='ignore'):
884        """Validates all values in row.
885        """
886        errs, inv_errs, conv_dict = super(
887            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
888
889        # We have to check p_id.
890        p_id = row.get('p_id', None)
891        if mode == 'create' and p_id in (None, IGNORE_MARKER):
892            timestamp = ("%d" % int(time()*10000))[1:]
893            p_id = "p%s" % timestamp
894            conv_dict['p_id'] = p_id
895            return errs, inv_errs, conv_dict
896        elif p_id in (None, IGNORE_MARKER):
897            errs.append(('p_id','missing'))
898            return errs, inv_errs, conv_dict
899        else:
900            p_id = p_id.strip('#')
901            if p_id.startswith('p'):
902                if not len(p_id) == 14:
903                    errs.append(('p_id','invalid length'))
904                    return errs, inv_errs, conv_dict
905            elif len(p_id.split('-')) == 3:
906                # The SRP used either pins as keys ...
907                if len(p_id.split('-')[2]) not in (9, 10):
908                    errs.append(('p_id','invalid pin'))
909                    return errs, inv_errs, conv_dict
910            else:
911                # ... or order_ids.
912                if not len(p_id) == 19:
913                    errs.append(('p_id','invalid format'))
914                    return errs, inv_errs, conv_dict
915        # Requirement added on 24/01/2015: p_id must be portal-wide unique.
916        if mode == 'create':
917            cat = getUtility(ICatalog, name='payments_catalog')
918            results = list(cat.searchResults(p_id=(p_id, p_id)))
919            if len(results) > 0:
920                sids = [IPayer(payment).id for payment in results]
921                sids_string = ''
922                for id in sids:
923                    sids_string += '%s ' % id
924                errs.append(('p_id','p_id exists in %s' % sids_string))
925                return errs, inv_errs, conv_dict
926        return errs, inv_errs, conv_dict
927
928class StudentVerdictProcessor(StudentStudyCourseProcessor):
929    """The Student Verdict Processor inherits from the Student Study
930    Course Processor. It's a pure updater. Import step 2 raises a warning
931    message if a datacenter manager tries to select another mode.
932    But it does more than only overwriting study course attributes.
933
934    The Student Verdict Processor is the only processor which cannot be
935    used for restoring data. Purpose is to announce a verdict at the end of
936    each academic session. The processor does not only import a verdict,
937    it also conditions the student data so that the student can pay for the
938    next session and proceed to the next study level.
939
940    The `checkUpdateRequirements` method ensures that the imported data
941    really correspond to the actual state of the student.
942    `current_level` and `current_session` in the row must be on par
943    with the attributes of the study course object. Thus, the processor
944    does not use these values to overwrite the attributes of the study course
945    but to control that the verdict is really meant for the current session of
946    the student. The verdict is only imported if a corresponding study level
947    object exists and the student is in the right registration state,
948    either ``courses validated`` or ``courses registered``. Course registration
949    can be bypassed by setting `bypass_validation` to ``True``.
950
951    The `updateEntry` method does not only update the current verdict of
952    the student study course, it also updates the matching student study
953    level object. It saves the current verdict as `level_verdict` and sets
954    the `validated_by` and `validation_date` attributes, whereas `validated_by`
955    is taken from the row of the import file and `validation_date` is set to the
956    actual UTC datetime. Finally, the student is moved to state ``returning``.
957    """
958
959    util_name = 'verdictupdater'
960    grok.name(util_name)
961
962    name = _('Verdict Processor (special processor, update only)')
963    iface = IStudentVerdictUpdate
964    factory_name = 'waeup.StudentStudyCourse'
965
966    additional_fields_required = [
967        'current_level', 'current_session', 'current_verdict']
968
969    def checkUpdateRequirements(self, obj, row, site):
970        """Checks requirements the studycourse and the student must fulfill
971        before being updated.
972        """
973        # Check if current_levels correspond
974        if obj.current_level != row['current_level']:
975            return 'Current level does not correspond.'
976        # Check if current_sessions correspond
977        if obj.current_session != row['current_session']:
978            return 'Current session does not correspond.'
979        # Check if new verdict is provided
980        if row['current_verdict'] in (IGNORE_MARKER, ''):
981            return 'No verdict in import file.'
982        # Check if studylevel exists
983        level_string = str(obj.current_level)
984        if obj.get(level_string) is None:
985            return 'Study level object is missing.'
986        # Check if student is in state REGISTERED or VALIDATED
987        if row.get('bypass_validation'):
988            if obj.student.state not in (VALIDATED, REGISTERED):
989                return 'Student in wrong state.'
990        else:
991            if obj.student.state != VALIDATED:
992                return 'Student in wrong state.'
993        return None
994
995    def updateEntry(self, obj, row, site, filename):
996        """Update obj to the values given in row.
997        """
998        # Don't set current_session, current_level
999        vals_to_set = dict((key, val) for key, val in row.items()
1000                           if key not in ('current_session','current_level'))
1001        super(StudentVerdictProcessor, self).updateEntry(
1002            obj, vals_to_set, site, filename)
1003        parent = self.getParent(row, site)
1004        # Set current_verdict in corresponding studylevel
1005        level_string = str(obj.current_level)
1006        obj[level_string].level_verdict = row['current_verdict']
1007        # Fire transition and set studylevel attributes
1008        # depending on student's state
1009        if obj.__parent__.state == REGISTERED:
1010            validated_by = row.get('validated_by', '')
1011            if validated_by in (IGNORE_MARKER, ''):
1012                portal_language = getUtility(IKofaUtils).PORTAL_LANGUAGE
1013                system = translate(_('System'),'waeup.kofa',
1014                                  target_language=portal_language)
1015                obj[level_string].validated_by = system
1016            else:
1017                obj[level_string].validated_by = validated_by
1018            obj[level_string].validation_date = datetime.utcnow()
1019            IWorkflowInfo(obj.__parent__).fireTransition('bypass_validation')
1020        else:
1021            IWorkflowInfo(obj.__parent__).fireTransition('return')
1022        # Update the students_catalog
1023        notify(grok.ObjectModifiedEvent(obj.__parent__))
1024        return
Note: See TracBrowser for help on using the repository browser.