source: main/waeup.kofa/branches/uli-rm-bootstrap/src/waeup/kofa/students/batching.py @ 16536

Last change on this file since 16536 was 14901, checked in by Henrik Bettermann, 7 years ago

Set additional_fields_required.

  • Property svn:keywords set to Id
File size: 41.7 KB
Line 
1## $Id: batching.py 14901 2017-11-16 07:02:47Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import unicodecsv as csv # XXX: csv ops should move to dedicated module.
28from time import time
29from datetime import datetime
30from zope.i18n import translate
31from zope.interface import Interface
32from zope.schema import getFields
33from zope.component import queryUtility, getUtility, createObject
34from zope.event import notify
35from zope.catalog.interfaces import ICatalog
36from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
37from waeup.kofa.interfaces import (
38    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
39    IObjectHistory, VALIDATED, REGISTERED, IGNORE_MARKER)
40from waeup.kofa.interfaces import IKofaUtils, DuplicationError
41from waeup.kofa.interfaces import MessageFactory as _
42from waeup.kofa.students.interfaces import (
43    IStudent, IStudentStudyCourse, IStudentStudyCourseTransfer,
44    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
45    IStudentStudyLevel, ICourseTicketImport,
46    IStudentOnlinePayment, IStudentVerdictUpdate)
47from waeup.kofa.payments.interfaces import IPayer
48from waeup.kofa.students.workflow import  (
49    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS,
50    FORBIDDEN_POSTGRAD_TRANS, FORBIDDEN_POSTGRAD_STATES)
51from waeup.kofa.utils.batching import BatchProcessor
52
53class StudentProcessor(BatchProcessor):
54    """The Student Processor imports student base data.
55
56    In create mode no locator is required. If no `student_id` is given,
57    the portal automatically assigns a new student id.
58
59    In update or remove mode the processor uses
60    either the `student_id`, `reg_number` or `matric_number` to localize the
61    student object, exactly in this order. If `student_id` is given and an
62    object can be found, `reg_number` and `matric_number` will be overwritten
63    by the values provided in the import file. If `student_id` is missing,
64    `reg_number` is used to localize the object and only `matric_number`
65    will be overwritten. `matric_number` is used as locator only if both
66    `student_id` and `reg_number` are missing. `student_id` can't be changed
67    by the batch processor.
68
69    There are two ways to change the workflow state of the student,
70    an unsafe and a safe way. The safe way makes use of workflow transitions.
71    Transitions are only possible between allowed workflow states. Only
72    transitions ensure that the registration workflow is maintained.
73
74    **Always prefer the safe way!**
75    """
76    grok.implements(IBatchProcessor)
77    grok.provides(IBatchProcessor)
78    grok.context(Interface)
79    util_name = 'studentprocessor'
80    grok.name(util_name)
81
82    name = _('Student Processor')
83    iface = IStudent
84    iface_byregnumber = IStudentUpdateByRegNo
85    iface_bymatricnumber = IStudentUpdateByMatricNo
86
87    factory_name = 'waeup.Student'
88
89    @property
90    def available_fields(self):
91        fields = getFields(self.iface)
92        return sorted(list(set(
93            ['student_id','reg_number','matric_number',
94            'password', 'state', 'transition'] + fields.keys())))
95
96    def checkHeaders(self, headerfields, mode='create'):
97        if 'state' in headerfields and 'transition' in headerfields:
98            raise FatalCSVError(
99                "State and transition can't be imported at the same time!")
100        if not 'reg_number' in headerfields and not 'student_id' \
101            in headerfields and not 'matric_number' in headerfields:
102            raise FatalCSVError(
103                "Need at least columns student_id or reg_number " +
104                "or matric_number for import!")
105        if mode == 'create':
106            for field in self.required_fields:
107                if not field in headerfields:
108                    raise FatalCSVError(
109                        "Need at least columns %s for import!" %
110                        ', '.join(["'%s'" % x for x in self.required_fields]))
111        # Check for fields to be ignored...
112        not_ignored_fields = [x for x in headerfields
113                              if not x.startswith('--')]
114        if len(set(not_ignored_fields)) < len(not_ignored_fields):
115            raise FatalCSVError(
116                "Double headers: each column name may only appear once.")
117        return True
118
119    def parentsExist(self, row, site):
120        return 'students' in site.keys()
121
122    def getLocator(self, row):
123        if row.get('student_id',None) not in (None, IGNORE_MARKER):
124            return 'student_id'
125        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
126            return 'reg_number'
127        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
128            return 'matric_number'
129        else:
130            return None
131
132    # The entry never exists in create mode.
133    def entryExists(self, row, site):
134        return self.getEntry(row, site) is not None
135
136    def getParent(self, row, site):
137        return site['students']
138
139    def getEntry(self, row, site):
140        if not 'students' in site.keys():
141            return None
142        if self.getLocator(row) == 'student_id':
143            if row['student_id'] in site['students']:
144                student = site['students'][row['student_id']]
145                return student
146        elif self.getLocator(row) == 'reg_number':
147            reg_number = row['reg_number']
148            cat = queryUtility(ICatalog, name='students_catalog')
149            results = list(
150                cat.searchResults(reg_number=(reg_number, reg_number)))
151            if results:
152                return results[0]
153        elif self.getLocator(row) == 'matric_number':
154            matric_number = row['matric_number']
155            cat = queryUtility(ICatalog, name='students_catalog')
156            results = list(
157                cat.searchResults(matric_number=(matric_number, matric_number)))
158            if results:
159                return results[0]
160        return None
161
162    def addEntry(self, obj, row, site):
163        parent = self.getParent(row, site)
164        parent.addStudent(obj)
165        # Reset _curr_stud_id if student_id has been imported
166        if self.getLocator(row) == 'student_id':
167            parent._curr_stud_id -= 1
168        # We have to log this if state is provided. If not,
169        # logging is done by the event handler handle_student_added
170        if 'state' in row:
171            parent.logger.info('%s - Student record created' % obj.student_id)
172        history = IObjectHistory(obj)
173        history.addMessage(_('Student record created'))
174        return
175
176    def delEntry(self, row, site):
177        student = self.getEntry(row, site)
178        if student is not None:
179            parent = self.getParent(row, site)
180            parent.logger.info('%s - Student removed' % student.student_id)
181            del parent[student.student_id]
182        pass
183
184    def checkUpdateRequirements(self, obj, row, site):
185        """Checks requirements the object must fulfill when being updated.
186
187        This method is not used in case of deleting or adding objects.
188
189        Returns error messages as strings in case of requirement
190        problems.
191        """
192        transition = row.get('transition', IGNORE_MARKER)
193        if transition not in (IGNORE_MARKER, ''):
194            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
195            if transition not in allowed_transitions:
196                return 'Transition not allowed.'
197            if transition in FORBIDDEN_POSTGRAD_TRANS and \
198                obj.is_postgrad:
199                return 'Transition not allowed (pg student).'
200        state = row.get('state', IGNORE_MARKER)
201        if state not in (IGNORE_MARKER, ''):
202            if state in FORBIDDEN_POSTGRAD_STATES and \
203                obj.is_postgrad:
204                return 'State not allowed (pg student).'
205        return None
206
207    def updateEntry(self, obj, row, site, filename):
208        """Update obj to the values given in row.
209        """
210        items_changed = ''
211
212        # Remove student_id from row if empty
213        if 'student_id' in row and row['student_id'] in (None, IGNORE_MARKER):
214            row.pop('student_id')
215
216        # Update password
217        # XXX: Take DELETION_MARKER into consideration
218        if 'password' in row:
219            passwd = row.get('password', IGNORE_MARKER)
220            if passwd not in ('', IGNORE_MARKER):
221                if passwd.startswith('{SSHA}'):
222                    # already encrypted password
223                    obj.password = passwd
224                else:
225                    # not yet encrypted password
226                    IUserAccount(obj).setPassword(passwd)
227                items_changed += ('%s=%s, ' % ('password', passwd))
228            row.pop('password')
229
230        # Update registration state
231        if 'state' in row:
232            state = row.get('state', IGNORE_MARKER)
233            if state not in (IGNORE_MARKER, ''):
234                value = row['state']
235                IWorkflowState(obj).setState(value)
236                msg = _("State '${a}' set", mapping = {'a':value})
237                history = IObjectHistory(obj)
238                history.addMessage(msg)
239                items_changed += ('%s=%s, ' % ('state', state))
240            row.pop('state')
241
242        if 'transition' in row:
243            transition = row.get('transition', IGNORE_MARKER)
244            if transition not in (IGNORE_MARKER, ''):
245                value = row['transition']
246                IWorkflowInfo(obj).fireTransition(value)
247                items_changed += ('%s=%s, ' % ('transition', transition))
248            row.pop('transition')
249
250        # apply other values...
251        items_changed += super(StudentProcessor, self).updateEntry(
252            obj, row, site, filename)
253
254        # Log actions...
255        parent = self.getParent(row, site)
256        if hasattr(obj,'student_id'):
257            # Update mode: the student exists and we can get the student_id.
258            # Create mode: the record contains the student_id
259            parent.logger.info(
260                '%s - %s - %s - updated: %s'
261                % (self.name, filename, obj.student_id, items_changed))
262        else:
263            # Create mode: the student does not yet exist
264            # XXX: It seems that this never happens because student_id
265            # is always set.
266            parent.logger.info(
267                '%s - %s - %s - imported: %s'
268                % (self.name, filename, obj.student_id, items_changed))
269        return items_changed
270
271    def getMapping(self, path, headerfields, mode):
272        """Get a mapping from CSV file headerfields to actually used fieldnames.
273        """
274        result = dict()
275        reader = csv.reader(open(path, 'rb'))
276        raw_header = reader.next()
277        for num, field in enumerate(headerfields):
278            if field not in ['student_id', 'reg_number', 'matric_number'
279                             ] and mode == 'remove':
280                continue
281            if field == u'--IGNORE--':
282                # Skip ignored columns in failed and finished data files.
283                continue
284            result[raw_header[num]] = field
285        return result
286
287    def checkConversion(self, row, mode='create'):
288        """Validates all values in row.
289        """
290        iface = self.iface
291        if mode in ['update', 'remove']:
292            if self.getLocator(row) == 'reg_number':
293                iface = self.iface_byregnumber
294            elif self.getLocator(row) == 'matric_number':
295                iface = self.iface_bymatricnumber
296        converter = IObjectConverter(iface)
297        errs, inv_errs, conv_dict =  converter.fromStringDict(
298            row, self.factory_name, mode=mode)
299        # We cannot import both state and transition.
300        if 'transition' in row and 'state' in row:
301            if row['transition'] not in (IGNORE_MARKER, '') and \
302                row['state'] not in (IGNORE_MARKER, ''):
303                errs.append(('workflow','not allowed'))
304                return errs, inv_errs, conv_dict
305        if 'transition' in row:
306            if row['transition'] not in IMPORTABLE_TRANSITIONS:
307                if row['transition'] not in (IGNORE_MARKER, ''):
308                    errs.append(('transition','not allowed'))
309        if 'state' in row:
310            if row['state'] not in IMPORTABLE_STATES:
311                if row['state'] not in (IGNORE_MARKER, ''):
312                    errs.append(('state','not allowed'))
313                else:
314                    # State is an attribute of Student and must not
315                    # be changed if empty.
316                    conv_dict['state'] = IGNORE_MARKER
317        try:
318            # Correct stud_id counter. As the IConverter for students
319            # creates student objects that are not used afterwards, we
320            # have to fix the site-wide student_id counter.
321            site = grok.getSite()
322            students = site['students']
323            students._curr_stud_id -= 1
324        except (KeyError, TypeError, AttributeError):
325                pass
326        return errs, inv_errs, conv_dict
327
328
329class StudentProcessorBase(BatchProcessor):
330    """A base for student subitem processor.
331
332    Helps reducing redundancy.
333    """
334    grok.baseclass()
335
336    # additional available fields
337    # beside 'student_id', 'reg_number' and 'matric_number'
338    additional_fields = []
339
340    # additional required fields (subset of additional_fields)
341    additional_fields_required = []
342
343    @property
344    def available_fields(self):
345        fields = ['student_id','reg_number','matric_number'
346                  ] + self.additional_fields
347        return sorted(list(set(fields + getFields(
348                self.iface).keys())))
349
350    def checkHeaders(self, headerfields, mode='ignore'):
351        if not 'reg_number' in headerfields and not 'student_id' \
352            in headerfields and not 'matric_number' in headerfields:
353            raise FatalCSVError(
354                "Need at least columns student_id " +
355                "or reg_number or matric_number for import!")
356        for name in self.additional_fields_required:
357            if not name in headerfields:
358                raise FatalCSVError(
359                    "Need %s for import!" % name)
360
361        # Check for fields to be ignored...
362        not_ignored_fields = [x for x in headerfields
363                              if not x.startswith('--')]
364        if len(set(not_ignored_fields)) < len(not_ignored_fields):
365            raise FatalCSVError(
366                "Double headers: each column name may only appear once.")
367        return True
368
369    def _getStudent(self, row, site):
370        NON_VALUES = ['', IGNORE_MARKER]
371        if not 'students' in site.keys():
372            return None
373        if row.get('student_id', '') not in NON_VALUES:
374            if row['student_id'] in site['students']:
375                student = site['students'][row['student_id']]
376                return student
377        elif row.get('reg_number', '') not in NON_VALUES:
378            reg_number = row['reg_number']
379            cat = queryUtility(ICatalog, name='students_catalog')
380            results = list(
381                cat.searchResults(reg_number=(reg_number, reg_number)))
382            if results:
383                return results[0]
384        elif row.get('matric_number', '') not in NON_VALUES:
385            matric_number = row['matric_number']
386            cat = queryUtility(ICatalog, name='students_catalog')
387            results = list(
388                cat.searchResults(matric_number=(matric_number, matric_number)))
389            if results:
390                return results[0]
391        return None
392
393    def parentsExist(self, row, site):
394        return self.getParent(row, site) is not None
395
396    def entryExists(self, row, site):
397        return self.getEntry(row, site) is not None
398
399    def checkConversion(self, row, mode='ignore'):
400        """Validates all values in row.
401        """
402        converter = IObjectConverter(self.iface)
403        errs, inv_errs, conv_dict =  converter.fromStringDict(
404            row, self.factory_name, mode=mode)
405        return errs, inv_errs, conv_dict
406
407    def getMapping(self, path, headerfields, mode):
408        """Get a mapping from CSV file headerfields to actually used fieldnames.
409        """
410        result = dict()
411        reader = csv.reader(open(path, 'rb'))
412        raw_header = reader.next()
413        for num, field in enumerate(headerfields):
414            if field not in ['student_id', 'reg_number', 'matric_number',
415                             'p_id', 'code', 'level'
416                             ] and mode == 'remove':
417                continue
418            if field == u'--IGNORE--':
419                # Skip ignored columns in failed and finished data files.
420                continue
421            result[raw_header[num]] = field
422        return result
423
424
425class StudentStudyCourseProcessor(StudentProcessorBase):
426    """The Student Study Course Processor imports data which refer
427    to the student's course of study. The study course container data
428    describe the current state of the course of study and it stores the
429    entry conditions, i.e. when the student started the course.
430
431    Most important is the `certificate` attribute which tells us which course
432    the student is studying. The terms 'study course' and 'course of study'
433    are used synonymously. The 'certificate' is the study programme described
434    in the acadmic section. The study course object stores a referrer to a
435    certificate in the acadmic section.
436
437    When importing a new certificate code, `checkConversion` does not only
438    check whether a certificate with the same code exists, it also
439    proves if `current_level` is inside the level range of the certificate.
440    For example, some study programmes start at level 200. The imported
441    current level must thus be 200 or higher.
442
443    `checkUpdateRequirements` looks up if the imported values match the
444    certificate already stored with the study course object. The imported
445    `current_level` must be in the range of the certificate already
446    stored.
447
448    .. note::
449
450      The processor does only offer an update mode. An 'empty' study course
451      object is automatically created when the student object is added. So this
452      object always exists. It can neither be added a second time nor
453      be removed.
454
455    Students can be transferred by import. A transfer is initialized if the
456    `entry_mode` value is ``transfer``. In this case `checkConversion` uses a
457    different interface for data validation and `checkUpdateRequirements`
458    ensures that a student can only be transferred twice. The student transfer
459    process is described elsewhere.
460    """
461    grok.implements(IBatchProcessor)
462    grok.provides(IBatchProcessor)
463    grok.context(Interface)
464    util_name = 'studycourseupdater'
465    grok.name(util_name)
466
467    name = _('StudentStudyCourse Processor (update only)')
468    iface = IStudentStudyCourse
469    iface_transfer = IStudentStudyCourseTransfer
470    factory_name = 'waeup.StudentStudyCourse'
471
472    def getParent(self, row, site):
473        return self._getStudent(row, site)
474
475    def getEntry(self, row, site):
476        student = self.getParent(row, site)
477        if student is None:
478            return None
479        return student.get('studycourse')
480
481    def updateEntry(self, obj, row, site, filename):
482        """Update obj to the values given in row.
483        """
484        entry_mode = row.get('entry_mode', None)
485        certificate = row.get('certificate', None)
486        current_session = row.get('current_session', None)
487        student = self.getParent(row, site)
488        if entry_mode == 'transfer':
489            # We do not expect any error here since we
490            # checked all constraints in checkConversion and
491            # in checkUpdateRequirements
492            student.transfer(
493                certificate=certificate, current_session=current_session)
494            obj = student['studycourse']
495        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
496            obj, row, site, filename)
497        student.__parent__.logger.info(
498            '%s - %s - %s - updated: %s'
499            % (self.name, filename, student.student_id, items_changed))
500        # Update the students_catalog
501        notify(grok.ObjectModifiedEvent(student))
502        return
503
504    def checkConversion(self, row, mode='ignore'):
505        """Validates all values in row.
506        """
507        # We have to use the correct interface. Transfer
508        # updates have different constraints.
509        entry_mode = row.get('entry_mode', None)
510        if entry_mode == 'transfer':
511            converter = IObjectConverter(self.iface_transfer)
512        else:
513            converter = IObjectConverter(self.iface)
514        errs, inv_errs, conv_dict =  converter.fromStringDict(
515            row, self.factory_name, mode=mode)
516
517        # We have to check if current_level is in range of certificate.
518        if 'certificate' in conv_dict and 'current_level' in conv_dict:
519            cert = conv_dict['certificate']
520            level = conv_dict['current_level']
521            if level < cert.start_level or level > cert.end_level+120:
522                errs.append(('current_level','not in range'))
523        return errs, inv_errs, conv_dict
524
525    def checkUpdateRequirements(self, obj, row, site):
526        """Checks requirements the object must fulfill when being updated.
527        Returns error messages as strings in case of requirement
528        problems.
529        """
530        certificate = getattr(obj, 'certificate', None)
531        entry_session = getattr(obj, 'entry_session', None)
532        current_level = row.get('current_level', None)
533        entry_mode = row.get('entry_mode', None)
534        # We have to ensure that the student can be transferred.
535        if entry_mode == 'transfer':
536            if certificate is None or entry_session is None:
537                return 'Former study course record incomplete.'
538            if 'studycourse_1' in obj.__parent__.keys() and \
539                'studycourse_2' in obj.__parent__.keys():
540                return 'Maximum number of transfers exceeded.'
541        if current_level:
542            if current_level == 999 and \
543                obj.__parent__.state in FORBIDDEN_POSTGRAD_STATES:
544                return 'Not a pg student.'
545            cert = row.get('certificate', None)
546            if certificate is None and cert is None:
547                return 'No certificate to check level.'
548            if certificate is not None and cert is None and (
549                current_level < certificate.start_level or \
550                current_level > certificate.end_level+120):
551                return 'current_level not in range.'
552        return None
553
554class StudentStudyLevelProcessor(StudentProcessorBase):
555    """The Student Study Level Processor imports study level data.
556    It overwrites the container attributes but not the content of the container,
557    i.e. the course tickets stored inside the container. There is nothing
558    special about this processor.
559    """
560    grok.implements(IBatchProcessor)
561    grok.provides(IBatchProcessor)
562    grok.context(Interface)
563    util_name = 'studylevelprocessor'
564    grok.name(util_name)
565
566    name = _('StudentStudyLevel Processor')
567    iface = IStudentStudyLevel
568    factory_name = 'waeup.StudentStudyLevel'
569
570    @property
571    def available_fields(self):
572        fields = super(StudentStudyLevelProcessor, self).available_fields
573        fields.remove('total_credits')
574        fields.remove('gpa')
575        return  fields
576
577    def getParent(self, row, site):
578        student = self._getStudent(row, site)
579        if student is None:
580            return None
581        return student['studycourse']
582
583    def getEntry(self, row, site):
584        studycourse = self.getParent(row, site)
585        if studycourse is None:
586            return None
587        try:
588            entry = studycourse.get(str(row['level']))
589        except KeyError:
590            return None
591        return entry
592
593    def delEntry(self, row, site):
594        studylevel = self.getEntry(row, site)
595        parent = self.getParent(row, site)
596        if studylevel is not None:
597            student = self._getStudent(row, site)
598            student.__parent__.logger.info('%s - Level removed: %s'
599                % (student.student_id, studylevel.__name__))
600            del parent[studylevel.__name__]
601        return
602
603    def updateEntry(self, obj, row, site, filename):
604        """Update obj to the values given in row.
605        """
606        items_changed = super(StudentStudyLevelProcessor, self).updateEntry(
607            obj, row, site, filename)
608        student = self.getParent(row, site).__parent__
609        student.__parent__.logger.info(
610            '%s - %s - %s - updated: %s'
611            % (self.name, filename, student.student_id, items_changed))
612        return
613
614    def addEntry(self, obj, row, site):
615        parent = self.getParent(row, site)
616        parent[str(row['level'])] = obj
617        return
618
619class CourseTicketProcessor(StudentProcessorBase):
620    """The Course Ticket Processor imports course tickets, the subobjects
621    of student study levels (= course lists).
622
623    An imported course ticket contains a copy of the original course data.
624    During import only a few attributes can be set/overwritten.
625
626    Like all other student data importers, this processor also requires
627    either `student_id`, `reg_number` or `matric_number` to find the student.
628    Then  it needs `level` and `code` to localize the course ticket.
629
630    `checkConversion` first searches the courses catalog for the imported
631    `code` and ensures that a course with such a code really exists
632    in the academic section. It furthermore checks if `level_session` in
633    the row corresponds with the session of the parent student
634    study level object. It fails if one of the conditions is not met.
635
636    In create mode `fcode`, `dcode`, `title`, `credits`, `passmark` and
637    `semester` are taken from the course found in the academic section.
638    `fcode` and `dcode` can nevermore be changed, neither via the user interface
639    nor by import. Other values can be overwritten by import.
640    """
641    grok.implements(IBatchProcessor)
642    grok.provides(IBatchProcessor)
643    grok.context(Interface)
644    util_name = 'courseticketprocessor'
645    grok.name(util_name)
646
647    name = _('CourseTicket Processor')
648    iface = ICourseTicketImport
649    factory_name = 'waeup.CourseTicket'
650
651    additional_fields = ['level', 'code']
652    additional_fields_required = additional_fields
653
654    @property
655    def available_fields(self):
656        fields = [
657            'student_id','reg_number','matric_number',
658            'mandatory', 'score', 'carry_over', 'automatic',
659            'outstanding', 'course_category', 'level_session',
660            'title', 'credits', 'passmark', 'semester'
661            ] + self.additional_fields
662        return sorted(fields)
663
664    def getParent(self, row, site):
665        student = self._getStudent(row, site)
666        if student is None:
667            return None
668        return student['studycourse'].get(str(row['level']))
669
670    def getEntry(self, row, site):
671        level = self.getParent(row, site)
672        if level is None:
673            return None
674        return level.get(row['code'])
675
676    def updateEntry(self, obj, row, site, filename):
677        """Update obj to the values given in row.
678        """
679        items_changed = super(CourseTicketProcessor, self).updateEntry(
680            obj, row, site, filename)
681        parent = self.getParent(row, site)
682        student = self.getParent(row, site).__parent__.__parent__
683        student.__parent__.logger.info(
684            '%s - %s - %s - %s - updated: %s'
685            % (self.name, filename, student.student_id, parent.level, items_changed))
686        return
687
688    def addEntry(self, obj, row, site):
689        parent = self.getParent(row, site)
690        catalog = getUtility(ICatalog, name='courses_catalog')
691        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
692        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
693        obj.dcode = entries[0].__parent__.__parent__.code
694        if getattr(obj, 'title', None) is None:
695            obj.title = entries[0].title
696        if getattr(obj, 'credits', None) is None:
697            obj.credits = entries[0].credits
698        if getattr(obj, 'passmark', None) is None:
699            obj.passmark = entries[0].passmark
700        if getattr(obj, 'semester', None) is None:
701            obj.semester = entries[0].semester
702        parent[row['code']] = obj
703        return
704
705    def delEntry(self, row, site):
706        ticket = self.getEntry(row, site)
707        parent = self.getParent(row, site)
708        if ticket is not None:
709            student = self._getStudent(row, site)
710            student.__parent__.logger.info('%s - Course ticket in %s removed: %s'
711                % (student.student_id, parent.level, ticket.code))
712            del parent[ticket.code]
713        return
714
715    def checkConversion(self, row, mode='ignore'):
716        """Validates all values in row.
717        """
718        errs, inv_errs, conv_dict = super(
719            CourseTicketProcessor, self).checkConversion(row, mode=mode)
720        if mode == 'remove':
721            return errs, inv_errs, conv_dict
722        # In update and create mode we have to check if course really exists.
723        # This is not done by the converter.
724        catalog = getUtility(ICatalog, name='courses_catalog')
725        entries = catalog.searchResults(code=(row['code'],row['code']))
726        if len(entries) == 0:
727            errs.append(('code','non-existent'))
728            return errs, inv_errs, conv_dict
729        # If level_session is provided in row we have to check if
730        # the parent studylevel exists and if its level_session
731        # attribute corresponds with the expected value in row.
732        level_session = conv_dict.get('level_session', IGNORE_MARKER)
733        if level_session not in (IGNORE_MARKER, None):
734            site = grok.getSite()
735            studylevel = self.getParent(row, site)
736            if studylevel is not None:
737                if studylevel.level_session != level_session:
738                    errs.append(('level_session','does not match %s'
739                        % studylevel.level_session))
740            else:
741                errs.append(('level object','does not exist'))
742        return errs, inv_errs, conv_dict
743
744class StudentOnlinePaymentProcessor(StudentProcessorBase):
745    """The Student Online Payment Processor imports student payment tickets.
746    The tickets are located in the ``payments`` subfolder of the student
747    container. The only additional locator is `p_id`, the object id.
748
749    The `checkConversion` method checks the format of the payment identifier.
750    In create mode it does also ensures that same `p_id` does not exist
751    elsewhere. It must be portal-wide unique.
752
753    When adding a payment ticket, the `addEntry` method checks if the same
754    payment has already been made. It compares `p_category` and `p_session`
755    in the row with the corresponding attributes  of existing payment
756    tickets in state ``paid``. If they match, a `DuplicationError` is raised.
757    """
758    grok.implements(IBatchProcessor)
759    grok.provides(IBatchProcessor)
760    grok.context(Interface)
761    util_name = 'paymentprocessor'
762    grok.name(util_name)
763
764    name = _('StudentOnlinePayment Processor')
765    iface = IStudentOnlinePayment
766    factory_name = 'waeup.StudentOnlinePayment'
767
768    additional_fields = ['p_id']
769
770    @property
771    def available_fields(self):
772        af = super(
773            StudentOnlinePaymentProcessor, self).available_fields
774        af.remove('display_item')
775        return af
776
777    def checkHeaders(self, headerfields, mode='ignore'):
778        super(StudentOnlinePaymentProcessor, self).checkHeaders(headerfields)
779        if mode in ('update', 'remove') and not 'p_id' in headerfields:
780            raise FatalCSVError(
781                "Need p_id for import in update and remove modes!")
782        return True
783
784    def parentsExist(self, row, site):
785        return self.getParent(row, site) is not None
786
787    def getParent(self, row, site):
788        student = self._getStudent(row, site)
789        if student is None:
790            return None
791        return student['payments']
792
793    def getEntry(self, row, site):
794        payments = self.getParent(row, site)
795        if payments is None:
796            return None
797        p_id = row.get('p_id', None)
798        if p_id in (None, IGNORE_MARKER):
799            return None
800        # We can use the hash symbol at the end of p_id in import files
801        # to avoid annoying automatic number transformation
802        # by Excel or Calc
803        p_id = p_id.strip('#')
804        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
805            # For data migration from old SRP only
806            p_id = 'p' + p_id[7:] + '0'
807        entry = payments.get(p_id)
808        return entry
809
810    def updateEntry(self, obj, row, site, filename):
811        """Update obj to the values given in row.
812        """
813        items_changed = super(StudentOnlinePaymentProcessor, self).updateEntry(
814            obj, row, site, filename)
815        student = self.getParent(row, site).__parent__
816        student.__parent__.logger.info(
817            '%s - %s - %s - updated: %s'
818            % (self.name, filename, student.student_id, items_changed))
819        return
820
821    def samePaymentMade(self, student, category, p_session):
822        for key in student['payments'].keys():
823            ticket = student['payments'][key]
824            if ticket.p_state == 'paid' and\
825               ticket.p_category == category and \
826               ticket.p_session == p_session and \
827               ticket.p_item != 'Balance':
828                  return True
829        return False
830
831    def addEntry(self, obj, row, site):
832        parent = self.getParent(row, site)
833        student = parent.student
834        p_id = row['p_id'].strip('#')
835        # Requirement added on 19/02/2015: same payment must not exist.
836        if obj.p_item != 'Balance' and self.samePaymentMade(
837            student, obj.p_category, obj.p_session):
838            student.__parent__.logger.info(
839                '%s - %s - previous update cancelled'
840                % (self.name, student.student_id))
841            raise DuplicationError('Same payment has already been made.')
842        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
843            # For data migration from old SRP
844            obj.p_id = 'p' + p_id[7:] + '0'
845            parent[obj.p_id] = obj
846        else:
847            parent[p_id] = obj
848        return
849
850    def delEntry(self, row, site):
851        payment = self.getEntry(row, site)
852        parent = self.getParent(row, site)
853        if payment is not None:
854            student = self._getStudent(row, site)
855            student.__parent__.logger.info('%s - Payment ticket removed: %s'
856                % (student.student_id, payment.p_id))
857            del parent[payment.p_id]
858        return
859
860    def checkConversion(self, row, mode='ignore'):
861        """Validates all values in row.
862        """
863        errs, inv_errs, conv_dict = super(
864            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
865
866        # We have to check p_id.
867        p_id = row.get('p_id', None)
868        if mode == 'create' and p_id in (None, IGNORE_MARKER):
869            timestamp = ("%d" % int(time()*10000))[1:]
870            p_id = "p%s" % timestamp
871            conv_dict['p_id'] = p_id
872            return errs, inv_errs, conv_dict
873        elif p_id in (None, IGNORE_MARKER):
874            errs.append(('p_id','missing'))
875            return errs, inv_errs, conv_dict
876        else:
877            p_id = p_id.strip('#')
878            if p_id.startswith('p'):
879                if not len(p_id) == 14:
880                    errs.append(('p_id','invalid length'))
881                    return errs, inv_errs, conv_dict
882            elif len(p_id.split('-')) == 3:
883                # The SRP used either pins as keys ...
884                if len(p_id.split('-')[2]) not in (9, 10):
885                    errs.append(('p_id','invalid pin'))
886                    return errs, inv_errs, conv_dict
887            else:
888                # ... or order_ids.
889                if not len(p_id) == 19:
890                    errs.append(('p_id','invalid format'))
891                    return errs, inv_errs, conv_dict
892        # Requirement added on 24/01/2015: p_id must be portal-wide unique.
893        if mode == 'create':
894            cat = getUtility(ICatalog, name='payments_catalog')
895            results = list(cat.searchResults(p_id=(p_id, p_id)))
896            if len(results) > 0:
897                sids = [IPayer(payment).id for payment in results]
898                sids_string = ''
899                for id in sids:
900                    sids_string += '%s ' % id
901                errs.append(('p_id','p_id exists in %s' % sids_string))
902                return errs, inv_errs, conv_dict
903        return errs, inv_errs, conv_dict
904
905class StudentVerdictProcessor(StudentStudyCourseProcessor):
906    """The Student Verdict Processor inherits from the Student Study
907    Course Processor. It's a pure updater. Import step 2 raises a warning
908    message if a datacenter manager tries to select another mode.
909    But it does more than only overwriting study course attributes.
910
911    The Student Verdict Processor is the only processor which cannot be
912    used for restoring data. Purpose is to announce a verdict at the end of
913    each academic session. The processor does not only import a verdict,
914    it also conditions the student data so that the student can pay for the
915    next session and proceed to the next study level.
916
917    The `checkUpdateRequirements` method ensures that the imported data
918    really correspond to the actual state of the student.
919    `current_level` and `current_session` in the row must be on par
920    with the attributes of the study course object. Thus, the processor
921    does not use these values to overwrite the attributes of the study course
922    but to control that the verdict is really meant for the current session of
923    the student. The verdict is only imported if a corresponding study level
924    object exists and the student is in the right registration state,
925    either ``courses validated`` or ``courses registered``. Course registration
926    can be bypassed by setting `bypass_validation` to ``True``.
927
928    The `updateEntry` method does not only update the current verdict of
929    the student study course, it also updates the matching student study
930    level object. It saves the current verdict as `level_verdict` and sets
931    the `validated_by` and `validation_date` attributes, whereas `validated_by`
932    is taken from the row of the import file and `validation_date` is set to the
933    actual UTC datetime. Finally, the student is moved to state ``returning``.
934    """
935
936    util_name = 'verdictupdater'
937    grok.name(util_name)
938
939    name = _('Verdict Processor (special processor, update only)')
940    iface = IStudentVerdictUpdate
941    factory_name = 'waeup.StudentStudyCourse'
942
943    additional_fields_required = [
944        'current_level', 'current_session', 'current_verdict']
945
946    def checkUpdateRequirements(self, obj, row, site):
947        """Checks requirements the studycourse and the student must fulfill
948        before being updated.
949        """
950        # Check if current_levels correspond
951        if obj.current_level != row['current_level']:
952            return 'Current level does not correspond.'
953        # Check if current_sessions correspond
954        if obj.current_session != row['current_session']:
955            return 'Current session does not correspond.'
956        # Check if new verdict is provided
957        if row['current_verdict'] in (IGNORE_MARKER, ''):
958            return 'No verdict in import file.'
959        # Check if studylevel exists
960        level_string = str(obj.current_level)
961        if obj.get(level_string) is None:
962            return 'Study level object is missing.'
963        # Check if student is in state REGISTERED or VALIDATED
964        if row.get('bypass_validation'):
965            if obj.student.state not in (VALIDATED, REGISTERED):
966                return 'Student in wrong state.'
967        else:
968            if obj.student.state != VALIDATED:
969                return 'Student in wrong state.'
970        return None
971
972    def updateEntry(self, obj, row, site, filename):
973        """Update obj to the values given in row.
974        """
975        # Don't set current_session, current_level
976        vals_to_set = dict((key, val) for key, val in row.items()
977                           if key not in ('current_session','current_level'))
978        super(StudentVerdictProcessor, self).updateEntry(
979            obj, vals_to_set, site, filename)
980        parent = self.getParent(row, site)
981        # Set current_verdict in corresponding studylevel
982        level_string = str(obj.current_level)
983        obj[level_string].level_verdict = row['current_verdict']
984        # Fire transition and set studylevel attributes
985        # depending on student's state
986        if obj.__parent__.state == REGISTERED:
987            validated_by = row.get('validated_by', '')
988            if validated_by in (IGNORE_MARKER, ''):
989                portal_language = getUtility(IKofaUtils).PORTAL_LANGUAGE
990                system = translate(_('System'),'waeup.kofa',
991                                  target_language=portal_language)
992                obj[level_string].validated_by = system
993            else:
994                obj[level_string].validated_by = validated_by
995            obj[level_string].validation_date = datetime.utcnow()
996            IWorkflowInfo(obj.__parent__).fireTransition('bypass_validation')
997        else:
998            IWorkflowInfo(obj.__parent__).fireTransition('return')
999        # Update the students_catalog
1000        notify(grok.ObjectModifiedEvent(obj.__parent__))
1001        return
Note: See TracBrowser for help on using the repository browser.