source: main/waeup.kofa/trunk/src/waeup/kofa/students/batching.py @ 14194

Last change on this file since 14194 was 14189, checked in by Henrik Bettermann, 8 years ago

Make 'title', 'credits', 'passmark' and 'semester' importable.

  • Property svn:keywords set to Id
File size: 41.5 KB
Line 
1## $Id: batching.py 14189 2016-09-26 10:38:29Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import unicodecsv as csv # XXX: csv ops should move to dedicated module.
28from time import time
29from datetime import datetime
30from zope.i18n import translate
31from zope.interface import Interface
32from zope.schema import getFields
33from zope.component import queryUtility, getUtility, createObject
34from zope.event import notify
35from zope.catalog.interfaces import ICatalog
36from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
37from waeup.kofa.interfaces import (
38    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
39    IObjectHistory, VALIDATED, REGISTERED, IGNORE_MARKER)
40from waeup.kofa.interfaces import IKofaUtils, DuplicationError
41from waeup.kofa.interfaces import MessageFactory as _
42from waeup.kofa.students.interfaces import (
43    IStudent, IStudentStudyCourse, IStudentStudyCourseTransfer,
44    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
45    IStudentStudyLevel, ICourseTicketImport,
46    IStudentOnlinePayment, IStudentVerdictUpdate)
47from waeup.kofa.payments.interfaces import IPayer
48from waeup.kofa.students.workflow import  (
49    IMPORTABLE_STATES, IMPORTABLE_TRANSITIONS,
50    FORBIDDEN_POSTGRAD_TRANS, FORBIDDEN_POSTGRAD_STATES)
51from waeup.kofa.utils.batching import BatchProcessor
52
53class StudentProcessor(BatchProcessor):
54    """The Student Processor imports student base data.
55
56    In create mode no locator is required. If no `student_id` is given,
57    the portal automatically assigns a new student id.
58
59    In update or remove mode the processor uses
60    either the `student_id`, `reg_number` or `matric_number` to localize the
61    student object, exactly in this order. If `student_id` is given and an
62    object can be found, `reg_number` and `matric_number` will be overwritten
63    by the values provided in the import file. If `student_id` is missing,
64    `reg_number` is used to localize the object and only `matric_number`
65    will be overwritten. `matric_number` is used as locator only if both
66    `student_id` and `reg_number` are missing. `student_id` can't be changed
67    by the batch processor.
68
69    There are two ways to change the workflow state of the student,
70    an unsafe and a safe way. The safe way makes use of workflow transitions.
71    Transitions are only possible between allowed workflow states. Only
72    transitions ensure that the registration workflow is maintained.
73
74    **Always prefer the safe way!**
75    """
76    grok.implements(IBatchProcessor)
77    grok.provides(IBatchProcessor)
78    grok.context(Interface)
79    util_name = 'studentprocessor'
80    grok.name(util_name)
81
82    name = _('Student Processor')
83    iface = IStudent
84    iface_byregnumber = IStudentUpdateByRegNo
85    iface_bymatricnumber = IStudentUpdateByMatricNo
86
87    factory_name = 'waeup.Student'
88
89    @property
90    def available_fields(self):
91        fields = getFields(self.iface)
92        return sorted(list(set(
93            ['student_id','reg_number','matric_number',
94            'password', 'state', 'transition'] + fields.keys())))
95
96    def checkHeaders(self, headerfields, mode='create'):
97        if 'state' in headerfields and 'transition' in headerfields:
98            raise FatalCSVError(
99                "State and transition can't be imported at the same time!")
100        if not 'reg_number' in headerfields and not 'student_id' \
101            in headerfields and not 'matric_number' in headerfields:
102            raise FatalCSVError(
103                "Need at least columns student_id or reg_number " +
104                "or matric_number for import!")
105        if mode == 'create':
106            for field in self.required_fields:
107                if not field in headerfields:
108                    raise FatalCSVError(
109                        "Need at least columns %s for import!" %
110                        ', '.join(["'%s'" % x for x in self.required_fields]))
111        # Check for fields to be ignored...
112        not_ignored_fields = [x for x in headerfields
113                              if not x.startswith('--')]
114        if len(set(not_ignored_fields)) < len(not_ignored_fields):
115            raise FatalCSVError(
116                "Double headers: each column name may only appear once.")
117        return True
118
119    def parentsExist(self, row, site):
120        return 'students' in site.keys()
121
122    def getLocator(self, row):
123        if row.get('student_id',None) not in (None, IGNORE_MARKER):
124            return 'student_id'
125        elif row.get('reg_number',None) not in (None, IGNORE_MARKER):
126            return 'reg_number'
127        elif row.get('matric_number',None) not in (None, IGNORE_MARKER):
128            return 'matric_number'
129        else:
130            return None
131
132    # The entry never exists in create mode.
133    def entryExists(self, row, site):
134        return self.getEntry(row, site) is not None
135
136    def getParent(self, row, site):
137        return site['students']
138
139    def getEntry(self, row, site):
140        if not 'students' in site.keys():
141            return None
142        if self.getLocator(row) == 'student_id':
143            if row['student_id'] in site['students']:
144                student = site['students'][row['student_id']]
145                return student
146        elif self.getLocator(row) == 'reg_number':
147            reg_number = row['reg_number']
148            cat = queryUtility(ICatalog, name='students_catalog')
149            results = list(
150                cat.searchResults(reg_number=(reg_number, reg_number)))
151            if results:
152                return results[0]
153        elif self.getLocator(row) == 'matric_number':
154            matric_number = row['matric_number']
155            cat = queryUtility(ICatalog, name='students_catalog')
156            results = list(
157                cat.searchResults(matric_number=(matric_number, matric_number)))
158            if results:
159                return results[0]
160        return None
161
162    def addEntry(self, obj, row, site):
163        parent = self.getParent(row, site)
164        parent.addStudent(obj)
165        # Reset _curr_stud_id if student_id has been imported
166        if self.getLocator(row) == 'student_id':
167            parent._curr_stud_id -= 1
168        # We have to log this if state is provided. If not,
169        # logging is done by the event handler handle_student_added
170        if 'state' in row:
171            parent.logger.info('%s - Student record created' % obj.student_id)
172        history = IObjectHistory(obj)
173        history.addMessage(_('Student record created'))
174        return
175
176    def delEntry(self, row, site):
177        student = self.getEntry(row, site)
178        if student is not None:
179            parent = self.getParent(row, site)
180            parent.logger.info('%s - Student removed' % student.student_id)
181            del parent[student.student_id]
182        pass
183
184    def checkUpdateRequirements(self, obj, row, site):
185        """Checks requirements the object must fulfill when being updated.
186
187        This method is not used in case of deleting or adding objects.
188
189        Returns error messages as strings in case of requirement
190        problems.
191        """
192        transition = row.get('transition', IGNORE_MARKER)
193        if transition not in (IGNORE_MARKER, ''):
194            allowed_transitions = IWorkflowInfo(obj).getManualTransitionIds()
195            if transition not in allowed_transitions:
196                return 'Transition not allowed.'
197            if transition in FORBIDDEN_POSTGRAD_TRANS and \
198                obj.is_postgrad:
199                return 'Transition not allowed (pg student).'
200        state = row.get('state', IGNORE_MARKER)
201        if state not in (IGNORE_MARKER, ''):
202            if state in FORBIDDEN_POSTGRAD_STATES and \
203                obj.is_postgrad:
204                return 'State not allowed (pg student).'
205        return None
206
207    def updateEntry(self, obj, row, site, filename):
208        """Update obj to the values given in row.
209        """
210        items_changed = ''
211
212        # Remove student_id from row if empty
213        if 'student_id' in row and row['student_id'] in (None, IGNORE_MARKER):
214            row.pop('student_id')
215
216        # Update password
217        # XXX: Take DELETION_MARKER into consideration
218        if 'password' in row:
219            passwd = row.get('password', IGNORE_MARKER)
220            if passwd not in ('', IGNORE_MARKER):
221                if passwd.startswith('{SSHA}'):
222                    # already encrypted password
223                    obj.password = passwd
224                else:
225                    # not yet encrypted password
226                    IUserAccount(obj).setPassword(passwd)
227                items_changed += ('%s=%s, ' % ('password', passwd))
228            row.pop('password')
229
230        # Update registration state
231        if 'state' in row:
232            state = row.get('state', IGNORE_MARKER)
233            if state not in (IGNORE_MARKER, ''):
234                value = row['state']
235                IWorkflowState(obj).setState(value)
236                msg = _("State '${a}' set", mapping = {'a':value})
237                history = IObjectHistory(obj)
238                history.addMessage(msg)
239                items_changed += ('%s=%s, ' % ('state', state))
240            row.pop('state')
241
242        if 'transition' in row:
243            transition = row.get('transition', IGNORE_MARKER)
244            if transition not in (IGNORE_MARKER, ''):
245                value = row['transition']
246                IWorkflowInfo(obj).fireTransition(value)
247                items_changed += ('%s=%s, ' % ('transition', transition))
248            row.pop('transition')
249
250        # apply other values...
251        items_changed += super(StudentProcessor, self).updateEntry(
252            obj, row, site, filename)
253
254        # Log actions...
255        parent = self.getParent(row, site)
256        if hasattr(obj,'student_id'):
257            # Update mode: the student exists and we can get the student_id.
258            # Create mode: the record contains the student_id
259            parent.logger.info(
260                '%s - %s - %s - updated: %s'
261                % (self.name, filename, obj.student_id, items_changed))
262        else:
263            # Create mode: the student does not yet exist
264            # XXX: It seems that this never happens because student_id
265            # is always set.
266            parent.logger.info(
267                '%s - %s - %s - imported: %s'
268                % (self.name, filename, obj.student_id, items_changed))
269        return items_changed
270
271    def getMapping(self, path, headerfields, mode):
272        """Get a mapping from CSV file headerfields to actually used fieldnames.
273        """
274        result = dict()
275        reader = csv.reader(open(path, 'rb'))
276        raw_header = reader.next()
277        for num, field in enumerate(headerfields):
278            if field not in ['student_id', 'reg_number', 'matric_number'
279                             ] and mode == 'remove':
280                continue
281            if field == u'--IGNORE--':
282                # Skip ignored columns in failed and finished data files.
283                continue
284            result[raw_header[num]] = field
285        return result
286
287    def checkConversion(self, row, mode='create'):
288        """Validates all values in row.
289        """
290        iface = self.iface
291        if mode in ['update', 'remove']:
292            if self.getLocator(row) == 'reg_number':
293                iface = self.iface_byregnumber
294            elif self.getLocator(row) == 'matric_number':
295                iface = self.iface_bymatricnumber
296        converter = IObjectConverter(iface)
297        errs, inv_errs, conv_dict =  converter.fromStringDict(
298            row, self.factory_name, mode=mode)
299        # We cannot import both state and transition.
300        if 'transition' in row and 'state' in row:
301            if row['transition'] not in (IGNORE_MARKER, '') and \
302                row['state'] not in (IGNORE_MARKER, ''):
303                errs.append(('workflow','not allowed'))
304                return errs, inv_errs, conv_dict
305        if 'transition' in row:
306            if row['transition'] not in IMPORTABLE_TRANSITIONS:
307                if row['transition'] not in (IGNORE_MARKER, ''):
308                    errs.append(('transition','not allowed'))
309        if 'state' in row:
310            if row['state'] not in IMPORTABLE_STATES:
311                if row['state'] not in (IGNORE_MARKER, ''):
312                    errs.append(('state','not allowed'))
313                else:
314                    # State is an attribute of Student and must not
315                    # be changed if empty.
316                    conv_dict['state'] = IGNORE_MARKER
317        try:
318            # Correct stud_id counter. As the IConverter for students
319            # creates student objects that are not used afterwards, we
320            # have to fix the site-wide student_id counter.
321            site = grok.getSite()
322            students = site['students']
323            students._curr_stud_id -= 1
324        except (KeyError, TypeError, AttributeError):
325                pass
326        return errs, inv_errs, conv_dict
327
328
329class StudentProcessorBase(BatchProcessor):
330    """A base for student subitem processor.
331
332    Helps reducing redundancy.
333    """
334    grok.baseclass()
335
336    # additional available fields
337    # beside 'student_id', 'reg_number' and 'matric_number'
338    additional_fields = []
339
340    # additional required fields (subset of additional_fields)
341    additional_fields_required = []
342
343    @property
344    def available_fields(self):
345        fields = ['student_id','reg_number','matric_number'
346                  ] + self.additional_fields
347        return sorted(list(set(fields + getFields(
348                self.iface).keys())))
349
350    def checkHeaders(self, headerfields, mode='ignore'):
351        if not 'reg_number' in headerfields and not 'student_id' \
352            in headerfields and not 'matric_number' in headerfields:
353            raise FatalCSVError(
354                "Need at least columns student_id " +
355                "or reg_number or matric_number for import!")
356        for name in self.additional_fields_required:
357            if not name in headerfields:
358                raise FatalCSVError(
359                    "Need %s for import!" % name)
360
361        # Check for fields to be ignored...
362        not_ignored_fields = [x for x in headerfields
363                              if not x.startswith('--')]
364        if len(set(not_ignored_fields)) < len(not_ignored_fields):
365            raise FatalCSVError(
366                "Double headers: each column name may only appear once.")
367        return True
368
369    def _getStudent(self, row, site):
370        NON_VALUES = ['', IGNORE_MARKER]
371        if not 'students' in site.keys():
372            return None
373        if row.get('student_id', '') not in NON_VALUES:
374            if row['student_id'] in site['students']:
375                student = site['students'][row['student_id']]
376                return student
377        elif row.get('reg_number', '') not in NON_VALUES:
378            reg_number = row['reg_number']
379            cat = queryUtility(ICatalog, name='students_catalog')
380            results = list(
381                cat.searchResults(reg_number=(reg_number, reg_number)))
382            if results:
383                return results[0]
384        elif row.get('matric_number', '') not in NON_VALUES:
385            matric_number = row['matric_number']
386            cat = queryUtility(ICatalog, name='students_catalog')
387            results = list(
388                cat.searchResults(matric_number=(matric_number, matric_number)))
389            if results:
390                return results[0]
391        return None
392
393    def parentsExist(self, row, site):
394        return self.getParent(row, site) is not None
395
396    def entryExists(self, row, site):
397        return self.getEntry(row, site) is not None
398
399    def checkConversion(self, row, mode='ignore'):
400        """Validates all values in row.
401        """
402        converter = IObjectConverter(self.iface)
403        errs, inv_errs, conv_dict =  converter.fromStringDict(
404            row, self.factory_name, mode=mode)
405        return errs, inv_errs, conv_dict
406
407    def getMapping(self, path, headerfields, mode):
408        """Get a mapping from CSV file headerfields to actually used fieldnames.
409        """
410        result = dict()
411        reader = csv.reader(open(path, 'rb'))
412        raw_header = reader.next()
413        for num, field in enumerate(headerfields):
414            if field not in ['student_id', 'reg_number', 'matric_number',
415                             'p_id', 'code', 'level'
416                             ] and mode == 'remove':
417                continue
418            if field == u'--IGNORE--':
419                # Skip ignored columns in failed and finished data files.
420                continue
421            result[raw_header[num]] = field
422        return result
423
424
425class StudentStudyCourseProcessor(StudentProcessorBase):
426    """The Student Study Course Processor imports data which refer
427    to the student's course of study. The study course container data
428    describe the current state of the course of study and it stores the
429    entry conditions, i.e. when the student started the course.
430
431    Most important is the `certificate` attribute which tells us which course
432    the student is studying. The terms 'study course' and 'course of study'
433    are used synonymously. The 'certificate' is the study programme described
434    in the acadmic section. The study course object stores a referrer to a
435    certificate in the acadmic section.
436
437    When importing a new certificate code, `checkConversion` does not only
438    check whether a certificate with the same code exists, it also
439    proves if `current_level` is inside the level range of the certificate.
440    For example, some study programmes start at level 200. The imported
441    current level must thus be 200 or higher.
442
443    `checkUpdateRequirements` looks up if the imported values match the
444    certificate already stored with the study course object. The imported
445    `current_level` must be in the range of the certificate already
446    stored.
447
448    .. note::
449
450      The processor does only offer an update mode. An 'empty' study course
451      object is automatically created when the student object is added. So this
452      object always exists. It can neither be added a second time nor
453      be removed.
454
455    Students can be transferred by import. A transfer is initialized if the
456    `entry_mode` value is ``transfer``. In this case `checkConversion` uses a
457    different interface for data validation and `checkUpdateRequirements`
458    ensures that a student can only be transferred twice. The student transfer
459    process is described elsewhere.
460    """
461    grok.implements(IBatchProcessor)
462    grok.provides(IBatchProcessor)
463    grok.context(Interface)
464    util_name = 'studycourseupdater'
465    grok.name(util_name)
466
467    name = _('StudentStudyCourse Processor (update only)')
468    iface = IStudentStudyCourse
469    iface_transfer = IStudentStudyCourseTransfer
470    factory_name = 'waeup.StudentStudyCourse'
471
472    def getParent(self, row, site):
473        return self._getStudent(row, site)
474
475    def getEntry(self, row, site):
476        student = self.getParent(row, site)
477        if student is None:
478            return None
479        return student.get('studycourse')
480
481    def updateEntry(self, obj, row, site, filename):
482        """Update obj to the values given in row.
483        """
484        entry_mode = row.get('entry_mode', None)
485        certificate = row.get('certificate', None)
486        current_session = row.get('current_session', None)
487        student = self.getParent(row, site)
488        if entry_mode == 'transfer':
489            # We do not expect any error here since we
490            # checked all constraints in checkConversion and
491            # in checkUpdateRequirements
492            student.transfer(
493                certificate=certificate, current_session=current_session)
494            obj = student['studycourse']
495        items_changed = super(StudentStudyCourseProcessor, self).updateEntry(
496            obj, row, site, filename)
497        student.__parent__.logger.info(
498            '%s - %s - %s - updated: %s'
499            % (self.name, filename, student.student_id, items_changed))
500        # Update the students_catalog
501        notify(grok.ObjectModifiedEvent(student))
502        return
503
504    def checkConversion(self, row, mode='ignore'):
505        """Validates all values in row.
506        """
507        # We have to use the correct interface. Transfer
508        # updates have different constraints.
509        entry_mode = row.get('entry_mode', None)
510        if entry_mode == 'transfer':
511            converter = IObjectConverter(self.iface_transfer)
512        else:
513            converter = IObjectConverter(self.iface)
514        errs, inv_errs, conv_dict =  converter.fromStringDict(
515            row, self.factory_name, mode=mode)
516
517        # We have to check if current_level is in range of certificate.
518        if 'certificate' in conv_dict and 'current_level' in conv_dict:
519            cert = conv_dict['certificate']
520            level = conv_dict['current_level']
521            if level < cert.start_level or level > cert.end_level+120:
522                errs.append(('current_level','not in range'))
523        return errs, inv_errs, conv_dict
524
525    def checkUpdateRequirements(self, obj, row, site):
526        """Checks requirements the object must fulfill when being updated.
527        Returns error messages as strings in case of requirement
528        problems.
529        """
530        certificate = getattr(obj, 'certificate', None)
531        entry_session = getattr(obj, 'entry_session', None)
532        current_level = row.get('current_level', None)
533        entry_mode = row.get('entry_mode', None)
534        # We have to ensure that the student can be transferred.
535        if entry_mode == 'transfer':
536            if certificate is None or entry_session is None:
537                return 'Former study course record incomplete.'
538            if 'studycourse_1' in obj.__parent__.keys() and \
539                'studycourse_2' in obj.__parent__.keys():
540                return 'Maximum number of transfers exceeded.'
541        if current_level:
542            if current_level == 999 and \
543                obj.__parent__.state in FORBIDDEN_POSTGRAD_STATES:
544                return 'Not a pg student.'
545            cert = row.get('certificate', None)
546            if certificate is None and cert is None:
547                return 'No certificate to check level.'
548            if certificate is not None and cert is None and (
549                current_level < certificate.start_level or \
550                current_level > certificate.end_level+120):
551                return 'current_level not in range.'
552        return None
553
554class StudentStudyLevelProcessor(StudentProcessorBase):
555    """The Student Study Level Processor imports study level data.
556    It overwrites the container attributes but not the content of the container,
557    i.e. the course tickets stored inside the container. There is nothing
558    special about this processor.
559    """
560    grok.implements(IBatchProcessor)
561    grok.provides(IBatchProcessor)
562    grok.context(Interface)
563    util_name = 'studylevelprocessor'
564    grok.name(util_name)
565
566    name = _('StudentStudyLevel Processor')
567    iface = IStudentStudyLevel
568    factory_name = 'waeup.StudentStudyLevel'
569
570    @property
571    def available_fields(self):
572        fields = super(StudentStudyLevelProcessor, self).available_fields
573        fields.remove('total_credits')
574        fields.remove('gpa')
575        return  fields
576
577    def getParent(self, row, site):
578        student = self._getStudent(row, site)
579        if student is None:
580            return None
581        return student['studycourse']
582
583    def getEntry(self, row, site):
584        studycourse = self.getParent(row, site)
585        if studycourse is None:
586            return None
587        return studycourse.get(str(row['level']))
588
589    def delEntry(self, row, site):
590        studylevel = self.getEntry(row, site)
591        parent = self.getParent(row, site)
592        if studylevel is not None:
593            student = self._getStudent(row, site)
594            student.__parent__.logger.info('%s - Level removed: %s'
595                % (student.student_id, studylevel.__name__))
596            del parent[studylevel.__name__]
597        return
598
599    def updateEntry(self, obj, row, site, filename):
600        """Update obj to the values given in row.
601        """
602        items_changed = super(StudentStudyLevelProcessor, self).updateEntry(
603            obj, row, site, filename)
604        student = self.getParent(row, site).__parent__
605        student.__parent__.logger.info(
606            '%s - %s - %s - updated: %s'
607            % (self.name, filename, student.student_id, items_changed))
608        return
609
610    def addEntry(self, obj, row, site):
611        parent = self.getParent(row, site)
612        parent[str(row['level'])] = obj
613        return
614
615class CourseTicketProcessor(StudentProcessorBase):
616    """The Course Ticket Processor imports course tickets, the subobjects
617    of student study levels (= course lists).
618
619    An imported course ticket contains a copy of the original course data.
620    During import only a few attributes can be set/overwritten.
621
622    Like all other student data importers, this processor also requires
623    either `student_id`, `reg_number` or `matric_number` to find the student.
624    Then  it needs `level` and `code` to localize the course ticket.
625
626    `checkConversion` first searches the courses catalog for the imported
627    `code` and ensures that a course with such a code really exists
628    in the academic section. It furthermore checks if `level_session` in
629    the row corresponds with the session of the parent student
630    study level object. It fails if one of the conditions is not met.
631
632    In create mode `fcode`, `dcode`, `title`, `credits`, `passmark` and
633    `semester` are taken from the course found in the academic section.
634    `fcode` and `dcode` can nevermore be changed, neither via the user interface
635    nor by import. 'title', 'credits', 'passmark' and 'semester' can be
636    overwritten by import.
637    """
638    grok.implements(IBatchProcessor)
639    grok.provides(IBatchProcessor)
640    grok.context(Interface)
641    util_name = 'courseticketprocessor'
642    grok.name(util_name)
643
644    name = _('CourseTicket Processor')
645    iface = ICourseTicketImport
646    factory_name = 'waeup.CourseTicket'
647
648    additional_fields = ['level', 'code']
649    additional_fields_required = additional_fields
650
651    @property
652    def available_fields(self):
653        fields = [
654            'student_id','reg_number','matric_number',
655            'mandatory', 'score', 'carry_over', 'automatic',
656            'level_session',
657            'title', 'credits', 'passmark', 'semester'
658            ] + self.additional_fields
659        return sorted(fields)
660
661    def getParent(self, row, site):
662        student = self._getStudent(row, site)
663        if student is None:
664            return None
665        return student['studycourse'].get(str(row['level']))
666
667    def getEntry(self, row, site):
668        level = self.getParent(row, site)
669        if level is None:
670            return None
671        return level.get(row['code'])
672
673    def updateEntry(self, obj, row, site, filename):
674        """Update obj to the values given in row.
675        """
676        items_changed = super(CourseTicketProcessor, self).updateEntry(
677            obj, row, site, filename)
678        parent = self.getParent(row, site)
679        student = self.getParent(row, site).__parent__.__parent__
680        student.__parent__.logger.info(
681            '%s - %s - %s - %s - updated: %s'
682            % (self.name, filename, student.student_id, parent.level, items_changed))
683        return
684
685    def addEntry(self, obj, row, site):
686        parent = self.getParent(row, site)
687        catalog = getUtility(ICatalog, name='courses_catalog')
688        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
689        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
690        obj.dcode = entries[0].__parent__.__parent__.code
691        if getattr(obj, 'title', None) is None:
692            obj.title = entries[0].title
693        if getattr(obj, 'credits', None) is None:
694            obj.credits = entries[0].credits
695        if getattr(obj, 'passmark', None) is None:
696            obj.passmark = entries[0].passmark
697        if getattr(obj, 'semester', None) is None:
698            obj.semester = entries[0].semester
699        parent[row['code']] = obj
700        return
701
702    def delEntry(self, row, site):
703        ticket = self.getEntry(row, site)
704        parent = self.getParent(row, site)
705        if ticket is not None:
706            student = self._getStudent(row, site)
707            student.__parent__.logger.info('%s - Course ticket in %s removed: %s'
708                % (student.student_id, parent.level, ticket.code))
709            del parent[ticket.code]
710        return
711
712    def checkConversion(self, row, mode='ignore'):
713        """Validates all values in row.
714        """
715        errs, inv_errs, conv_dict = super(
716            CourseTicketProcessor, self).checkConversion(row, mode=mode)
717        if mode == 'remove':
718            return errs, inv_errs, conv_dict
719        # In update and create mode we have to check if course really exists.
720        # This is not done by the converter.
721        catalog = getUtility(ICatalog, name='courses_catalog')
722        entries = catalog.searchResults(code=(row['code'],row['code']))
723        if len(entries) == 0:
724            errs.append(('code','non-existent'))
725            return errs, inv_errs, conv_dict
726        # If level_session is provided in row we have to check if
727        # the parent studylevel exists and if its level_session
728        # attribute corresponds with the expected value in row.
729        level_session = conv_dict.get('level_session', IGNORE_MARKER)
730        if level_session not in (IGNORE_MARKER, None):
731            site = grok.getSite()
732            studylevel = self.getParent(row, site)
733            if studylevel is not None:
734                if studylevel.level_session != level_session:
735                    errs.append(('level_session','does not match %s'
736                        % studylevel.level_session))
737            else:
738                errs.append(('level object','does not exist'))
739        return errs, inv_errs, conv_dict
740
741class StudentOnlinePaymentProcessor(StudentProcessorBase):
742    """The Student Online Payment Processor imports student payment tickets.
743    The tickets are located in the ``payments`` subfolder of the student
744    container. The only additional locator is `p_id`, the object id.
745
746    The `checkConversion` method checks the format of the payment identifier.
747    In create mode it does also ensures that same p_id does not exist
748    elsewhere. It must be portal-wide unique.
749
750    When adding a payment ticket, the `addEntry` method checks if the same
751    payment has already been made. It compares `p_category` and `p_session`
752    in the row with the corresponding attributes  of existing payment
753    tickets in state ``paid``. If they match, a `DuplicationError` is raised.
754    """
755    grok.implements(IBatchProcessor)
756    grok.provides(IBatchProcessor)
757    grok.context(Interface)
758    util_name = 'paymentprocessor'
759    grok.name(util_name)
760
761    name = _('StudentOnlinePayment Processor')
762    iface = IStudentOnlinePayment
763    factory_name = 'waeup.StudentOnlinePayment'
764
765    additional_fields = ['p_id']
766
767    @property
768    def available_fields(self):
769        af = super(
770            StudentOnlinePaymentProcessor, self).available_fields
771        af.remove('display_item')
772        return af
773
774    def checkHeaders(self, headerfields, mode='ignore'):
775        super(StudentOnlinePaymentProcessor, self).checkHeaders(headerfields)
776        if mode in ('update', 'remove') and not 'p_id' in headerfields:
777            raise FatalCSVError(
778                "Need p_id for import in update and remove modes!")
779        return True
780
781    def parentsExist(self, row, site):
782        return self.getParent(row, site) is not None
783
784    def getParent(self, row, site):
785        student = self._getStudent(row, site)
786        if student is None:
787            return None
788        return student['payments']
789
790    def getEntry(self, row, site):
791        payments = self.getParent(row, site)
792        if payments is None:
793            return None
794        p_id = row.get('p_id', None)
795        if p_id in (None, IGNORE_MARKER):
796            return None
797        # We can use the hash symbol at the end of p_id in import files
798        # to avoid annoying automatic number transformation
799        # by Excel or Calc
800        p_id = p_id.strip('#')
801        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
802            # For data migration from old SRP only
803            p_id = 'p' + p_id[7:] + '0'
804        entry = payments.get(p_id)
805        return entry
806
807    def updateEntry(self, obj, row, site, filename):
808        """Update obj to the values given in row.
809        """
810        items_changed = super(StudentOnlinePaymentProcessor, self).updateEntry(
811            obj, row, site, filename)
812        student = self.getParent(row, site).__parent__
813        student.__parent__.logger.info(
814            '%s - %s - %s - updated: %s'
815            % (self.name, filename, student.student_id, items_changed))
816        return
817
818    def samePaymentMade(self, student, category, p_session):
819        for key in student['payments'].keys():
820            ticket = student['payments'][key]
821            if ticket.p_state == 'paid' and\
822               ticket.p_category == category and \
823               ticket.p_session == p_session and \
824               ticket.p_item != 'Balance':
825                  return True
826        return False
827
828    def addEntry(self, obj, row, site):
829        parent = self.getParent(row, site)
830        student = parent.student
831        p_id = row['p_id'].strip('#')
832        # Requirement added on 19/02/2015: same payment must not exist.
833        if self.samePaymentMade(student, obj.p_category, obj.p_session):
834            student.__parent__.logger.info(
835                '%s - %s - previous update cancelled'
836                % (self.name, student.student_id))
837            raise DuplicationError('Same payment has already been made.')
838        if len(p_id.split('-')) != 3 and not p_id.startswith('p'):
839            # For data migration from old SRP
840            obj.p_id = 'p' + p_id[7:] + '0'
841            parent[obj.p_id] = obj
842        else:
843            parent[p_id] = obj
844        return
845
846    def delEntry(self, row, site):
847        payment = self.getEntry(row, site)
848        parent = self.getParent(row, site)
849        if payment is not None:
850            student = self._getStudent(row, site)
851            student.__parent__.logger.info('%s - Payment ticket removed: %s'
852                % (student.student_id, payment.p_id))
853            del parent[payment.p_id]
854        return
855
856    def checkConversion(self, row, mode='ignore'):
857        """Validates all values in row.
858        """
859        errs, inv_errs, conv_dict = super(
860            StudentOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
861
862        # We have to check p_id.
863        p_id = row.get('p_id', None)
864        if mode == 'create' and p_id in (None, IGNORE_MARKER):
865            timestamp = ("%d" % int(time()*10000))[1:]
866            p_id = "p%s" % timestamp
867            conv_dict['p_id'] = p_id
868            return errs, inv_errs, conv_dict
869        elif p_id in (None, IGNORE_MARKER):
870            errs.append(('p_id','missing'))
871            return errs, inv_errs, conv_dict
872        else:
873            p_id = p_id.strip('#')
874            if p_id.startswith('p'):
875                if not len(p_id) == 14:
876                    errs.append(('p_id','invalid length'))
877                    return errs, inv_errs, conv_dict
878            elif len(p_id.split('-')) == 3:
879                # The SRP used either pins as keys ...
880                if len(p_id.split('-')[2]) not in (9, 10):
881                    errs.append(('p_id','invalid pin'))
882                    return errs, inv_errs, conv_dict
883            else:
884                # ... or order_ids.
885                if not len(p_id) == 19:
886                    errs.append(('p_id','invalid format'))
887                    return errs, inv_errs, conv_dict
888        # Requirement added on 24/01/2015: p_id must be portal-wide unique.
889        if mode == 'create':
890            cat = getUtility(ICatalog, name='payments_catalog')
891            results = list(cat.searchResults(p_id=(p_id, p_id)))
892            if len(results) > 0:
893                sids = [IPayer(payment).id for payment in results]
894                sids_string = ''
895                for id in sids:
896                    sids_string += '%s ' % id
897                errs.append(('p_id','p_id exists in %s' % sids_string))
898                return errs, inv_errs, conv_dict
899        return errs, inv_errs, conv_dict
900
901class StudentVerdictProcessor(StudentStudyCourseProcessor):
902    """The Student Verdict Processor inherits from the Student Study
903    Course Processor. It's a pure updater. Import step 2 raises a warning
904    message if a datacenter manager tries to select another mode.
905    But it does more than only overwriting study course attributes.
906
907    The Student Verdict Processor is the only processor which cannot be
908    used for restoring data. Purpose is to announce a verdict at the end of
909    each academic session. The processor does not only import a verdict,
910    it also conditions the student data so that the student can pay for the
911    next session and proceed to the next study level.
912
913    The `checkUpdateRequirements` method ensures that the imported data
914    really correspond to the actual state of the student.
915    `current_level` and `current_session` in the row must be on par
916    with the attributes of the study course object. Thus, the processor
917    does not use these values to overwrite the attributes of the study course
918    but to control that the verdict is really meant for the current session of
919    the student. The verdict is only imported if a corresponding study level
920    object exists and the student is in the right registration state,
921    either ``courses validated`` or ``courses registered``. Course registration
922    can be bypassed by setting `bypass_validation` to ``True``.
923
924    The `updateEntry` method does not only update the current verdict of
925    the student study course, it also updates the matching student study
926    level object. It saves the current verdict as `level_verdict` and sets
927    the `validated_by` and `validation_date` attributes, whereas `validated_by`
928    is taken from the row of the import file and `validation_date` is set to the
929    actual UTC datetime. Finally, the student is moved to state ``returning``.
930    """
931
932    util_name = 'verdictupdater'
933    grok.name(util_name)
934
935    name = _('Verdict Processor (special processor, update only)')
936    iface = IStudentVerdictUpdate
937    factory_name = 'waeup.StudentStudyCourse'
938
939    def checkUpdateRequirements(self, obj, row, site):
940        """Checks requirements the studycourse and the student must fulfill
941        before being updated.
942        """
943        # Check if current_levels correspond
944        if obj.current_level != row['current_level']:
945            return 'Current level does not correspond.'
946        # Check if current_sessions correspond
947        if obj.current_session != row['current_session']:
948            return 'Current session does not correspond.'
949        # Check if new verdict is provided
950        if row['current_verdict'] in (IGNORE_MARKER, ''):
951            return 'No verdict in import file.'
952        # Check if studylevel exists
953        level_string = str(obj.current_level)
954        if obj.get(level_string) is None:
955            return 'Study level object is missing.'
956        # Check if student is in state REGISTERED or VALIDATED
957        if row.get('bypass_validation'):
958            if obj.student.state not in (VALIDATED, REGISTERED):
959                return 'Student in wrong state.'
960        else:
961            if obj.student.state != VALIDATED:
962                return 'Student in wrong state.'
963        return None
964
965    def updateEntry(self, obj, row, site, filename):
966        """Update obj to the values given in row.
967        """
968        # Don't set current_session, current_level
969        vals_to_set = dict((key, val) for key, val in row.items()
970                           if key not in ('current_session','current_level'))
971        super(StudentVerdictProcessor, self).updateEntry(
972            obj, vals_to_set, site, filename)
973        parent = self.getParent(row, site)
974        # Set current_verdict in corresponding studylevel
975        level_string = str(obj.current_level)
976        obj[level_string].level_verdict = row['current_verdict']
977        # Fire transition and set studylevel attributes
978        # depending on student's state
979        if obj.__parent__.state == REGISTERED:
980            validated_by = row.get('validated_by', '')
981            if validated_by in (IGNORE_MARKER, ''):
982                portal_language = getUtility(IKofaUtils).PORTAL_LANGUAGE
983                system = translate(_('System'),'waeup.kofa',
984                                  target_language=portal_language)
985                obj[level_string].validated_by = system
986            else:
987                obj[level_string].validated_by = validated_by
988            obj[level_string].validation_date = datetime.utcnow()
989            IWorkflowInfo(obj.__parent__).fireTransition('bypass_validation')
990        else:
991            IWorkflowInfo(obj.__parent__).fireTransition('return')
992        # Update the students_catalog
993        notify(grok.ObjectModifiedEvent(obj.__parent__))
994        return
Note: See TracBrowser for help on using the repository browser.