source: main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py @ 7549

Last change on this file since 7549 was 7548, checked in by Henrik Bettermann, 13 years ago

Implement course ticket importer.

Add checkConversion tests.
  • Property svn:keywords set to Id
File size: 19.3 KB
Line 
1## $Id: batching.py 7548 2012-02-01 11:19:56Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility, getUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState
34from waeup.sirp.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory)
37from waeup.sirp.students.interfaces import (
38    IStudent, IStudentStudyCourse,
39    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
40    IStudentStudyLevel, ICourseTicket)
41from waeup.sirp.students.workflow import  IMPORTABLE_STATES
42from waeup.sirp.utils.batching import BatchProcessor
43from waeup.sirp.utils.helpers import get_current_principal
44
45class StudentProcessor(BatchProcessor):
46    """A batch processor for IStudent objects.
47    """
48    grok.implements(IBatchProcessor)
49    grok.provides(IBatchProcessor)
50    grok.context(Interface)
51    util_name = 'studentimporter'
52    grok.name(util_name)
53
54    name = u'Student Importer'
55    iface = IStudent
56
57    location_fields = []
58    factory_name = 'waeup.Student'
59
60    mode = None
61
62    @property
63    def available_fields(self):
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'reg_state'] + getFields(
67                self.iface).keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if not 'reg_number' in headerfields and not 'student_id' \
71            in headerfields and not 'matric_number' in headerfields:
72            raise FatalCSVError(
73                "Need at least columns student_id or reg_number " +
74                "or matric_number for import!")
75        if mode == 'create':
76            for field in self.required_fields:
77                if not field in headerfields:
78                    raise FatalCSVError(
79                        "Need at least columns %s for import!" %
80                        ', '.join(["'%s'" % x for x in self.required_fields]))
81        # Check for fields to be ignored...
82        not_ignored_fields = [x for x in headerfields
83                              if not x.startswith('--')]
84        if len(set(not_ignored_fields)) < len(not_ignored_fields):
85            raise FatalCSVError(
86                "Double headers: each column name may only appear once.")
87        return True
88
89    def parentsExist(self, row, site):
90        return 'students' in site.keys()
91
92    def getLocator(self, row):
93        if row.get('student_id',None):
94            return 'student_id'
95        elif row.get('reg_number',None):
96            return 'reg_number'
97        elif row.get('matric_number',None):
98            return 'matric_number'
99        else:
100            return None
101
102    # The entry never exists in create mode.
103    def entryExists(self, row, site):
104        return self.getEntry(row, site) is not None
105
106    def getParent(self, row, site):
107        return site['students']
108
109    def getEntry(self, row, site):
110        if not 'students' in site.keys():
111            return None
112        if self.getLocator(row) == 'student_id':
113            if row['student_id'] in site['students']:
114                student = site['students'][row['student_id']]
115                return student
116        elif self.getLocator(row) == 'reg_number':
117            reg_number = row['reg_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(reg_number=(reg_number, reg_number)))
121            if results:
122                return results[0]
123        elif self.getLocator(row) == 'matric_number':
124            matric_number = row['matric_number']
125            cat = queryUtility(ICatalog, name='students_catalog')
126            results = list(
127                cat.searchResults(matric_number=(matric_number, matric_number)))
128            if results:
129                return results[0]
130        return None
131
132       
133    def addEntry(self, obj, row, site):
134        parent = self.getParent(row, site)
135        parent.addStudent(obj)
136        # In some tests we don't have a students container or a user
137        try:
138            user = get_current_principal()
139            parent.logger.info('%s - %s - Student record imported' % (
140                user.id,obj.student_id))
141            history = IObjectHistory(obj)
142            history.addMessage('Student record imported')
143        except (TypeError, AttributeError):
144            pass
145        return
146
147    def delEntry(self, row, site):
148        student = self.getEntry(row, site)
149        if student is not None:
150            parent = self.getParent(row, site)
151            del parent[student.student_id]
152        pass
153
154    def updateEntry(self, obj, row, site):
155        """Update obj to the values given in row.
156        """
157        for key, value in row.items():
158            # Set student password and all fields declared in interface.
159            if key == 'password' and value != '':
160                IUserAccount(obj).setPassword(value)
161            elif key == 'reg_state':
162                IWorkflowState(obj).setState(value)
163                msg = "State '%s' set" % value
164                history = IObjectHistory(obj)
165                history.addMessage(msg)
166            elif hasattr(obj, key):
167                setattr(obj, key, value)
168        return
169
170    def getMapping(self, path, headerfields, mode):
171        """Get a mapping from CSV file headerfields to actually used fieldnames.
172        """
173        result = dict()
174        reader = csv.reader(open(path, 'rb'))
175        raw_header = reader.next()
176        for num, field in enumerate(headerfields):
177            if field not in [
178                'student_id', 'reg_number', 'matric_number'] and mode == 'remove':
179                continue
180            if field == u'--IGNORE--':
181                # Skip ignored columns in failed and finished data files.
182                continue
183            result[raw_header[num]] = field
184        return result
185
186    def checkConversion(self, row, mode='create'):
187        """Validates all values in row.
188        """
189        if mode in ['update', 'remove']:
190            if self.getLocator(row) == 'reg_number':
191                iface = IStudentUpdateByRegNo
192            elif self.getLocator(row) == 'matric_number':
193                iface = IStudentUpdateByMatricNo
194        else:
195            iface = self.iface
196        converter = IObjectConverter(iface)
197        errs, inv_errs, conv_dict =  converter.fromStringDict(
198            row, self.factory_name)
199        if row.has_key('reg_state') and \
200            not row['reg_state'] in IMPORTABLE_STATES:
201            if row['reg_state'] != '':
202                errs.append(('reg_state','not allowed'))
203            else:
204                errs.append(('reg_state','no value provided'))
205        return errs, inv_errs, conv_dict
206
207class StudentStudyCourseProcessor(BatchProcessor):
208    """A batch processor for IStudentStudyCourse objects.
209    """
210    grok.implements(IBatchProcessor)
211    grok.provides(IBatchProcessor)
212    grok.context(Interface)
213    util_name = 'studycourseupdater'
214    grok.name(util_name)
215
216    name = u'StudentStudyCourse Importer (update only)'
217    iface = IStudentStudyCourse
218    factory_name = 'waeup.StudentStudyCourse'
219
220    location_fields = []
221
222    mode = None
223
224    @property
225    def available_fields(self):
226        return sorted(list(set(
227            ['student_id','reg_number','matric_number'] + getFields(
228                self.iface).keys())))
229
230    def checkHeaders(self, headerfields, mode='ignore'):
231        if not 'reg_number' in headerfields and not 'student_id' \
232            in headerfields and not 'matric_number' in headerfields:
233            raise FatalCSVError(
234                "Need at least columns student_id " +
235                "or reg_number or matric_number for import!")
236        # Check for fields to be ignored...
237        not_ignored_fields = [x for x in headerfields
238                              if not x.startswith('--')]
239        if len(set(not_ignored_fields)) < len(not_ignored_fields):
240            raise FatalCSVError(
241                "Double headers: each column name may only appear once.")
242        return True
243
244    def getParent(self, row, site):
245        if not 'students' in site.keys():
246            return None
247        if 'student_id' in row.keys() and row['student_id']:
248            if row['student_id'] in site['students']:
249                student = site['students'][row['student_id']]
250                return student
251        elif 'reg_number' in row.keys() and row['reg_number']:
252            reg_number = row['reg_number']
253            cat = queryUtility(ICatalog, name='students_catalog')
254            results = list(
255                cat.searchResults(reg_number=(reg_number, reg_number)))
256            if results:
257                return results[0]
258        elif 'matric_number' in row.keys() and row['matric_number']:
259            matric_number = row['matric_number']
260            cat = queryUtility(ICatalog, name='students_catalog')
261            results = list(
262                cat.searchResults(matric_number=(matric_number, matric_number)))
263            if results:
264                return results[0]
265        return None
266
267    def parentsExist(self, row, site):
268        return self.getParent(row, site) is not None
269
270    def entryExists(self, row, site):
271        return self.getEntry(row, site) is not None
272
273    def getEntry(self, row, site):
274        student = self.getParent(row, site)
275        if student is None:
276            return None
277        return student.get('studycourse')
278
279    def updateEntry(self, obj, row, site):
280        """Update obj to the values given in row.
281        """
282        for key, value in row.items():
283            # Skip fields not declared in interface.
284            if hasattr(obj, key):
285                setattr(obj, key, value)
286        # Update the students_catalog
287        notify(grok.ObjectModifiedEvent(obj.__parent__))
288        return
289
290    def checkConversion(self, row, mode='ignore'):
291        """Validates all values in row.
292        """
293        converter = IObjectConverter(self.iface)
294        errs, inv_errs, conv_dict =  converter.fromStringDict(
295            row, self.factory_name)
296        # We have to check if current_level is in range of certificate.
297        # This is not done by the converter. This kind of conversion
298        # checking does only work if a combination of certificate and
299        # current_level is provided.
300        if conv_dict.has_key('certificate'):
301          certificate = conv_dict['certificate']
302          start_level = certificate.start_level
303          end_level = certificate.end_level
304          if conv_dict['current_level'] < start_level or \
305              conv_dict['current_level'] > end_level:
306              errs.append(('current_level','not in range'))
307        return errs, inv_errs, conv_dict
308
309class StudentStudyLevelProcessor(BatchProcessor):
310    """A batch processor for IStudentStudyLevel objects.
311    """
312    grok.implements(IBatchProcessor)
313    grok.provides(IBatchProcessor)
314    grok.context(Interface)
315    util_name = 'studylevelimporter'
316    grok.name(util_name)
317
318    name = u'StudentStudyLevel Importer'
319    iface = IStudentStudyLevel
320    factory_name = 'waeup.StudentStudyLevel'
321
322    location_fields = []
323
324    mode = None
325
326    @property
327    def available_fields(self):
328        return sorted(list(set(
329            ['student_id','reg_number','matric_number','level'] + getFields(
330                self.iface).keys())))
331
332    def checkHeaders(self, headerfields, mode='ignore'):
333        if not 'reg_number' in headerfields and not 'student_id' \
334            in headerfields and not 'matric_number' in headerfields:
335            raise FatalCSVError(
336                "Need at least columns student_id " +
337                "or reg_number or matric_number for import!")
338        if not 'level' in headerfields:
339            raise FatalCSVError(
340                "Need level for import!")
341        # Check for fields to be ignored...
342        not_ignored_fields = [x for x in headerfields
343                              if not x.startswith('--')]
344        if len(set(not_ignored_fields)) < len(not_ignored_fields):
345            raise FatalCSVError(
346                "Double headers: each column name may only appear once.")
347        return True
348
349    def getParent(self, row, site):
350        if not 'students' in site.keys():
351            return None
352        if 'student_id' in row.keys() and row['student_id']:
353            if row['student_id'] in site['students']:
354                student = site['students'][row['student_id']]
355                return student['studycourse']
356        elif 'reg_number' in row.keys() and row['reg_number']:
357            reg_number = row['reg_number']
358            cat = queryUtility(ICatalog, name='students_catalog')
359            results = list(
360                cat.searchResults(reg_number=(reg_number, reg_number)))
361            if results:
362                return results[0]['studycourse']
363        elif 'matric_number' in row.keys() and row['matric_number']:
364            matric_number = row['matric_number']
365            cat = queryUtility(ICatalog, name='students_catalog')
366            results = list(
367                cat.searchResults(matric_number=(matric_number, matric_number)))
368            if results:
369                return results[0]['studycourse']
370        return None
371
372    def parentsExist(self, row, site):
373        return self.getParent(row, site) is not None
374
375    def entryExists(self, row, site):
376        return self.getEntry(row, site) is not None
377
378    def getEntry(self, row, site):
379        studycourse = self.getParent(row, site)
380        if studycourse is None:
381            return None
382        return studycourse.get(row['level'])
383
384    def addEntry(self, obj, row, site):
385        parent = self.getParent(row, site)
386        obj.level = int(row['level'])
387        parent[row['level']] = obj
388        return
389
390    def checkConversion(self, row, mode='ignore'):
391        """Validates all values in row.
392        """
393        converter = IObjectConverter(self.iface)
394        errs, inv_errs, conv_dict =  converter.fromStringDict(
395            row, self.factory_name)
396        # We have to check if level is a valid integer.
397        # This is not done by the converter.
398        try:
399            level = int(row['level'])
400            if level not in range(0,600,10):
401                errs.append(('level','no valid integer'))
402        except ValueError:
403            errs.append(('level','no integer'))
404        return errs, inv_errs, conv_dict
405
406class CourseTicketProcessor(BatchProcessor):
407    """A batch processor for ICourseTicket objects.
408    """
409    grok.implements(IBatchProcessor)
410    grok.provides(IBatchProcessor)
411    grok.context(Interface)
412    util_name = 'courseticketimporter'
413    grok.name(util_name)
414
415    name = u'CourseTicket Importer'
416    iface = ICourseTicket
417    factory_name = 'waeup.CourseTicket'
418
419    location_fields = []
420
421    mode = None
422
423    @property
424    def available_fields(self):
425        return sorted(list(set(
426            ['student_id','reg_number','matric_number','level','code'] + getFields(
427                self.iface).keys())))
428
429    def checkHeaders(self, headerfields, mode='ignore'):
430        if not 'reg_number' in headerfields and not 'student_id' \
431            in headerfields and not 'matric_number' in headerfields:
432            raise FatalCSVError(
433                "Need at least columns student_id " +
434                "or reg_number or matric_number for import!")
435        if not 'level' in headerfields:
436            raise FatalCSVError(
437                "Need level for import!")
438        if not 'code' in headerfields:
439            raise FatalCSVError(
440                "Need code for import!")
441        # Check for fields to be ignored...
442        not_ignored_fields = [x for x in headerfields
443                              if not x.startswith('--')]
444        if len(set(not_ignored_fields)) < len(not_ignored_fields):
445            raise FatalCSVError(
446                "Double headers: each column name may only appear once.")
447        return True
448
449    def getParent(self, row, site):
450        if not 'students' in site.keys():
451            return None
452        if 'student_id' in row.keys() and row['student_id']:
453            if row['student_id'] in site['students']:
454                student = site['students'][row['student_id']]
455                return student['studycourse'].get(row['level'])
456        elif 'reg_number' in row.keys() and row['reg_number']:
457            reg_number = row['reg_number']
458            #import pdb; pdb.set_trace()
459            cat = queryUtility(ICatalog, name='students_catalog')
460            results = list(
461                cat.searchResults(reg_number=(reg_number, reg_number)))
462            if results:
463                return results[0]['studycourse'].get(row['level'])
464        elif 'matric_number' in row.keys() and row['matric_number']:
465            matric_number = row['matric_number']
466            cat = queryUtility(ICatalog, name='students_catalog')
467            results = list(
468                cat.searchResults(matric_number=(matric_number, matric_number)))
469            if results:
470                return results[0]['studycourse'].get(row['level'])
471        return None
472
473    def parentsExist(self, row, site):
474        return self.getParent(row, site) is not None
475
476    def entryExists(self, row, site):
477        return self.getEntry(row, site) is not None
478
479    def getEntry(self, row, site):
480        level = self.getParent(row, site)
481        if level is None:
482            return None
483        return level.get(row['code'])
484
485    def addEntry(self, obj, row, site):
486        parent = self.getParent(row, site)
487        catalog = getUtility(ICatalog, name='courses_catalog')
488        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
489        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
490        obj.dcode = entries[0].__parent__.__parent__.code
491        obj.title = entries[0].title
492        obj.credits = entries[0].credits
493        obj.passmark = entries[0].passmark
494        obj.semester = entries[0].semester
495        parent[row['code']] = obj
496        return
497
498    def checkConversion(self, row, mode='ignore'):
499        """Validates all values in row.
500        """
501        converter = IObjectConverter(self.iface)
502        errs, inv_errs, conv_dict =  converter.fromStringDict(
503            row, self.factory_name)
504        # We have to check if course really exists.
505        # This is not done by the converter.
506        catalog = getUtility(ICatalog, name='courses_catalog')
507        entries = catalog.searchResults(code=(row['code'],row['code']))
508        if len(entries) == 0:
509            errs.append(('code','non-existent'))
510            return errs, inv_errs, conv_dict
511        return errs, inv_errs, conv_dict
Note: See TracBrowser for help on using the repository browser.