source: main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py @ 7542

Last change on this file since 7542 was 7536, checked in by Henrik Bettermann, 13 years ago

Implement study level importer.
  • Property svn:keywords set to Id
File size: 15.1 KB
Line 
1## $Id: batching.py 7536 2012-01-30 07:41:17Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState
34from waeup.sirp.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory)
37from waeup.sirp.students.interfaces import (
38    IStudent, IStudentStudyCourse,
39    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
40    IStudentStudyLevel)
41from waeup.sirp.students.workflow import  IMPORTABLE_STATES
42from waeup.sirp.utils.batching import BatchProcessor
43from waeup.sirp.utils.helpers import get_current_principal
44
45class StudentProcessor(BatchProcessor):
46    """A batch processor for IStudent objects.
47    """
48    grok.implements(IBatchProcessor)
49    grok.provides(IBatchProcessor)
50    grok.context(Interface)
51    util_name = 'studentimporter'
52    grok.name(util_name)
53
54    name = u'Student Importer'
55    iface = IStudent
56
57    location_fields = []
58    factory_name = 'waeup.Student'
59
60    mode = None
61
62    @property
63    def available_fields(self):
64        return sorted(list(set(
65            ['student_id','reg_number','matric_number',
66            'password', 'reg_state'] + getFields(
67                self.iface).keys())))
68
69    def checkHeaders(self, headerfields, mode='create'):
70        if not 'reg_number' in headerfields and not 'student_id' \
71            in headerfields and not 'matric_number' in headerfields:
72            raise FatalCSVError(
73                "Need at least columns student_id or reg_number " +
74                "or matric_number for import!")
75        if mode == 'create':
76            for field in self.required_fields:
77                if not field in headerfields:
78                    raise FatalCSVError(
79                        "Need at least columns %s for import!" %
80                        ', '.join(["'%s'" % x for x in self.required_fields]))
81        # Check for fields to be ignored...
82        not_ignored_fields = [x for x in headerfields
83                              if not x.startswith('--')]
84        if len(set(not_ignored_fields)) < len(not_ignored_fields):
85            raise FatalCSVError(
86                "Double headers: each column name may only appear once.")
87        return True
88
89    def parentsExist(self, row, site):
90        return 'students' in site.keys()
91
92    def getLocator(self, row):
93        if row.get('student_id',None):
94            return 'student_id'
95        elif row.get('reg_number',None):
96            return 'reg_number'
97        elif row.get('matric_number',None):
98            return 'matric_number'
99        else:
100            return None
101
102    # The entry never exists in create mode.
103    def entryExists(self, row, site):
104        return self.getEntry(row, site) is not None
105
106    def getParent(self, row, site):
107        return site['students']
108
109    def getEntry(self, row, site):
110        if not 'students' in site.keys():
111            return None
112        if self.getLocator(row) == 'student_id':
113            if row['student_id'] in site['students']:
114                student = site['students'][row['student_id']]
115                return student
116        elif self.getLocator(row) == 'reg_number':
117            reg_number = row['reg_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(reg_number=(reg_number, reg_number)))
121            if results:
122                return results[0]
123        elif self.getLocator(row) == 'matric_number':
124            matric_number = row['matric_number']
125            cat = queryUtility(ICatalog, name='students_catalog')
126            results = list(
127                cat.searchResults(matric_number=(matric_number, matric_number)))
128            if results:
129                return results[0]
130        return None
131
132       
133    def addEntry(self, obj, row, site):
134        parent = self.getParent(row, site)
135        parent.addStudent(obj)
136        # In some tests we don't have a students container or a user
137        try:
138            user = get_current_principal()
139            parent.logger.info('%s - %s - Student record imported' % (
140                user.id,obj.student_id))
141            history = IObjectHistory(obj)
142            history.addMessage('Student record imported')
143        except (TypeError, AttributeError):
144            pass
145        return
146
147    def delEntry(self, row, site):
148        student = self.getEntry(row, site)
149        if student is not None:
150            parent = self.getParent(row, site)
151            del parent[student.student_id]
152        pass
153
154    def updateEntry(self, obj, row, site):
155        """Update obj to the values given in row.
156        """
157        for key, value in row.items():
158            # Set student password and all fields declared in interface.
159            if key == 'password' and value != '':
160                IUserAccount(obj).setPassword(value)
161            elif key == 'reg_state':
162                IWorkflowState(obj).setState(value)
163                msg = "State '%s' set" % value
164                history = IObjectHistory(obj)
165                history.addMessage(msg)
166            elif hasattr(obj, key):
167                setattr(obj, key, value)
168        return
169
170    def getMapping(self, path, headerfields, mode):
171        """Get a mapping from CSV file headerfields to actually used fieldnames.
172        """
173        result = dict()
174        reader = csv.reader(open(path, 'rb'))
175        raw_header = reader.next()
176        for num, field in enumerate(headerfields):
177            if field not in [
178                'student_id', 'reg_number', 'matric_number'] and mode == 'remove':
179                continue
180            if field == u'--IGNORE--':
181                # Skip ignored columns in failed and finished data files.
182                continue
183            result[raw_header[num]] = field
184        return result
185
186    def checkConversion(self, row, mode='create'):
187        """Validates all values in row.
188        """
189        if mode in ['update', 'remove']:
190            if self.getLocator(row) == 'reg_number':
191                iface = IStudentUpdateByRegNo
192            elif self.getLocator(row) == 'matric_number':
193                iface = IStudentUpdateByMatricNo
194        else:
195            iface = self.iface
196        converter = IObjectConverter(iface)
197        errs, inv_errs, conv_dict =  converter.fromStringDict(
198            row, self.factory_name)
199        if row.has_key('reg_state') and \
200            not row['reg_state'] in IMPORTABLE_STATES:
201            if row['reg_state'] != '':
202                errs.append(('reg_state','not allowed'))
203            else:
204                errs.append(('reg_state','no value provided'))
205        return errs, inv_errs, conv_dict
206
207class StudentStudyCourseProcessor(BatchProcessor):
208    """A batch processor for IStudentStudyCourse objects.
209    """
210    grok.implements(IBatchProcessor)
211    grok.provides(IBatchProcessor)
212    grok.context(Interface)
213    util_name = 'studycourseupdater'
214    grok.name(util_name)
215
216    name = u'StudentStudyCourse Importer (update only)'
217    iface = IStudentStudyCourse
218    factory_name = 'waeup.StudentStudyCourse'
219
220    location_fields = []
221
222    mode = None
223
224    @property
225    def available_fields(self):
226        return sorted(list(set(
227            ['student_id','reg_number','matric_number'] + getFields(
228                self.iface).keys())))
229
230    def checkHeaders(self, headerfields, mode='ignore'):
231        if not 'reg_number' in headerfields and not 'student_id' \
232            in headerfields and not 'matric_number' in headerfields:
233            raise FatalCSVError(
234                "Need at least columns student_id " +
235                "or reg_number or matric_number for import!")
236        # Check for fields to be ignored...
237        not_ignored_fields = [x for x in headerfields
238                              if not x.startswith('--')]
239        if len(set(not_ignored_fields)) < len(not_ignored_fields):
240            raise FatalCSVError(
241                "Double headers: each column name may only appear once.")
242        return True
243
244    def getParent(self, row, site):
245        if not 'students' in site.keys():
246            return None
247        if 'student_id' in row.keys() and row['student_id']:
248            if row['student_id'] in site['students']:
249                student = site['students'][row['student_id']]
250                return student
251        elif 'reg_number' in row.keys() and row['reg_number']:
252            reg_number = row['reg_number']
253            #import pdb; pdb.set_trace()
254            cat = queryUtility(ICatalog, name='students_catalog')
255            results = list(
256                cat.searchResults(reg_number=(reg_number, reg_number)))
257            if results:
258                return results[0]
259        elif 'matric_number' in row.keys() and row['matric_number']:
260            matric_number = row['matric_number']
261            cat = queryUtility(ICatalog, name='students_catalog')
262            results = list(
263                cat.searchResults(matric_number=(matric_number, matric_number)))
264            if results:
265                return results[0]
266        return None
267
268    def parentsExist(self, row, site):
269        return self.getParent(row, site) is not None
270
271    def entryExists(self, row, site):
272        return self.getEntry(row, site) is not None
273
274    def getEntry(self, row, site):
275        student = self.getParent(row, site)
276        if student is None:
277            return None
278        return student.get('studycourse')
279
280    def updateEntry(self, obj, row, site):
281        """Update obj to the values given in row.
282        """
283        for key, value in row.items():
284            # Skip fields not declared in interface.
285            if hasattr(obj, key):
286                setattr(obj, key, value)
287        # Update the students_catalog
288        notify(grok.ObjectModifiedEvent(obj.__parent__))
289        return
290
291    def checkConversion(self, row, mode='ignore'):
292        """Validates all values in row.
293        """
294        converter = IObjectConverter(self.iface)
295        errs, inv_errs, conv_dict =  converter.fromStringDict(
296            row, self.factory_name)
297        # We have to check if current_level is in range of certificate.
298        # This is not done by the converter.
299        if conv_dict.has_key('certificate'):
300          certificate = conv_dict['certificate']
301          start_level = certificate.start_level
302          end_level = certificate.end_level
303          if conv_dict['current_level'] < start_level or \
304              conv_dict['current_level'] > end_level:
305              errs.append(('current_level','not in range'))
306        return errs, inv_errs, conv_dict
307
308class StudentStudyLevelProcessor(BatchProcessor):
309    """A batch processor for IStudentStudyLevel objects.
310    """
311    grok.implements(IBatchProcessor)
312    grok.provides(IBatchProcessor)
313    grok.context(Interface)
314    util_name = 'studylevelimporter'
315    grok.name(util_name)
316
317    name = u'StudentStudyLevel Importer'
318    iface = IStudentStudyLevel
319    factory_name = 'waeup.StudentStudyLevel'
320
321    location_fields = []
322
323    mode = None
324
325    @property
326    def available_fields(self):
327        return sorted(list(set(
328            ['student_id','reg_number','matric_number','level'] + getFields(
329                self.iface).keys())))
330
331    def checkHeaders(self, headerfields, mode='ignore'):
332        if not 'reg_number' in headerfields and not 'student_id' \
333            in headerfields and not 'matric_number' in headerfields:
334            raise FatalCSVError(
335                "Need at least columns student_id " +
336                "or reg_number or matric_number for import!")
337        if not 'level' in headerfields:
338            raise FatalCSVError(
339                "Need level for import!")
340        # Check for fields to be ignored...
341        not_ignored_fields = [x for x in headerfields
342                              if not x.startswith('--')]
343        if len(set(not_ignored_fields)) < len(not_ignored_fields):
344            raise FatalCSVError(
345                "Double headers: each column name may only appear once.")
346        return True
347
348    def getParent(self, row, site):
349        if not 'students' in site.keys():
350            return None
351        if 'student_id' in row.keys() and row['student_id']:
352            if row['student_id'] in site['students']:
353                student = site['students'][row['student_id']]
354                return student['studycourse']
355        elif 'reg_number' in row.keys() and row['reg_number']:
356            reg_number = row['reg_number']
357            #import pdb; pdb.set_trace()
358            cat = queryUtility(ICatalog, name='students_catalog')
359            results = list(
360                cat.searchResults(reg_number=(reg_number, reg_number)))
361            if results:
362                return results[0]['studycourse']
363        elif 'matric_number' in row.keys() and row['matric_number']:
364            matric_number = row['matric_number']
365            cat = queryUtility(ICatalog, name='students_catalog')
366            results = list(
367                cat.searchResults(matric_number=(matric_number, matric_number)))
368            if results:
369                return results[0]['studycourse']
370        return None
371
372    def parentsExist(self, row, site):
373        return self.getParent(row, site) is not None
374
375    def entryExists(self, row, site):
376        return self.getEntry(row, site) is not None
377
378    def getEntry(self, row, site):
379        studycourse = self.getParent(row, site)
380        if studycourse is None:
381            return None
382        return studycourse.get(row['level'])
383
384    def addEntry(self, obj, row, site):
385        parent = self.getParent(row, site)
386        obj.level = int(row['level'])
387        parent[row['level']] = obj
388        return
389
390    def checkConversion(self, row, mode='ignore'):
391        """Validates all values in row.
392        """
393        converter = IObjectConverter(self.iface)
394        errs, inv_errs, conv_dict =  converter.fromStringDict(
395            row, self.factory_name)
396        # We have to check if level is a valid integer.
397        # This is not by the converter.
398        try:
399            level = int(row['level'])
400            if level not in range(0,600,10):
401                errs.append(('level','no valid integer'))
402        except ValueError:
403            errs.append(('level','no integer'))
404        return errs, inv_errs, conv_dict
Note: See TracBrowser for help on using the repository browser.