source: main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py @ 7522

Last change on this file since 7522 was 7522, checked in by Henrik Bettermann, 13 years ago

Add history messages and log file entries when importing students.

Do not accept empty reg_state fields. If the the reg_state column exists a valid value must be provided.
  • Property svn:keywords set to Id
File size: 10.8 KB
Line 
1## $Id: batching.py 7522 2012-01-27 16:33:02Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for student objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility
31from zope.event import notify
32from zope.catalog.interfaces import ICatalog
33from hurry.workflow.interfaces import IWorkflowState
34from waeup.sirp.interfaces import (
35    IBatchProcessor, FatalCSVError, IObjectConverter, IUserAccount,
36    IObjectHistory)
37from waeup.sirp.students.interfaces import (
38    IStudent, IStudentStudyCourseImport,
39    IStudentUpdateByRegNo, IStudentUpdateByMatricNo)
40from waeup.sirp.students.workflow import  IMPORTABLE_STATES
41from waeup.sirp.utils.batching import BatchProcessor
42from waeup.sirp.utils.helpers import get_current_principal
43
44class StudentProcessor(BatchProcessor):
45    """A batch processor for IStudent objects.
46    """
47    grok.implements(IBatchProcessor)
48    grok.provides(IBatchProcessor)
49    grok.context(Interface)
50    util_name = 'studentimporter'
51    grok.name(util_name)
52
53    name = u'Student Importer'
54    iface = IStudent
55
56    location_fields = []
57    factory_name = 'waeup.Student'
58
59    mode = None
60
61    @property
62    def available_fields(self):
63        return sorted(list(set(
64            ['student_id','reg_number','matric_number',
65            'password', 'reg_state'] + getFields(
66                self.iface).keys())))
67
68    def checkHeaders(self, headerfields, mode='create'):
69        if not 'reg_number' in headerfields and not 'student_id' \
70            in headerfields and not 'matric_number' in headerfields:
71            raise FatalCSVError(
72                "Need at least columns student_id or reg_number " +
73                "or matric_number for import!")
74        if mode == 'create':
75            for field in self.required_fields:
76                if not field in headerfields:
77                    raise FatalCSVError(
78                        "Need at least columns %s for import!" %
79                        ', '.join(["'%s'" % x for x in self.required_fields]))
80        # Check for fields to be ignored...
81        not_ignored_fields = [x for x in headerfields
82                              if not x.startswith('--')]
83        if len(set(not_ignored_fields)) < len(not_ignored_fields):
84            raise FatalCSVError(
85                "Double headers: each column name may only appear once.")
86        return True
87
88    def parentsExist(self, row, site):
89        return 'students' in site.keys()
90
91    def getLocator(self, row):
92        if row.get('student_id',None):
93            return 'student_id'
94        elif row.get('reg_number',None):
95            return 'reg_number'
96        elif row.get('matric_number',None):
97            return 'matric_number'
98        else:
99            return None
100
101    # The entry never exists in create mode.
102    def entryExists(self, row, site):
103        return self.getEntry(row, site) is not None
104
105    def getParent(self, row, site):
106        return site['students']
107
108    def getEntry(self, row, site):
109        if not 'students' in site.keys():
110            return None
111        if self.getLocator(row) == 'student_id':
112            if row['student_id'] in site['students']:
113                student = site['students'][row['student_id']]
114                return student
115        elif self.getLocator(row) == 'reg_number':
116            reg_number = row['reg_number']
117            cat = queryUtility(ICatalog, name='students_catalog')
118            results = list(
119                cat.searchResults(reg_number=(reg_number, reg_number)))
120            if results:
121                return results[0]
122        elif self.getLocator(row) == 'matric_number':
123            matric_number = row['matric_number']
124            cat = queryUtility(ICatalog, name='students_catalog')
125            results = list(
126                cat.searchResults(matric_number=(matric_number, matric_number)))
127            if results:
128                return results[0]
129        return None
130
131       
132    def addEntry(self, obj, row, site):
133        parent = self.getParent(row, site)
134        parent.addStudent(obj)
135        # In some tests we don't have a students container or a user
136        try:
137            user = get_current_principal()
138            parent.logger.info('%s - %s - Student record imported' % (
139                user.id,obj.student_id))
140            history = IObjectHistory(obj)
141            history.addMessage('Student record imported')
142        except (TypeError, AttributeError):
143            pass
144        return
145
146    def delEntry(self, row, site):
147        student = self.getEntry(row, site)
148        if student is not None:
149            parent = self.getParent(row, site)
150            del parent[student.student_id]
151        pass
152
153    def updateEntry(self, obj, row, site):
154        """Update obj to the values given in row.
155        """
156        for key, value in row.items():
157            # Set student password and all fields declared in interface.
158            if key == 'password' and value != '':
159                IUserAccount(obj).setPassword(value)
160            elif key == 'reg_state':
161                IWorkflowState(obj).setState(value)
162                msg = "State '%s' set" % value
163                history = IObjectHistory(obj)
164                history.addMessage(msg)
165            elif hasattr(obj, key):
166                setattr(obj, key, value)
167        return
168
169    def getMapping(self, path, headerfields, mode):
170        """Get a mapping from CSV file headerfields to actually used fieldnames.
171        """
172        result = dict()
173        reader = csv.reader(open(path, 'rb'))
174        raw_header = reader.next()
175        for num, field in enumerate(headerfields):
176            if field not in [
177                'student_id', 'reg_number', 'matric_number'] and mode == 'remove':
178                continue
179            if field == u'--IGNORE--':
180                # Skip ignored columns in failed and finished data files.
181                continue
182            result[raw_header[num]] = field
183        return result
184
185    def checkConversion(self, row, mode='create'):
186        """Validates all values in row.
187        """
188        if mode in ['update', 'remove']:
189            if self.getLocator(row) == 'reg_number':
190                iface = IStudentUpdateByRegNo
191            elif self.getLocator(row) == 'matric_number':
192                iface = IStudentUpdateByMatricNo
193        else:
194            iface = self.iface
195        converter = IObjectConverter(iface)
196        errs, inv_errs, conv_dict =  converter.fromStringDict(
197            row, self.factory_name)
198        if row.has_key('reg_state') and \
199            not row['reg_state'] in IMPORTABLE_STATES:
200            if row['reg_state'] != '':
201                errs.append(('reg_state','not allowed'))
202            else:
203                errs.append(('reg_state','no value provided'))
204        return errs, inv_errs, conv_dict
205
206class StudentStudyCourseProcessor(BatchProcessor):
207    """A batch processor for IStudentStudyCourse objects.
208    """
209    grok.implements(IBatchProcessor)
210    grok.provides(IBatchProcessor)
211    grok.context(Interface)
212    util_name = 'studycourseupdater'
213    grok.name(util_name)
214
215    name = u'StudentStudyCourse Importer (update only)'
216    iface = IStudentStudyCourseImport
217    factory_name = 'waeup.StudentStudyCourse'
218
219    location_fields = []
220
221    mode = None
222
223    @property
224    def available_fields(self):
225        return sorted(list(set(
226            ['student_id','reg_number','matric_number'] + getFields(
227                self.iface).keys())))
228
229    def checkHeaders(self, headerfields, mode='ignore'):
230        if not 'reg_number' in headerfields and not 'student_id' \
231            in headerfields and not 'matric_number' in headerfields:
232            raise FatalCSVError(
233                "Need at least columns student_id " +
234                "or reg_number or matric_number for import!")
235        # Check for fields to be ignored...
236        not_ignored_fields = [x for x in headerfields
237                              if not x.startswith('--')]
238        if len(set(not_ignored_fields)) < len(not_ignored_fields):
239            raise FatalCSVError(
240                "Double headers: each column name may only appear once.")
241        return True
242
243    def getParent(self, row, site):
244        if not 'students' in site.keys():
245            return None
246        if 'student_id' in row.keys() and row['student_id']:
247            if row['student_id'] in site['students']:
248                student = site['students'][row['student_id']]
249                return student
250        elif 'reg_number' in row.keys() and row['reg_number']:
251            reg_number = row['reg_number']
252            #import pdb; pdb.set_trace()
253            cat = queryUtility(ICatalog, name='students_catalog')
254            results = list(
255                cat.searchResults(reg_number=(reg_number, reg_number)))
256            if results:
257                return results[0]
258        elif 'matric_number' in row.keys() and row['matric_number']:
259            matric_number = row['matric_number']
260            cat = queryUtility(ICatalog, name='students_catalog')
261            results = list(
262                cat.searchResults(matric_number=(matric_number, matric_number)))
263            if results:
264                return results[0]
265        return None
266
267    def parentsExist(self, row, site):
268        return self.getParent(row, site) is not None
269
270    def entryExists(self, row, site):
271        student = self.getParent(row, site)
272        if not student:
273            return None
274        if 'studycourse' in student:
275            return student
276        return None
277
278    def getEntry(self, row, site):
279        student = self.entryExists(row, site)
280        if not student:
281            return None
282        return student.get('studycourse')
283
284    def updateEntry(self, obj, row, site):
285        """Update obj to the values given in row.
286        """
287        for key, value in row.items():
288            # Skip fields not declared in interface.
289            if hasattr(obj, key):
290                setattr(obj, key, value)
291        # Update the students_catalog
292        notify(grok.ObjectModifiedEvent(obj.__parent__))
293        return
294
Note: See TracBrowser for help on using the repository browser.