source: main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py @ 7329

Last change on this file since 7329 was 7269, checked in by Henrik Bettermann, 13 years ago

Fix test and simplify if statements in batching.py.
  • Property svn:keywords set to Id
File size: 8.9 KB
Line 
1## $Id: batching.py 7269 2011-12-04 17:41:40Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for academics objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for students specific objects like
24students, studycourses, payment tickets and accommodation tickets.
25"""
26import grok
27import csv
28from zope.interface import Interface
29from zope.schema import getFields
30from zope.component import queryUtility
31from zope.catalog.interfaces import ICatalog
32from waeup.sirp.interfaces import (
33    IBatchProcessor, FatalCSVError, IObjectConverter)
34from waeup.sirp.students.interfaces import (
35    IStudent, IStudentStudyCourseImport,
36    IStudentUpdateByRegNo, IStudentUpdateByMatricNo)
37from waeup.sirp.utils.batching import BatchProcessor
38
39class StudentProcessor(BatchProcessor):
40    """A batch processor for IStudent objects.
41    """
42    grok.implements(IBatchProcessor)
43    grok.provides(IBatchProcessor)
44    grok.context(Interface)
45    util_name = 'studentimporter'
46    grok.name(util_name)
47
48    name = u'Student Importer'
49    iface = IStudent
50
51    location_fields = []
52    factory_name = 'waeup.Student'
53
54    mode = None
55
56    @property
57    def available_fields(self):
58        return sorted(list(set(
59            ['student_id','reg_number','matric_number'] + getFields(
60                self.iface).keys())))
61
62    def checkHeaders(self, headerfields, mode='create'):
63        if not 'reg_number' in headerfields and not 'student_id' \
64            in headerfields and not 'matric_number' in headerfields:
65            raise FatalCSVError(
66                "Need at least columns student_id or reg_number " +
67                "or matric_number for import!")
68        if mode == 'create':
69            for field in self.required_fields:
70                if not field in headerfields:
71                    raise FatalCSVError(
72                        "Need at least columns %s for import!" %
73                        ', '.join(["'%s'" % x for x in self.required_fields]))
74        # Check for fields to be ignored...
75        not_ignored_fields = [x for x in headerfields
76                              if not x.startswith('--')]
77        if len(set(not_ignored_fields)) < len(not_ignored_fields):
78            raise FatalCSVError(
79                "Double headers: each column name may only appear once.")
80        return True
81
82    def parentsExist(self, row, site):
83        return 'students' in site.keys()
84
85    def getLocator(self, row):
86        if row.get('student_id',None):
87            return 'student_id'
88        elif row.get('reg_number',None):
89            return 'reg_number'
90        elif row.get('matric_number',None):
91            return 'matric_number'
92        else:
93            return None
94
95    # The entry never exists in create mode.
96    def entryExists(self, row, site):
97        return self.getEntry(row, site) is not None
98
99    def getParent(self, row, site):
100        return site['students']
101
102    def getEntry(self, row, site):
103        if not 'students' in site.keys():
104            return None
105        if self.getLocator(row) == 'student_id':
106            if row['student_id'] in site['students']:
107                student = site['students'][row['student_id']]
108                return student
109        elif self.getLocator(row) == 'reg_number':
110            reg_number = row['reg_number']
111            cat = queryUtility(ICatalog, name='students_catalog')
112            results = list(
113                cat.searchResults(reg_number=(reg_number, reg_number)))
114            if results:
115                return results[0]
116        elif self.getLocator(row) == 'matric_number':
117            matric_number = row['matric_number']
118            cat = queryUtility(ICatalog, name='students_catalog')
119            results = list(
120                cat.searchResults(matric_number=(matric_number, matric_number)))
121            if results:
122                return results[0]
123        return None
124
125       
126    def addEntry(self, obj, row, site):
127        parent = self.getParent(row, site)
128        parent.addStudent(obj)
129        return
130
131    def delEntry(self, row, site):
132        student = self.getEntry(row, site)
133        if student is not None:
134            parent = self.getParent(row, site)
135            del parent[student.student_id]
136        pass
137
138    def getMapping(self, path, headerfields, mode):
139        """Get a mapping from CSV file headerfields to actually used fieldnames.
140        """
141        result = dict()
142        reader = csv.reader(open(path, 'rb'))
143        raw_header = reader.next()
144        for num, field in enumerate(headerfields):
145            if field not in [
146                'student_id', 'reg_number', 'matric_number'] and mode == 'remove':
147                continue
148            if field == u'--IGNORE--':
149                # Skip ignored columns in failed and finished data files.
150                continue
151            result[raw_header[num]] = field
152        return result
153
154    def checkConversion(self, row, mode='create'):
155        """Validates all values in row.
156        """
157        if mode in ['update', 'remove']:
158            if self.getLocator(row) == 'reg_number':
159                iface = IStudentUpdateByRegNo
160            elif self.getLocator(row) == 'matric_number':
161                iface = IStudentUpdateByMatricNo
162        else:
163            iface = self.iface
164        converter = IObjectConverter(iface)
165        errs, inv_errs, conv_dict =  converter.fromStringDict(
166            row, self.factory_name)
167        return errs, inv_errs, conv_dict
168
169class StudentStudyCourseProcessor(BatchProcessor):
170    """A batch processor for IStudentStudyCourse objects.
171    """
172    grok.implements(IBatchProcessor)
173    grok.provides(IBatchProcessor)
174    grok.context(Interface)
175    util_name = 'studycourseupdater'
176    grok.name(util_name)
177
178    name = u'StudentStudyCourse Importer (update only)'
179    iface = IStudentStudyCourseImport
180    factory_name = 'waeup.StudentStudyCourse'
181
182    location_fields = []
183
184    mode = None
185
186    @property
187    def available_fields(self):
188        return sorted(list(set(
189            ['student_id','reg_number','matric_number'] + getFields(
190                self.iface).keys())))
191
192    def checkHeaders(self, headerfields, mode='ignore'):
193        if not 'reg_number' in headerfields and not 'student_id' \
194            in headerfields and not 'matric_number' in headerfields:
195            raise FatalCSVError(
196                "Need at least columns student_id " +
197                "or reg_number or matric_number for import!")
198        # Check for fields to be ignored...
199        not_ignored_fields = [x for x in headerfields
200                              if not x.startswith('--')]
201        if len(set(not_ignored_fields)) < len(not_ignored_fields):
202            raise FatalCSVError(
203                "Double headers: each column name may only appear once.")
204        return True
205
206    def getParent(self, row, site):
207        if not 'students' in site.keys():
208            return None
209        if 'student_id' in row.keys() and row['student_id']:
210            if row['student_id'] in site['students']:
211                student = site['students'][row['student_id']]
212                return student
213        elif 'reg_number' in row.keys() and row['reg_number']:
214            reg_number = row['reg_number']
215            #import pdb; pdb.set_trace()
216            cat = queryUtility(ICatalog, name='students_catalog')
217            results = list(
218                cat.searchResults(reg_number=(reg_number, reg_number)))
219            if results:
220                return results[0]
221        elif 'matric_number' in row.keys() and row['matric_number']:
222            matric_number = row['matric_number']
223            cat = queryUtility(ICatalog, name='students_catalog')
224            results = list(
225                cat.searchResults(matric_number=(matric_number, matric_number)))
226            if results:
227                return results[0]
228        return None
229
230    def parentsExist(self, row, site):
231        return self.getParent(row, site) is not None
232
233    def entryExists(self, row, site):
234        student = self.getParent(row, site)
235        if not student:
236            return None
237        if 'studycourse' in student:
238            return student
239        return None
240
241    def getEntry(self, row, site):
242        student = self.entryExists(row, site)
243        if not student:
244            return None
245        return student.get('studycourse')
Note: See TracBrowser for help on using the repository browser.