source: main/waeup.kofa/trunk/src/waeup/kofa/university/batching.py @ 14395

Last change on this file since 14395 was 13607, checked in by Henrik Bettermann, 9 years ago

Fix certificate course importer. Empty mandatory fields were imported as <IGNORE>.
  • Property svn:keywords set to Id
File size: 15.8 KB
Line 
1## $Id: batching.py 13607 2016-01-13 06:34:05Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing components for academics objects.
19
20Batch processors eat CSV files to add, update or remove large numbers
21of certain kinds of objects at once.
22
23Here we define the processors for academics specific objects like
24faculties, departments and the like.
25"""
26import grok
27from zope.interface import Interface
28from zope.component import queryUtility
29from zope.schema import getFields
30from zope.catalog.interfaces import ICatalog
31from zope.event import notify
32from zope.securitypolicy.interfaces import (
33    IPrincipalRoleManager, IPrincipalRoleMap)
34from waeup.kofa.authentication import LocalRoleSetEvent
35from waeup.kofa.interfaces import (
36    IBatchProcessor, IGNORE_MARKER, DELETION_MARKER, FatalCSVError)
37from waeup.kofa.university.interfaces import (
38    IFacultiesContainer, IFaculty, ICourse, IDepartment, ICertificate,
39    ICertificateCourse)
40from waeup.kofa.university import (
41    Faculty, Department, Course, Certificate)
42from waeup.kofa.utils.batching import BatchProcessor
43from waeup.kofa.interfaces import MessageFactory as _
44
45class FacultyProcessor(BatchProcessor):
46    """The Faculty Processor processes faculties in the `faculties` container.
47    The `FacultyProcessor` class also serves as a baseclass for all other
48    batch processors in the academic section.
49
50    The processor makes some efforts to set local roles.
51    If new roles are provided, the `updateEntry` method first removes
52    all existing roles and then sets the new roles as given in the import
53    file. That means the entire set of local roles is replaced.
54    """
55    grok.implements(IBatchProcessor)
56    grok.provides(IBatchProcessor)
57    grok.context(Interface)
58    util_name = 'facultyprocessor'
59    grok.name(util_name)
60
61    name = _('Faculty Processor')
62    iface = IFaculty
63    allowed_roles = Faculty.local_roles
64
65    location_fields = ['code',]
66    factory_name = 'waeup.Faculty'
67
68    mode = None
69
70    def parentsExist(self, row, site):
71        return 'faculties' in site.keys()
72
73    @property
74    def available_fields(self):
75        fields = getFields(self.iface)
76        return sorted(list(set(
77            self.location_fields + fields.keys() + ['local_roles']
78            )))
79
80    def entryExists(self, row, site):
81        return row['code'] in site['faculties'].keys()
82
83    def getParent(self, row, site):
84        return site['faculties']
85
86    def getEntry(self, row, site):
87        if not self.entryExists(row, site):
88            return None
89        parent = self.getParent(row, site)
90        return parent.get(row['code'])
91
92    def addEntry(self, obj, row, site):
93        parent = self.getParent(row, site)
94        parent.addFaculty(obj)
95        return
96
97    def delEntry(self, row, site):
98        parent = self.getParent(row, site)
99        del parent[row['code']]
100        pass
101
102    def updateEntry(self, obj, row, site, filename):
103        """Update obj to the values given in row.
104        """
105        items_changed = ''
106
107        if 'local_roles' in row and row['local_roles'] not in (
108            None, IGNORE_MARKER):
109            role_manager = IPrincipalRoleManager(obj)
110            role_map = IPrincipalRoleMap(obj)
111            # Remove all existing local roles.
112            for local_role, user_name, setting in \
113                role_map.getPrincipalsAndRoles():
114                role_manager.unsetRoleForPrincipal(local_role, user_name)
115                notify(LocalRoleSetEvent(
116                        obj, local_role, user_name, granted=False))
117            # Add new local roles.
118            if row['local_roles'] != DELETION_MARKER:
119                local_roles = eval(row['local_roles'])
120                for rolemap in local_roles:
121                    user = rolemap['user_name']
122                    local_role = rolemap['local_role']
123                    role_manager.assignRoleToPrincipal(local_role, user)
124                    notify(LocalRoleSetEvent(
125                        obj, local_role, user, granted=True))
126                    items_changed += ('%s=%s, '
127                        % ('local_roles', '%s|%s' % (user,local_role)))
128            row.pop('local_roles')
129
130        # apply other values...
131        items_changed += super(FacultyProcessor, self).updateEntry(
132            obj, row, site, filename)
133
134        # Log actions...
135        location_field = self.location_fields[0]
136        grok.getSite().logger.info('%s - %s - %s - updated: %s'
137            % (self.name, filename, row[location_field], items_changed))
138        return items_changed
139
140    def checkConversion(self, row, mode='create'):
141        """Validates all values in row.
142        """
143        errs, inv_errs, conv_dict =  super(
144            FacultyProcessor, self).checkConversion(row, mode=mode)
145        if 'local_roles' in row:
146            if row['local_roles'] in (None, DELETION_MARKER, IGNORE_MARKER):
147                return errs, inv_errs, conv_dict
148            try:
149                local_roles = eval(row['local_roles'])
150            except:
151                errs.append(('local_roles','Error'))
152                return errs, inv_errs, conv_dict
153            if not isinstance(local_roles, list):
154                errs.append(('local_roles','no list'))
155                return errs, inv_errs, conv_dict
156            for rolemap in local_roles:
157                if not isinstance(rolemap, dict):
158                    errs.append(('local_roles','no dicts'))
159                    return errs, inv_errs, conv_dict
160                if not 'user_name' in rolemap.keys() or not \
161                    'local_role' in rolemap.keys():
162                    errs.append((
163                        'local_roles','user_name or local_role missing'))
164                    return errs, inv_errs, conv_dict
165                local_role = rolemap['local_role']
166                if not local_role in self.allowed_roles:
167                    errs.append(('local_roles','%s not allowed' % local_role))
168                    return errs, inv_errs, conv_dict
169                user = rolemap['user_name']
170                users = grok.getSite()['users']
171                if not user in users.keys():
172                    errs.append(('local_roles','%s does not exist' % user))
173                    return errs, inv_errs, conv_dict
174        return errs, inv_errs, conv_dict
175
176class DepartmentProcessor(FacultyProcessor):
177    """The Department Processor works in the same way as the Faculty
178    Processor. Since department codes are not necessarily unique, it needs the
179    `faculty_code` to create and update objects.
180    """
181    grok.implements(IBatchProcessor)
182    grok.provides(IBatchProcessor)
183    grok.context(Interface)
184    util_name = 'departmentprocessor'
185    grok.name(util_name)
186
187    name = _('Department Processor')
188    iface = IDepartment
189    allowed_roles = Department.local_roles
190
191    location_fields = ['code', 'faculty_code']
192    factory_name = 'waeup.Department'
193
194    mode = None
195
196    def parentsExist(self, row, site):
197        if not 'faculties' in site.keys():
198            return False
199        return row['faculty_code'] in site['faculties']
200
201    def entryExists(self, row, site):
202        if not self.parentsExist(row, site):
203            return False
204        parent = self.getParent(row, site)
205        return row['code'] in parent.keys()
206
207    def getParent(self, row, site):
208        return site['faculties'][row['faculty_code']]
209
210    def getEntry(self, row, site):
211        if not self.entryExists(row, site):
212            return None
213        parent = self.getParent(row, site)
214        return parent.get(row['code'])
215
216    def addEntry(self, obj, row, site):
217        parent = self.getParent(row, site)
218        parent.addDepartment(obj)
219        return
220
221    def delEntry(self, row, site):
222        parent = self.getParent(row, site)
223        del parent[row['code']]
224        return
225
226class CertificateProcessor(FacultyProcessor):
227    """The Certificate Processor gets the parent object (the
228    `certificates` attribute of the department container) in two ways.
229    If both faculty and department codes are provided, `getPartents` uses
230    these to locate the certificate. If department code or
231    faculty code are missing, it use the certificates catalog to find the
232    certificate.
233    """
234    grok.implements(IBatchProcessor)
235    grok.provides(IBatchProcessor)
236    grok.context(Interface)
237    util_name = 'certificateprocessor'
238    grok.name(util_name)
239
240    name = _('Certificate Processor')
241    iface = ICertificate
242    allowed_roles = Certificate.local_roles
243
244    location_fields = ['code']
245    factory_name = 'waeup.Certificate'
246
247    mode = None
248
249    @property
250    def available_fields(self):
251        fields = getFields(self.iface)
252        return sorted(list(set(
253            ['faculty_code','department_code'] + fields.keys()
254            + ['local_roles'])))
255
256    def checkHeaders(self, headerfields, mode='create'):
257        super(CertificateProcessor, self).checkHeaders(headerfields, mode)
258        if mode == 'create':
259            if not 'faculty_code' in headerfields \
260                and not 'department_code' in headerfields :
261                raise FatalCSVError(
262                    "Need at least columns faculty_code and department_code")
263        return True
264
265    def parentsExist(self, row, site):
266        return self.getParent(row,site) is not None
267
268    def entryExists(self, row, site):
269        parent = self.getParent(row, site)
270        if parent is not None:
271            return row['code'] in parent.keys()
272        return False
273
274    def getParent(self, row, site):
275        if not 'faculties' in site.keys():
276            return None
277        # If both faculty and department codes are provided, use
278        # these to get parent.
279        if row.get('faculty_code',None) not in (None, IGNORE_MARKER) and \
280            row.get('department_code',None) not in (None, IGNORE_MARKER):
281            if not row['faculty_code'] in site['faculties'].keys():
282                return None
283            faculty = site['faculties'][row['faculty_code']]
284            if not row['department_code'] in faculty.keys():
285                return None
286            dept = faculty[row['department_code']]
287            return dept.certificates
288        # If department code or faculty code is missing,
289        # use catalog to get parent.
290        cat = queryUtility(ICatalog, name='certificates_catalog')
291        results = list(
292            cat.searchResults(code=(row['code'], row['code'])))
293        if results:
294            return results[0].__parent__
295        return None
296
297    def getEntry(self, row, site):
298        parent = self.getParent(row, site)
299        if parent is not None:
300            return parent.get(row['code'])
301        return None
302
303    def addEntry(self, obj, row, site):
304        parent = self.getParent(row, site)
305        parent.addCertificate(obj)
306        return
307
308    def delEntry(self, row, site):
309        parent = self.getParent(row, site)
310        del parent[row['code']]
311        return
312
313class CourseProcessor(CertificateProcessor):
314    """The Course Processor works exactly in the same way as the
315    Certificate Processor. It uses the courses catalog instead of the
316    certificates catalog.
317    """
318    grok.implements(IBatchProcessor)
319    grok.provides(IBatchProcessor)
320    grok.context(Interface)
321    util_name = 'courseprocessor'
322    grok.name(util_name)
323
324    name = _('Course Processor')
325    iface = ICourse
326    allowed_roles = Course.local_roles
327
328    location_fields = ['code']
329    factory_name = 'waeup.Course'
330
331    mode = None
332
333    def getParent(self, row, site):
334        if not 'faculties' in site.keys():
335            return None
336        # If both faculty and department codes are provided, use
337        # these to get parent.
338        if row.get('faculty_code',None) not in (None, IGNORE_MARKER) and \
339            row.get('department_code',None) not in (None, IGNORE_MARKER):
340            if not row['faculty_code'] in site['faculties'].keys():
341                return None
342            faculty = site['faculties'][row['faculty_code']]
343            if not row['department_code'] in faculty.keys():
344                return None
345            dept = faculty[row['department_code']]
346            return dept.courses
347        # If department code or faculty code is missing,
348        # use catalog to get parent. Makes only sense in update mode but
349        # does also work in create mode.
350        cat = queryUtility(ICatalog, name='courses_catalog')
351        results = list(
352            cat.searchResults(code=(row['code'], row['code'])))
353        if results:
354            return results[0].__parent__
355        return None
356
357    def addEntry(self, obj, row, site):
358        parent = self.getParent(row, site)
359        parent.addCourse(obj)
360        return
361
362class CertificateCourseProcessor(FacultyProcessor):
363    """The Certificate Course Processor needs more location fields.
364    Certificate courses are stored inside the certificate container.
365    Thus, `faculty_code`, `department_code` and the
366    `certificate_code` are necessary to find the parent container.
367    It furthermore needs the `course` and the `level` field to locate
368    existing objects as they are part of the object id (code).
369    Consequently, `course` and `level` can't be updated. If the level
370    has changed, the object must be replaced (removed and re-created),
371    for instance by processing two import files.
372    """
373    grok.implements(IBatchProcessor)
374    grok.provides(IBatchProcessor)
375    grok.context(Interface)
376    util_name = 'certificatecourseprocessor'
377    grok.name(util_name)
378
379    name = _('CertificateCourse Processor')
380    iface = ICertificateCourse
381
382    location_fields = ['certificate_code', 'course', 'level', 'faculty_code',
383                       'department_code',]
384    factory_name = 'waeup.CertificateCourse'
385
386    mode = None
387
388    def parentsExist(self, row, site):
389        if not 'faculties' in site.keys():
390            return False
391        if not row['faculty_code'] in site['faculties'].keys():
392            return False
393        faculty = site['faculties'][row['faculty_code']]
394        if not row['department_code'] in faculty.keys():
395            return False
396        dept = faculty[row['department_code']]
397        return row['certificate_code'] in dept.certificates.keys()
398
399    def entryExists(self, row, site):
400        if not self.parentsExist(row, site):
401            return False
402        parent = self.getParent(row, site)
403        code = "%s_%s" % (row['course'].code, row['level'])
404        return code in parent.keys()
405
406    def getParent(self, row, site):
407        dept = site['faculties'][row['faculty_code']][row['department_code']]
408        return dept.certificates[row['certificate_code']]
409
410    def getEntry(self, row, site):
411        if not self.entryExists(row, site):
412            return None
413        parent = self.getParent(row, site)
414        return parent.get("%s_%s" % (row['course'].code, row['level']))
415
416    def addEntry(self, obj, row, site):
417        parent = self.getParent(row, site)
418        # mandatory is not a required and might be missing or can be
419        # the ignore-marker.
420        mandatory = row.get('mandatory', None)
421        if mandatory in (None, IGNORE_MARKER):
422            parent.addCertCourse(row['course'], row['level'])
423            return
424        parent.addCertCourse(row['course'], row['level'], mandatory)
425        return
426
427    def delEntry(self, row, site):
428        parent = self.getParent(row, site)
429        parent.delCertCourses(row['course'].code, row['level'])
430        return
Note: See TracBrowser for help on using the repository browser.