source: main/waeup.kofa/trunk/src/waeup/kofa/applicants/batching.py @ 12869

Last change on this file since 12869 was 12869, checked in by Henrik Bettermann, 9 years ago

Start documenting batch processors.
  • Property svn:keywords set to Id
File size: 13.2 KB
Line 
1## $Id: batching.py 12869 2015-04-22 19:14:14Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing for applicants.
19"""
20import unicodecsv as csv # XXX: csv ops should move to dedicated module.
21import grok
22from zope.schema import getFields
23from zope.interface import Interface
24from zope.component import queryUtility
25from hurry.workflow.interfaces import IWorkflowState
26from zope.catalog.interfaces import ICatalog
27from waeup.kofa.interfaces import (
28    IBatchProcessor, IObjectConverter, FatalCSVError, IGNORE_MARKER,
29    IObjectHistory, IUserAccount)
30from waeup.kofa.interfaces import MessageFactory as _
31from waeup.kofa.utils.batching import BatchProcessor
32from waeup.kofa.applicants.interfaces import (
33    IApplicantsContainer, IApplicant, IApplicantUpdateByRegNo)
34from waeup.kofa.applicants.workflow import  IMPORTABLE_STATES, CREATED
35
36class ApplicantsContainerProcessor(BatchProcessor):
37    """The Applicants Container Processor imports containers for applicants.
38    It does not import their content. There is nothing special about this
39    processor.
40    """
41    grok.implements(IBatchProcessor)
42    grok.provides(IBatchProcessor)
43    grok.context(Interface)
44    util_name = 'applicantscontainerprocessor'
45    grok.name(util_name)
46
47    name = _('ApplicantsContainer Processor')
48    mode = u'create'
49    iface = IApplicantsContainer
50
51    location_fields = ['code',]
52    factory_name = 'waeup.ApplicantsContainer'
53
54    def parentsExist(self, row, site):
55        return 'applicants' in site.keys()
56
57    def entryExists(self, row, site):
58        return row['code'] in site['applicants'].keys()
59
60    def getParent(self, row, site):
61        return site['applicants']
62
63    def getEntry(self, row, site):
64        if not self.entryExists(row, site):
65            return None
66        parent = self.getParent(row, site)
67        return parent.get(row['code'])
68
69    def addEntry(self, obj, row, site):
70        parent = self.getParent(row, site)
71        parent[row['code']] = obj
72        return
73
74    def delEntry(self, row, site):
75        parent = self.getParent(row, site)
76        del parent[row['code']]
77        return
78
79class ApplicantProcessor(BatchProcessor):
80    """The Applicant Processor imports application records (applicants).
81
82    In create mode `container_code` is required. If `application_number` is
83    given, an applicant with this number is created in the designated container.
84    If `application_number` is not given, a random `application_number` is
85    assigned. `applicant_id` is being determined by the system and can't be
86    imported.
87
88    In update or remove mode `container_code` and `application_number` columns
89    must not exist. The applicant object is solely localized by
90    `applicant_id` or `reg_number`.
91    """
92    grok.implements(IBatchProcessor)
93    grok.provides(IBatchProcessor)
94    grok.context(Interface)
95    util_name = 'applicantprocessor'
96    grok.name(util_name)
97    name = _('Applicant Processor')
98    iface = IApplicant
99    iface_byregnumber = IApplicantUpdateByRegNo
100    factory_name = 'waeup.Applicant'
101
102    mode = None
103
104    @property
105    def available_fields(self):
106        return sorted(list(set(
107            ['application_number',
108            'container_code','state','password'] + getFields(
109                self.iface).keys())))
110
111    def checkHeaders(self, headerfields, mode='create'):
112        cond1 = 'container_code' in headerfields
113        cond2 = 'application_number' in headerfields
114        cond3 = 'applicant_id' in headerfields
115        cond4 = 'reg_number' in headerfields
116        if mode == 'create':
117            if not cond1:
118                raise FatalCSVError(
119                    "Need at least container_code column!")
120            if cond3:
121                raise FatalCSVError(
122                    "applicant_id can't be imported in create mode!")
123            for field in self.required_fields:
124                if not field in headerfields:
125                    raise FatalCSVError(
126                        "Need at least columns %s for import!" %
127                        ', '.join(["'%s'" % x for x in self.required_fields]))
128        if mode in ('update', 'remove'):
129            if not cond3 and not cond4:
130                raise FatalCSVError(
131                    "Need at least column reg_number or applicant_id!")
132            if cond1 or cond2:
133                raise FatalCSVError(
134                    "container_code or application_number can't be imported " +
135                    "in update or remove mode!")
136        # Check for fields to be ignored...
137        not_ignored_fields = [x for x in headerfields
138                              if not x.startswith('--')]
139        if len(set(not_ignored_fields)) < len(not_ignored_fields):
140            raise FatalCSVError(
141                "Double headers: each column name may only appear once.")
142        return True
143
144    def getLocator(self, row):
145        if row.get('container_code', None) not in (IGNORE_MARKER, None):
146            # create mode
147            return 'container_code'
148        elif row.get('applicant_id', None) not in (IGNORE_MARKER, None):
149            # update or remove mode
150            return 'applicant_id'
151        elif row.get('reg_number', None) not in (IGNORE_MARKER, None):
152            # update or remove mode
153            return 'reg_number'
154        else:
155            return None
156
157    def getParent(self, row, site):
158        result = None
159        if self.getLocator(row) == 'container_code':
160            result = site['applicants'].get(row['container_code'], None)
161        elif self.getLocator(row) == 'reg_number':
162            reg_number = row['reg_number']
163            cat = queryUtility(ICatalog, name='applicants_catalog')
164            results = list(
165                cat.searchResults(reg_number=(reg_number, reg_number)))
166            if results:
167                result = results[0].__parent__
168        elif self.getLocator(row) == 'applicant_id':
169            applicant_id = row['applicant_id']
170            cat = queryUtility(ICatalog, name='applicants_catalog')
171            results = list(
172                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
173            if results:
174                result = results[0].__parent__
175        return result
176
177    def parentsExist(self, row, site):
178        return self.getParent(row, site) is not None
179
180    def getEntry(self, row, site):
181        if self.getLocator(row) == 'container_code':
182            if row.get('application_number', None) not in (IGNORE_MARKER, None):
183                if not self.parentsExist(row, site):
184                    return None
185                parent = self.getParent(row, site)
186                return parent.get(row['application_number'])
187            return None
188        if self.getLocator(row) == 'applicant_id':
189            applicant_id = row['applicant_id']
190            cat = queryUtility(ICatalog, name='applicants_catalog')
191            results = list(
192                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
193            if results:
194                return results[0]
195        if self.getLocator(row) == 'reg_number':
196            reg_number = row['reg_number']
197            cat = queryUtility(ICatalog, name='applicants_catalog')
198            results = list(
199                cat.searchResults(reg_number=(reg_number, reg_number)))
200            if results:
201                return results[0]
202        return None
203
204    def entryExists(self, row, site):
205        return self.getEntry(row, site) is not None
206
207    def addEntry(self, obj, row, site):
208        parent = self.getParent(row, site)
209        parent.addApplicant(obj)
210        #parent.__parent__.logger.info(
211        #    'Applicant imported: %s' % obj.applicant_id)
212        history = IObjectHistory(obj)
213        history.addMessage(_('Application record imported'))
214        return
215
216    def delEntry(self, row, site):
217        applicant = self.getEntry(row, site)
218        if applicant is not None:
219            parent = applicant.__parent__
220            del parent[applicant.application_number]
221            #parent.__parent__.logger.info(
222            #    'Applicant removed: %s' % applicant.applicant_id)
223        pass
224
225    def updateEntry(self, obj, row, site, filename):
226        """Update obj to the values given in row.
227        """
228        items_changed = ''
229        # Remove application_number from row if empty
230        if 'application_number' in row and row['application_number'] in (
231            None, IGNORE_MARKER):
232            row.pop('application_number')
233
234        # Update applicant_id fom application_number and container code
235        # if application_number is given
236        if 'application_number' in row:
237            obj.applicant_id = u'%s_%s' % (
238                row['container_code'], row['application_number'])
239            items_changed += ('%s=%s, ' % ('applicant_id', obj.applicant_id))
240            row.pop('application_number')
241
242        # Update password
243        if 'password' in row:
244            passwd = row.get('password', IGNORE_MARKER)
245            if passwd not in ('', IGNORE_MARKER):
246                if passwd.startswith('{SSHA}'):
247                    # already encrypted password
248                    obj.password = passwd
249                else:
250                    # not yet encrypted password
251                    IUserAccount(obj).setPassword(passwd)
252                items_changed += ('%s=%s, ' % ('password', passwd))
253            row.pop('password')
254
255        # Update registration state
256        if 'state' in row:
257            state = row.get('state', IGNORE_MARKER)
258            if state not in (IGNORE_MARKER, ''):
259                IWorkflowState(obj).setState(state)
260                msg = _("State '${a}' set", mapping = {'a':state})
261                history = IObjectHistory(obj)
262                history.addMessage(msg)
263                items_changed += ('%s=%s, ' % ('state', state))
264            row.pop('state')
265
266        # apply other values...
267        items_changed += super(ApplicantProcessor, self).updateEntry(
268            obj, row, site, filename)
269
270        # Log actions...
271        parent = self.getParent(row, site)
272        if self.getLocator(row) == 'container_code':
273            parent.__parent__.logger.info(
274                '%s - %s - imported: %s' % (self.name, filename, items_changed))
275        else:
276            parent.__parent__.logger.info(
277                '%s - %s - updated: %s' % (self.name, filename, items_changed))
278        return items_changed
279
280    def getMapping(self, path, headerfields, mode):
281        """Get a mapping from CSV file headerfields to actually used fieldnames.
282        """
283        result = dict()
284        reader = csv.reader(open(path, 'rb'))
285        raw_header = reader.next()
286        for num, field in enumerate(headerfields):
287            if field not in ['applicant_id', 'reg_number'] and mode == 'remove':
288                continue
289            if field == u'--IGNORE--':
290                # Skip ignored columns in failed and finished data files.
291                continue
292            result[raw_header[num]] = field
293        return result
294
295    def checkConversion(self, row, mode='create'):
296        """Validates all values in row.
297        """
298        iface = self.iface
299        if self.getLocator(row) == 'reg_number' or mode == 'remove':
300            iface = self.iface_byregnumber
301        converter = IObjectConverter(iface)
302        errs, inv_errs, conv_dict =  converter.fromStringDict(
303            row, self.factory_name, mode=mode)
304        cert = conv_dict.get('course1', None)
305        if cert is not None and (mode in ('create', 'update')):
306            # course1 application category must match container's.
307            site = grok.getSite()
308            parent = self.getParent(row, site)
309            if parent is None:
310                errs.append(('container', 'not found'))
311            elif cert.application_category != parent.application_category:
312                errs.append(('course1', 'wrong application category'))
313        if 'state' in row and \
314            not row['state'] in IMPORTABLE_STATES:
315            if row['state'] not in (IGNORE_MARKER, ''):
316                errs.append(('state','not allowed'))
317            else:
318                # state is an attribute of Applicant and must not
319                # be changed if empty
320                conv_dict['state'] = IGNORE_MARKER
321        application_number = row.get('application_number', None)
322        if application_number in (IGNORE_MARKER, ''):
323                conv_dict['application_number'] = IGNORE_MARKER
324        return errs, inv_errs, conv_dict
325
326    def checkUpdateRequirements(self, obj, row, site):
327        """Checks requirements the object must fulfill when being updated.
328
329        This method is not used in case of deleting or adding objects.
330
331        Returns error messages as strings in case of requirement
332        problems.
333        """
334        if obj.state == CREATED:
335            return 'Applicant is blocked.'
336        return None
Note: See TracBrowser for help on using the repository browser.