source: main/waeup.kofa/branches/henrik-transcript-workflow/src/waeup/kofa/applicants/batching.py @ 15127

Last change on this file since 15127 was 14804, checked in by Henrik Bettermann, 7 years ago

Allow ApplicantOnlinePaymentProcessor to import records without p_id column in create mode.
  • Property svn:keywords set to Id
File size: 18.7 KB
Line 
1## $Id: batching.py 14804 2017-08-18 06:43:53Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18"""Batch processing for applicants.
19"""
20import unicodecsv as csv # XXX: csv ops should move to dedicated module.
21import grok
22from time import time
23from zope.schema import getFields
24from zope.interface import Interface
25from zope.component import queryUtility, getUtility
26from hurry.workflow.interfaces import IWorkflowState
27from zope.catalog.interfaces import ICatalog
28from waeup.kofa.interfaces import (
29    IBatchProcessor, IObjectConverter, FatalCSVError, IGNORE_MARKER,
30    IObjectHistory, IUserAccount, DuplicationError)
31from waeup.kofa.interfaces import MessageFactory as _
32from waeup.kofa.payments.interfaces import IPayer
33from waeup.kofa.utils.batching import BatchProcessor
34from waeup.kofa.applicants.interfaces import (
35    IApplicantsContainer, IApplicant, IApplicantUpdateByRegNo,
36    IApplicantOnlinePayment)
37from waeup.kofa.applicants.workflow import  IMPORTABLE_STATES, CREATED
38
39class ApplicantsContainerProcessor(BatchProcessor):
40    """The Applicants Container Processor imports containers for applicants.
41    It does not import their content. There is nothing special about this
42    processor.
43    """
44    grok.implements(IBatchProcessor)
45    grok.provides(IBatchProcessor)
46    grok.context(Interface)
47    util_name = 'applicantscontainerprocessor'
48    grok.name(util_name)
49
50    name = _('ApplicantsContainer Processor')
51    mode = u'create'
52    iface = IApplicantsContainer
53
54    location_fields = ['code',]
55    factory_name = 'waeup.ApplicantsContainer'
56
57    def parentsExist(self, row, site):
58        return 'applicants' in site.keys()
59
60    def entryExists(self, row, site):
61        return row['code'] in site['applicants'].keys()
62
63    def getParent(self, row, site):
64        return site['applicants']
65
66    def getEntry(self, row, site):
67        if not self.entryExists(row, site):
68            return None
69        parent = self.getParent(row, site)
70        return parent.get(row['code'])
71
72    def addEntry(self, obj, row, site):
73        parent = self.getParent(row, site)
74        parent[row['code']] = obj
75        return
76
77    def delEntry(self, row, site):
78        parent = self.getParent(row, site)
79        del parent[row['code']]
80        return
81
82class ApplicantProcessor(BatchProcessor):
83    """The Applicant Processor imports application data (applicants).
84
85    In create mode `container_code` is required. If `application_number` is
86    given, an applicant with this number is created in the designated container.
87    If `application_number` is not given, a random `application_number` is
88    assigned. `applicant_id` is being determined by the system and can't be
89    imported.
90
91    In update or remove mode `container_code` and `application_number` columns
92    must not exist. The applicant object is solely localized by searching
93    the applicants catalog for `reg_number` or `applicant_id` .
94    """
95    grok.implements(IBatchProcessor)
96    grok.provides(IBatchProcessor)
97    grok.context(Interface)
98    util_name = 'applicantprocessor'
99    grok.name(util_name)
100    name = _('Applicant Processor')
101    iface = IApplicant
102    iface_byregnumber = IApplicantUpdateByRegNo
103    factory_name = 'waeup.Applicant'
104
105    mode = None
106
107    @property
108    def available_fields(self):
109        return sorted(list(set(
110            ['application_number',
111            'container_code','state','password'] + getFields(
112                self.iface).keys())))
113
114    def checkHeaders(self, headerfields, mode='create'):
115        cond1 = 'container_code' in headerfields
116        cond2 = 'application_number' in headerfields
117        cond3 = 'applicant_id' in headerfields
118        cond4 = 'reg_number' in headerfields
119        if mode == 'create':
120            if not cond1:
121                raise FatalCSVError(
122                    "Need at least container_code column!")
123            if cond3:
124                raise FatalCSVError(
125                    "applicant_id can't be imported in create mode!")
126            for field in self.required_fields:
127                if not field in headerfields:
128                    raise FatalCSVError(
129                        "Need at least columns %s for import!" %
130                        ', '.join(["'%s'" % x for x in self.required_fields]))
131        if mode in ('update', 'remove'):
132            if not cond3 and not cond4:
133                raise FatalCSVError(
134                    "Need at least column reg_number or applicant_id!")
135            if cond1 or cond2:
136                raise FatalCSVError(
137                    "container_code or application_number can't be imported " +
138                    "in update or remove mode!")
139        # Check for fields to be ignored...
140        not_ignored_fields = [x for x in headerfields
141                              if not x.startswith('--')]
142        if len(set(not_ignored_fields)) < len(not_ignored_fields):
143            raise FatalCSVError(
144                "Double headers: each column name may only appear once.")
145        return True
146
147    def getLocator(self, row):
148        if row.get('container_code', None) not in (IGNORE_MARKER, None):
149            # create mode
150            return 'container_code'
151        elif row.get('applicant_id', None) not in (IGNORE_MARKER, None):
152            # update or remove mode
153            return 'applicant_id'
154        elif row.get('reg_number', None) not in (IGNORE_MARKER, None):
155            # update or remove mode
156            return 'reg_number'
157        else:
158            return None
159
160    def getParent(self, row, site):
161        result = None
162        if self.getLocator(row) == 'container_code':
163            result = site['applicants'].get(row['container_code'], None)
164        elif self.getLocator(row) == 'reg_number':
165            reg_number = row['reg_number']
166            cat = queryUtility(ICatalog, name='applicants_catalog')
167            results = list(
168                cat.searchResults(reg_number=(reg_number, reg_number)))
169            if results:
170                result = results[0].__parent__
171        elif self.getLocator(row) == 'applicant_id':
172            applicant_id = row['applicant_id']
173            cat = queryUtility(ICatalog, name='applicants_catalog')
174            results = list(
175                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
176            if results:
177                result = results[0].__parent__
178        return result
179
180    def parentsExist(self, row, site):
181        return self.getParent(row, site) is not None
182
183    def getEntry(self, row, site):
184        if self.getLocator(row) == 'container_code':
185            if row.get('application_number', None) not in (IGNORE_MARKER, None):
186                if not self.parentsExist(row, site):
187                    return None
188                parent = self.getParent(row, site)
189                return parent.get(row['application_number'])
190            return None
191        if self.getLocator(row) == 'applicant_id':
192            applicant_id = row['applicant_id']
193            cat = queryUtility(ICatalog, name='applicants_catalog')
194            results = list(
195                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
196            if results:
197                return results[0]
198        if self.getLocator(row) == 'reg_number':
199            reg_number = row['reg_number']
200            cat = queryUtility(ICatalog, name='applicants_catalog')
201            results = list(
202                cat.searchResults(reg_number=(reg_number, reg_number)))
203            if results:
204                return results[0]
205        return None
206
207    def entryExists(self, row, site):
208        return self.getEntry(row, site) is not None
209
210    def addEntry(self, obj, row, site):
211        parent = self.getParent(row, site)
212        parent.addApplicant(obj)
213        #parent.__parent__.logger.info(
214        #    'Applicant imported: %s' % obj.applicant_id)
215        history = IObjectHistory(obj)
216        history.addMessage(_('Application record imported'))
217        return
218
219    def delEntry(self, row, site):
220        applicant = self.getEntry(row, site)
221        if applicant is not None:
222            parent = applicant.__parent__
223            del parent[applicant.application_number]
224            #parent.__parent__.logger.info(
225            #    'Applicant removed: %s' % applicant.applicant_id)
226        pass
227
228    def updateEntry(self, obj, row, site, filename):
229        """Update obj to the values given in row.
230        """
231        items_changed = ''
232        # Remove application_number from row if empty
233        if 'application_number' in row and row['application_number'] in (
234            None, IGNORE_MARKER):
235            row.pop('application_number')
236
237        # Update applicant_id fom application_number and container code
238        # if application_number is given
239        if 'application_number' in row:
240            obj.applicant_id = u'%s_%s' % (
241                row['container_code'], row['application_number'])
242            items_changed += ('%s=%s, ' % ('applicant_id', obj.applicant_id))
243            row.pop('application_number')
244
245        # Update password
246        if 'password' in row:
247            passwd = row.get('password', IGNORE_MARKER)
248            if passwd not in ('', IGNORE_MARKER):
249                if passwd.startswith('{SSHA}'):
250                    # already encrypted password
251                    obj.password = passwd
252                else:
253                    # not yet encrypted password
254                    IUserAccount(obj).setPassword(passwd)
255                items_changed += ('%s=%s, ' % ('password', passwd))
256            row.pop('password')
257
258        # Update registration state
259        if 'state' in row:
260            state = row.get('state', IGNORE_MARKER)
261            if state not in (IGNORE_MARKER, ''):
262                IWorkflowState(obj).setState(state)
263                msg = _("State '${a}' set", mapping = {'a':state})
264                history = IObjectHistory(obj)
265                history.addMessage(msg)
266                items_changed += ('%s=%s, ' % ('state', state))
267            row.pop('state')
268
269        # apply other values...
270        items_changed += super(ApplicantProcessor, self).updateEntry(
271            obj, row, site, filename)
272
273        # Log actions...
274        parent = self.getParent(row, site)
275        if self.getLocator(row) == 'container_code':
276            parent.__parent__.logger.info(
277                '%s - %s - imported: %s' % (self.name, filename, items_changed))
278        else:
279            parent.__parent__.logger.info(
280                '%s - %s - updated: %s' % (self.name, filename, items_changed))
281        return items_changed
282
283    def getMapping(self, path, headerfields, mode):
284        """Get a mapping from CSV file headerfields to actually used fieldnames.
285        """
286        result = dict()
287        reader = csv.reader(open(path, 'rb'))
288        raw_header = reader.next()
289        for num, field in enumerate(headerfields):
290            if field not in ['applicant_id', 'reg_number'] and mode == 'remove':
291                continue
292            if field == u'--IGNORE--':
293                # Skip ignored columns in failed and finished data files.
294                continue
295            result[raw_header[num]] = field
296        return result
297
298    def checkConversion(self, row, mode='create'):
299        """Validates all values in row.
300        """
301        iface = self.iface
302        if self.getLocator(row) == 'reg_number' or mode == 'remove':
303            iface = self.iface_byregnumber
304        converter = IObjectConverter(iface)
305        errs, inv_errs, conv_dict =  converter.fromStringDict(
306            row, self.factory_name, mode=mode)
307        cert = conv_dict.get('course1', None)
308        if cert is not None and (mode in ('create', 'update')):
309            # course1 application category must match container's.
310            site = grok.getSite()
311            parent = self.getParent(row, site)
312            if parent is None:
313                errs.append(('container', 'not found'))
314            elif cert.application_category != parent.application_category:
315                errs.append(('course1', 'wrong application category'))
316        if 'state' in row and \
317            not row['state'] in IMPORTABLE_STATES:
318            if row['state'] not in (IGNORE_MARKER, ''):
319                errs.append(('state','not allowed'))
320            else:
321                # state is an attribute of Applicant and must not
322                # be changed if empty
323                conv_dict['state'] = IGNORE_MARKER
324        application_number = row.get('application_number', None)
325        if application_number in (IGNORE_MARKER, ''):
326                conv_dict['application_number'] = IGNORE_MARKER
327        return errs, inv_errs, conv_dict
328
329    def checkUpdateRequirements(self, obj, row, site):
330        """Checks requirements the object must fulfill when being updated.
331
332        This method is not used in case of deleting or adding objects.
333
334        Returns error messages as strings in case of requirement
335        problems.
336        """
337        if obj.state == CREATED:
338            return 'Applicant is blocked.'
339        return None
340
341class ApplicantOnlinePaymentProcessor(BatchProcessor):
342    """The Applicant Online Payment Processor imports applicant payment tickets.
343    The tickets are located in the applicant container.
344
345    The `checkConversion` method checks the format of the payment identifier.
346    In create mode it does also ensures that same p_id does not exist
347    elsewhere. It must be portal-wide unique.
348
349    When adding a payment ticket, the `addEntry` method checks if a
350    payment has already been made. If so, a `DuplicationError` is raised.
351    """
352    grok.implements(IBatchProcessor)
353    grok.provides(IBatchProcessor)
354    grok.context(Interface)
355    util_name = 'applicantpaymentprocessor'
356    grok.name(util_name)
357
358    name = _('ApplicantOnlinePayment Processor')
359    iface = IApplicantOnlinePayment
360    factory_name = 'waeup.ApplicantOnlinePayment'
361
362    location_fields = ['applicant_id',]
363
364    @property
365    def available_fields(self):
366        af = sorted(list(set(
367            self.location_fields + getFields(self.iface).keys())) +
368            ['p_id',])
369        af.remove('display_item')
370        return af
371
372    def checkHeaders(self, headerfields, mode='ignore'):
373        super(ApplicantOnlinePaymentProcessor, self).checkHeaders(headerfields)
374        if mode in ('update', 'remove') and not 'p_id' in headerfields:
375            raise FatalCSVError(
376                "Need p_id for import in update and remove modes!")
377        return True
378
379    def parentsExist(self, row, site):
380        return self.getParent(row, site) is not None
381
382    def getParent(self, row, site):
383        applicant_id = row['applicant_id']
384        cat = queryUtility(ICatalog, name='applicants_catalog')
385        results = list(
386            cat.searchResults(applicant_id=(applicant_id, applicant_id)))
387        if results:
388            return results[0]
389        return None
390
391    def getEntry(self, row, site):
392        applicant = self.getParent(row, site)
393        if applicant is None:
394            return None
395        p_id = row.get('p_id', None)
396        if p_id in (None, IGNORE_MARKER):
397            return None
398        # We can use the hash symbol at the end of p_id in import files
399        # to avoid annoying automatic number transformation
400        # by Excel or Calc
401        p_id = p_id.strip('#')
402        entry = applicant.get(p_id)
403        return entry
404
405    def entryExists(self, row, site):
406        return self.getEntry(row, site) is not None
407
408    def updateEntry(self, obj, row, site, filename):
409        """Update obj to the values given in row.
410        """
411        items_changed = super(ApplicantOnlinePaymentProcessor, self).updateEntry(
412            obj, row, site, filename)
413        applicant = self.getParent(row, site)
414        applicant.__parent__.__parent__.logger.info(
415            '%s - %s - %s - updated: %s'
416            % (self.name, filename, applicant.applicant_id, items_changed))
417        return
418
419    def samePaymentMade(self, applicant, category):
420        for key in applicant.keys():
421            ticket = applicant[key]
422            if ticket.p_state == 'paid' and\
423                ticket.p_category == category:
424                  return True
425        return False
426
427    def addEntry(self, obj, row, site):
428        applicant = self.getParent(row, site)
429        p_id = row['p_id'].strip('#')
430        if self.samePaymentMade(applicant, obj.p_category):
431            applicant.__parent__.__parent__.logger.info(
432                '%s - %s - previous update cancelled'
433                % (self.name, applicant.applicant_id))
434            raise DuplicationError('Payment has already been made.')
435        applicant[p_id] = obj
436        return
437
438    def delEntry(self, row, site):
439        payment = self.getEntry(row, site)
440        applicant = self.getParent(row, site)
441        if payment is not None:
442            applicant.__parent__.__parent__.logger.info('%s - Payment ticket removed: %s'
443                % (applicant.applicant_id, payment.p_id))
444            del applicant[payment.p_id]
445        return
446
447    def checkConversion(self, row, mode='ignore'):
448        """Validates all values in row.
449        """
450        errs, inv_errs, conv_dict = super(
451            ApplicantOnlinePaymentProcessor, self).checkConversion(row, mode=mode)
452        # We have to check p_id.
453        p_id = row.get('p_id', None)
454        if mode == 'create' and p_id in (None, IGNORE_MARKER):
455            timestamp = ("%d" % int(time()*10000))[1:]
456            p_id = "p%s" % timestamp
457            conv_dict['p_id'] = p_id
458            return errs, inv_errs, conv_dict
459        elif p_id in (None, IGNORE_MARKER):
460            errs.append(('p_id','missing'))
461            return errs, inv_errs, conv_dict
462        else:
463            p_id = p_id.strip('#')
464            if not len(p_id) == 14:
465                errs.append(('p_id','invalid length'))
466                return errs, inv_errs, conv_dict
467        if mode == 'create':
468            cat = getUtility(ICatalog, name='payments_catalog')
469            results = list(cat.searchResults(p_id=(p_id, p_id)))
470            if len(results) > 0:
471                sids = [IPayer(payment).id for payment in results]
472                sids_string = ''
473                for id in sids:
474                    sids_string += '%s ' % id
475                errs.append(('p_id','p_id exists in %s' % sids_string))
476                return errs, inv_errs, conv_dict
477        return errs, inv_errs, conv_dict
Note: See TracBrowser for help on using the repository browser.