Changeset 8331 for main

Timestamp:

3 May 2012, 07:28:57 (13 years ago)

Author:

Henrik Bettermann

Message:

Let's do the applicant importer do what it's supposed to do.

Some catalog searches are performed twice during import of a row. This can only be improved by modifying doImport in the base class.

Location:

main/waeup.kofa/trunk/src/waeup/kofa/applicants

Files:

• batching.py (modified) (11 diffs)
• tests/sample_applicant_data_update.csv (modified) (1 diff)
• tests/sample_applicant_data_update2.csv (modified) (1 diff)
• tests/test_batching.py (modified) (4 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/applicants/batching.py

@@ -77 +77 @@
 class ApplicantProcessor(BatchProcessor):
     """A batch processor for IApplicant objects.
+
+    In create mode container_code is required. If application_number is given
+    an applicant with this number is created in the designated container.
+    If application_number is not given a random application_number is assigned.
+    applicant_id is being determined by the system and can't be imported.
+
+    In update or remove mode container_code and application_number columns
+    must not exist. The applicant object is solely searched by its applicant_id
+    or reg_number.
     """
     grok.implements(IBatchProcessor)
@@ -85 +94 @@
     name = u'Applicant Processor'
     iface = IApplicant
-    location_fields = ['container_code']
+    location_fields = ['']
     factory_name = 'waeup.Applicant'
 
@@ -93 +102 @@
     def available_fields(self):
         return sorted(list(set(
-            ['application_number','reg_number',
+            ['application_number',
             'container_code','state','password'] + getFields(
                 self.iface).keys())))
 
     def checkHeaders(self, headerfields, mode='create'):
-        if not 'reg_number' in headerfields and not 'application_number' \
-            in headerfields:
-            raise FatalCSVError(
-                "Need at least columns application_number or reg_number ")
+        cond1 = 'container_code' in headerfields
+        cond2 = 'application_number' in headerfields
+        cond3 = 'applicant_id' in headerfields
+        cond4 = 'reg_number' in headerfields
         if mode == 'create':
+            if not cond1:
+                raise FatalCSVError(
+                    "Need at least container_code column!")
+            if cond3:
+                raise FatalCSVError(
+                    "applicant_id can't be imported in create mode!")
             for field in self.required_fields:
                 if not field in headerfields:
@@ -108 +123 @@
                         "Need at least columns %s for import!" %
                         ', '.join(["'%s'" % x for x in self.required_fields]))
+        if mode in ('update', 'remove'):
+            if not cond3 and not cond4:
+                raise FatalCSVError(
+                    "Need at least column reg_number or applicant_id!")
+            if cond1 or cond2:
+                raise FatalCSVError(
+                    "container_code or application_number can't be imported " +
+                    "in update or remove mode!")
         # Check for fields to be ignored...
         not_ignored_fields = [x for x in headerfields
@@ -117 +140 @@
 
     def getLocator(self, row):
-        if row.get('application_number', None) not in (IGNORE_MARKER, None):
-            return 'application_number'
+        if row.get('container_code', None) not in (IGNORE_MARKER, None):
+            # create, update or remove
+            return 'container_code'
+        elif row.get('applicant_id', None) not in (IGNORE_MARKER, None):
+            # update or remove
+            return 'applicant_id'
         elif row.get('reg_number', None) not in (IGNORE_MARKER, None):
+            # update or remove
             return 'reg_number'
         else:
@@ -125 +153 @@
 
     def getParent(self, row, site):
-        if not 'applicants' in site.keys():
-            return False
-        return site['applicants'][row['container_code']]
+        if self.getLocator(row) == 'container_code':
+            return site['applicants'].get(row['container_code'], None)
+        if self.getLocator(row) == 'reg_number':
+            reg_number = row['reg_number']
+            cat = queryUtility(ICatalog, name='applicants_catalog')
+            results = list(
+                cat.searchResults(reg_number=(reg_number, reg_number)))
+            if results:
+                return results[0].__parent__
+        if self.getLocator(row) == 'applicant_id':
+            applicant_id = row['applicant_id']
+            cat = queryUtility(ICatalog, name='applicants_catalog')
+            results = list(
+                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
+            if results:
+                return results[0].__parent__
+        return None
 
     def parentsExist(self, row, site):
@@ -133 +175 @@
 
     def getEntry(self, row, site):
-        if not self.parentsExist(row, site):
+        if self.getLocator(row) == 'container_code':
+            if row.get('application_number', None) not in (IGNORE_MARKER, None):
+                if not self.parentsExist(row, site):
+                    return None
+                parent = self.getParent(row, site)
+                return parent.get(row['application_number'])
             return None
-        parent = self.getParent(row, site)
-        if self.getLocator(row) == 'application_number':
-            if row['application_number'] in parent:
-                applicant = parent[row['application_number']]
-                return applicant
-        elif self.getLocator(row) == 'reg_number':
+        if self.getLocator(row) == 'applicant_id':
+            applicant_id = row['applicant_id']
+            cat = queryUtility(ICatalog, name='applicants_catalog')
+            results = list(
+                cat.searchResults(applicant_id=(applicant_id, applicant_id)))
+            if results:
+                return results[0]
+        if self.getLocator(row) == 'reg_number':
             reg_number = row['reg_number']
             cat = queryUtility(ICatalog, name='applicants_catalog')
@@ -167 +216 @@
         applicant = self.getEntry(row, site)
         if applicant is not None:
-            parent = self.getParent(row, site)
+            parent = applicant.__parent__
             del parent[applicant.application_number]
         pass
@@ -227 +276 @@
         raw_header = reader.next()
         for num, field in enumerate(headerfields):
-            if field not in ['container_code',
-                'application_number', 'reg_number'] and mode == 'remove':
+            if field not in ['applicant_id', 'reg_number'] and mode == 'remove':
                 continue
             if field == u'--IGNORE--':
@@ -239 +287 @@
         """Validates all values in row.
         """
-        if mode in ['update', 'remove']:
-            if self.getLocator(row) == 'reg_number':
-                iface = IApplicantUpdateByRegNo
-        else:
-            iface = self.iface
+        iface = self.iface
+        if self.getLocator(row) == 'reg_number' or mode == 'remove':
+            iface = IApplicantUpdateByRegNo
         converter = IObjectConverter(iface)
         errs, inv_errs, conv_dict =  converter.fromStringDict(
@@ -255 +301 @@
                 # be changed if empty
                 conv_dict['state'] = IGNORE_MARKER
-        application_number = row.get('application_number', IGNORE_MARKER)
+        application_number = row.get('application_number', None)
         if application_number in (IGNORE_MARKER, ''):
                 conv_dict['application_number'] = IGNORE_MARKER

main/waeup.kofa/trunk/src/waeup/kofa/applicants/tests/sample_applicant_data_update.csv

@@ -1 @@
-container_code,reg_number,firstname,middlename,state
-dp2011,1001,Aaren,,
-dp2011,1002,Alfons,,admitted
-dp2011,1003,Abraham,,
+applicant_id,reg_number,firstname,middlename,state
+,1001,Aaren,,
+dp2011_2345,6666,Alfons,,admitted
+,1003,Abraham,,

main/waeup.kofa/trunk/src/waeup/kofa/applicants/tests/sample_applicant_data_update2.csv

@@ -1 @@
-container_code,reg_number,firstname,middlename,application_number
-dp2011,1001,Aaren,XXX,
-dp2011,1002,Alfons,XXX,
-dp2011,1003,Abraham,XXX,
+applicant_id,reg_number,firstname,middlename
+dp2011_1234,,Aaren,XXX
+,6666,Alfons,XXX
+,1003,Abraham,XXX

main/waeup.kofa/trunk/src/waeup/kofa/applicants/tests/test_batching.py

@@ -24 +24 @@
 import tempfile
 import unittest
+import grok
 from zope.component.hooks import setSite, clearSite
 from zope.component import createObject
 from zope.interface.verify import verifyClass, verifyObject
+from zope.event import notify
 
 from waeup.kofa.app import University
@@ -68 +70 @@
 
 APPLICANT_HEADER_FIELDS_UPDATE = APPLICANT_SAMPLE_DATA_UPDATE.split(
+    '\n')[0].split(',')
+
+APPLICANT_HEADER_FIELDS_UPDATE2 = APPLICANT_SAMPLE_DATA_UPDATE2.split(
     '\n')[0].split(',')
 
@@ -310 +315 @@
         num, num_warns, fin_file, fail_file = self.processor.doImport(
             self.csv_file_update, APPLICANT_HEADER_FIELDS_UPDATE, 'update')
-        #content = open(fail_file).read()
         self.assertEqual(num_warns,0)
         # The middlename import value was None.
@@ -324 +328 @@
         # password of Finau is still unset
         self.assertEqual(IUserAccount(container['2345']).password,'')
-        shutil.rmtree(os.path.dirname(fin_file))
-        # Now we import another file which clears all middlename attributes.
-        num, num_warns, fin_file, fail_file = self.processor.doImport(
-            self.csv_file_update2, APPLICANT_HEADER_FIELDS_UPDATE, 'update')
+        # reg_number of Finau has changed
+        self.assertEqual(container['2345'].reg_number, '6666')
+        shutil.rmtree(os.path.dirname(fin_file))
+        # We have to inform the catalog that the reg_number has changed
+        notify(grok.ObjectModifiedEvent(container['2345']))
+        # Now we import another file which clears all middlename attributes
+        # and uses the new reg_number as locator
+        num, num_warns, fin_file, fail_file = self.processor.doImport(
+            self.csv_file_update2, APPLICANT_HEADER_FIELDS_UPDATE2, 'update')
+        #content = open(fail_file).read()
+        #print content
         self.assertEqual(num_warns,0)
         assert container['1234'].middlename is None