Changeset 6824 for main

Timestamp:

28 Sep 2011, 17:08:05 (13 years ago)

Author:

Henrik Bettermann

Message:

Skip ignored columns in failed and finished data files.

In the finished data file we now clearly see which fields have been imported. In the pending data file (failed data file) ignored columns are omitted.

We could think about saving the original import file elsewhere.

Location:

main/waeup.sirp/trunk/src/waeup/sirp

Files:

• browser/batchprocessing.txt (modified) (3 diffs)
• utils/batching.py (modified) (8 diffs)
• utils/batching.txt (modified) (2 diffs)

main/waeup.sirp/trunk/src/waeup/sirp/browser/batchprocessing.txt

@@ -95 +95 @@
     ['Applicants Container Importer',
      'Student Importer',
+     'StudentStudyCourse Importer',
      'CertificateCourse Importer',
      'Certificate Importer',
@@ -254 +255 @@
 
     >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Course Importer', index=1).selected = True
+    >>> importerselect.getControl('Course Importer', index=2).selected = True
     >>> modeselect = browser.getControl(name='mode')
     >>> modeselect.getControl(value='create').selected = True
@@ -484 +485 @@
     >>> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv'
     >>> print open(pending_file).read()
-    title_prefix,--IGNORE--,code,title,--ERRORS--
+    title_prefix,code,title,--ERRORS--
     faculty,FAC1,Faculty 1,This object already exists in the same container. Skipping.
 
     >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv'
     >>> print open(finished_file).read()
-    title_prefix,--IGNORE--,code,title
+    title_prefix,code,title
     school,FAC4,Faculty 4
 

main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

@@ -102 +102 @@
     def applyMapping(self, row, mapping):
         """Apply mapping to a row of CSV data.
+
         """
         result = dict()
         for key, replacement in mapping.items():
+            if replacement == u'--IGNORE--':
+                # Skip ignored columns in failed and finished data files.
+                continue
             result[replacement] = row[key]
         return result
 
     def getMapping(self, path, headerfields, mode):
-        """Get a mapping from CSV file headerfields to actually used
-           fieldnames.
+        """Get a mapping from CSV file headerfields to actually used fieldnames.
+
         """
         result = dict()
@@ -117 +121 @@
         for num, field in enumerate(headerfields):
             if field not in self.location_fields and mode == 'remove':
-                # Ignore non-location fields when removing...
-                field = '--IGNORE--'
+                # Skip non-location fields when removing.
+                continue
+            if field == u'--IGNORE--':
+                # Skip ignored columns in failed and finished data files.
+                continue
             result[raw_header[num]] = field
         return result
@@ -250 +257 @@
                 conv_warnings = self.stringFromErrs(errs, inv_errs)
                 self.writeFailedRow(
-                    failed_writer, raw_row, conv_warnings)
+                    failed_writer, string_row, conv_warnings)
                 continue
             row.update(conv_dict)
@@ -258 +265 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Not all parents do exist yet. Skipping")
                     continue
@@ -264 +271 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "This object already exists in the same container. Skipping.")
                     continue
@@ -275 +282 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "%s Skipping." % error.message)
                     continue
@@ -282 +289 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Cannot remove: no such entry.")
                     continue
@@ -291 +298 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Cannot update: no such entry.")
                     continue

main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.txt

@@ -352 +352 @@
 
 If something goes wrong during processing, the respective --IGNORE--
-cols will be populated correctly in the resulting pending file:
+cols won't be populated  in the resulting pending file:
 
     >>> result = processor.doImport('newcomers.csv', ['name', 'dinoports',
@@ -361 +361 @@
 
     >>> print open(result[3], 'rb').read()
-    --IGNORE--,name,--IGNORE--,dinoports,--ERRORS--
-    Barney,Barneys Home,1,2,This object already exists in the same container. Skipping.
-    Wilma,Wilmas Asylum,1,1,This object already exists in the same container. Skipping.
-    Fred,Freds Dinoburgers,0,10,This object already exists in the same container. Skipping.
-    Joey,Joeys Drive-in,0,110,This object already exists in the same container. Skipping.
-
-The first ignored column ('owner') provides different contents than
-the second one ('taxpayer').
+    name,dinoports,--ERRORS--
+    Barneys Home,2,This object already exists in the same container. Skipping.
+    Wilmas Asylum,1,This object already exists in the same container. Skipping.
+    Freds Dinoburgers,10,This object already exists in the same container. Skipping.
+    Joeys Drive-in,110,This object already exists in the same container. Skipping.
+
 
 Clean up: