Changeset 6824 for main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

Timestamp:

28 Sep 2011, 17:08:05 (13 years ago)

Author:

Henrik Bettermann

Message:

Skip ignored columns in failed and finished data files.

In the finished data file we now clearly see which fields have been imported. In the pending data file (failed data file) ignored columns are omitted.

We could think about saving the original import file elsewhere.

File:

• main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py (modified) (8 diffs)

main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

@@ -102 +102 @@
     def applyMapping(self, row, mapping):
         """Apply mapping to a row of CSV data.
+
         """
         result = dict()
         for key, replacement in mapping.items():
+            if replacement == u'--IGNORE--':
+                # Skip ignored columns in failed and finished data files.
+                continue
             result[replacement] = row[key]
         return result
 
     def getMapping(self, path, headerfields, mode):
-        """Get a mapping from CSV file headerfields to actually used
-           fieldnames.
+        """Get a mapping from CSV file headerfields to actually used fieldnames.
+
         """
         result = dict()
@@ -117 +121 @@
         for num, field in enumerate(headerfields):
             if field not in self.location_fields and mode == 'remove':
-                # Ignore non-location fields when removing...
-                field = '--IGNORE--'
+                # Skip non-location fields when removing.
+                continue
+            if field == u'--IGNORE--':
+                # Skip ignored columns in failed and finished data files.
+                continue
             result[raw_header[num]] = field
         return result
@@ -250 +257 @@
                 conv_warnings = self.stringFromErrs(errs, inv_errs)
                 self.writeFailedRow(
-                    failed_writer, raw_row, conv_warnings)
+                    failed_writer, string_row, conv_warnings)
                 continue
             row.update(conv_dict)
@@ -258 +265 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Not all parents do exist yet. Skipping")
                     continue
@@ -264 +271 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "This object already exists in the same container. Skipping.")
                     continue
@@ -275 +282 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "%s Skipping." % error.message)
                     continue
@@ -282 +289 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Cannot remove: no such entry.")
                     continue
@@ -291 +298 @@
                     num_warns += 1
                     self.writeFailedRow(
-                        failed_writer, raw_row,
+                        failed_writer, string_row,
                         "Cannot update: no such entry.")
                     continue