Ignore:
Timestamp:
28 Sep 2011, 17:08:05 (13 years ago)
Author:
Henrik Bettermann
Message:

Skip ignored columns in failed and finished data files.

In the finished data file we now clearly see which fields have been imported. In the pending data file (failed data file) ignored columns are omitted.

We could think about saving the original import file elsewhere.

Location:
main/waeup.sirp/trunk/src/waeup/sirp/utils
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

    r6276 r6824  
    102102    def applyMapping(self, row, mapping):
    103103        """Apply mapping to a row of CSV data.
     104
    104105        """
    105106        result = dict()
    106107        for key, replacement in mapping.items():
     108            if replacement == u'--IGNORE--':
     109                # Skip ignored columns in failed and finished data files.
     110                continue
    107111            result[replacement] = row[key]
    108112        return result
    109113
    110114    def getMapping(self, path, headerfields, mode):
    111         """Get a mapping from CSV file headerfields to actually used
    112            fieldnames.
     115        """Get a mapping from CSV file headerfields to actually used fieldnames.
     116
    113117        """
    114118        result = dict()
     
    117121        for num, field in enumerate(headerfields):
    118122            if field not in self.location_fields and mode == 'remove':
    119                 # Ignore non-location fields when removing...
    120                 field = '--IGNORE--'
     123                # Skip non-location fields when removing.
     124                continue
     125            if field == u'--IGNORE--':
     126                # Skip ignored columns in failed and finished data files.
     127                continue
    121128            result[raw_header[num]] = field
    122129        return result
     
    250257                conv_warnings = self.stringFromErrs(errs, inv_errs)
    251258                self.writeFailedRow(
    252                     failed_writer, raw_row, conv_warnings)
     259                    failed_writer, string_row, conv_warnings)
    253260                continue
    254261            row.update(conv_dict)
     
    258265                    num_warns += 1
    259266                    self.writeFailedRow(
    260                         failed_writer, raw_row,
     267                        failed_writer, string_row,
    261268                        "Not all parents do exist yet. Skipping")
    262269                    continue
     
    264271                    num_warns += 1
    265272                    self.writeFailedRow(
    266                         failed_writer, raw_row,
     273                        failed_writer, string_row,
    267274                        "This object already exists in the same container. Skipping.")
    268275                    continue
     
    275282                    num_warns += 1
    276283                    self.writeFailedRow(
    277                         failed_writer, raw_row,
     284                        failed_writer, string_row,
    278285                        "%s Skipping." % error.message)
    279286                    continue
     
    282289                    num_warns += 1
    283290                    self.writeFailedRow(
    284                         failed_writer, raw_row,
     291                        failed_writer, string_row,
    285292                        "Cannot remove: no such entry.")
    286293                    continue
     
    291298                    num_warns += 1
    292299                    self.writeFailedRow(
    293                         failed_writer, raw_row,
     300                        failed_writer, string_row,
    294301                        "Cannot update: no such entry.")
    295302                    continue
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.txt

    r6741 r6824  
    352352
    353353If something goes wrong during processing, the respective --IGNORE--
    354 cols will be populated correctly in the resulting pending file:
     354cols won't be populated in the resulting pending file:
    355355
    356356    >>> result = processor.doImport('newcomers.csv', ['name', 'dinoports',
     
    361361
    362362    >>> print open(result[3], 'rb').read()
    363     --IGNORE--,name,--IGNORE--,dinoports,--ERRORS--
    364     Barney,Barneys Home,1,2,This object already exists in the same container. Skipping.
    365     Wilma,Wilmas Asylum,1,1,This object already exists in the same container. Skipping.
    366     Fred,Freds Dinoburgers,0,10,This object already exists in the same container. Skipping.
    367     Joey,Joeys Drive-in,0,110,This object already exists in the same container. Skipping.
    368 
    369 The first ignored column ('owner') provides different contents than
    370 the second one ('taxpayer').
     363    name,dinoports,--ERRORS--
     364    Barneys Home,2,This object already exists in the same container. Skipping.
     365    Wilmas Asylum,1,This object already exists in the same container. Skipping.
     366    Freds Dinoburgers,10,This object already exists in the same container. Skipping.
     367    Joeys Drive-in,110,This object already exists in the same container. Skipping.
     368
    371369
    372370Clean up:
Note: See TracChangeset for help on using the changeset viewer.