Changeset 6273 for main/waeup.sirp/trunk/src/waeup/sirp/utils

Timestamp:

4 Jun 2011, 02:29:21 (14 years ago)

Author:

uli

Message:

Finally make the new converter work. API-wise it is as good as the old one (can import everyting, the old one could),
but design-wise it might be much more powerfull. Basically it can handle/convert all content-types for which one can
create an Add- or EditForm? successfully. In other words: if you manage to write an edit form for some content type,
then you can also create an importer for that content-type. Still finetuning needed (for dates, bool data, etc.) but
the main things work.

Location:

main/waeup.sirp/trunk/src/waeup/sirp/utils

Files:

• batching.py (modified) (5 diffs)
• batching.txt (modified) (1 diff)
• converters.py (modified) (3 diffs)
• tests/test_batching.py (added)
• tests/test_converters.py (modified) (14 diffs)

main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

@@ -15 +15 @@
 from zope.schema import getFields
 from waeup.sirp.interfaces import (
-    IBatchProcessor, ISchemaTypeConverter, FatalCSVError, DuplicationError)
+    IBatchProcessor, ISchemaTypeConverter, FatalCSVError, DuplicationError,
+    IObjectConverter)
 
 class BatchProcessor(grok.GlobalUtility):
@@ -149 +150 @@
                     warnings.append(msg)
         return (row, warnings)
+
+    def stringFromErrs(self, errors, inv_errors):
+        result = []
+        for err in errors:
+            fieldname, message = err
+            result.append("%s: %s" % (fieldname, message))
+        for err in inv_errors:
+            result.append("invariant: %s" % err)
+        return '; '.join(result)
 
     def callFactory(self, *args, **kw):
@@ -239 +249 @@
         temp_dir = tempfile.mkdtemp()
 
-        (base, ext) = os.path.splitext(path)
+        base = os.path.basename(path)
+        (base, ext) = os.path.splitext(base)
         failed_path = os.path.join(temp_dir, "%s.pending%s" % (base, ext))
         failed_headers = mapping.keys()
@@ -258 +269 @@
         num_warns = 0
         site = grok.getSite()
+        converter = IObjectConverter(self.iface)
         for raw_row in reader:
             num += 1
             string_row = self.applyMapping(raw_row, mapping)
-            row, conv_warnings = self.convertToTypes(
-                copy.deepcopy(string_row), converters)
-            if len(conv_warnings):
+            row = dict(string_row.items()) # create deep copy
+            errs, inv_errs, conv_dict =  converter.fromStringDict(
+                string_row, self.factory_name)
+            if errs or inv_errs:
                 num_warns += 1
-                self.writeFailedRow(failed_writer, raw_row, conv_warnings)
+                conv_warnings = self.stringFromErrs(errs, inv_errs)
+                self.writeFailedRow(
+                    failed_writer, raw_row, conv_warnings)
                 continue
+            row.update(conv_dict)
 
             if mode == 'create':
@@ -286 +302 @@
                 try:
                     self.addEntry(obj, row, site)
-                except DuplicationError, error:
-                    num_warns += 1
-                    self.writeFailedRow(
-                        failed_writer, raw_row,
-                        "%s Skipping." % error.msg)
+                except KeyError, error:
+                    num_warns += 1
+                    self.writeFailedRow(
+                        failed_writer, raw_row,
+                        "%s Skipping." % error.message)
                     continue
             elif mode == 'remove':

main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.txt

@@ -238 +238 @@
 
     >>> processor = CaveProcessor()
-    >>> result = processor.doImport('newcomers.csv', 
+    >>> result = processor.doImport('newcomers.csv',
     ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
     ...                    mode='create', user='Bob', logger=logger)

main/waeup.sirp/trunk/src/waeup/sirp/utils/converters.py

@@ -245 +245 @@
 from zope.formlib import form
 from zope.formlib.form import (
-    _widgetKey, WidgetInputError, ValidationError, InputErrors, expandPrefix)
+    _widgetKey, WidgetInputError, ValidationError, InputErrors, expandPrefix,
+    setUpInputWidgets)
 from zope.formlib.interfaces import IInputWidget
 from zope.publisher.browser import TestRequest
+from waeup.sirp.interfaces import IObjectConverter
 
 def getWidgetsData(widgets, form_prefix, data):
+    """Get data and validation errors from `widgets` for `data`.
+
+    Updates the dict in `data` with values from the widgets in
+    `widgets`.
+
+    Returns a list of tuples ``(<WIDGET_NAME>, <ERROR>)`` where
+    ``<WIDGET_NAME>`` is a widget name (normally the same as the
+    associated field name) and ``<ERROR>`` is the exception that
+    happened for that widget/field.
+
+    This is merely a copy from the same-named function in
+    :mod:`zope.formlib.form`. The only difference is that we also
+    store the fieldname for which a validation error happened in the
+    returned error list (what the original does not do).
+
+    """
     errors = []
     form_prefix = expandPrefix(form_prefix)
@@ -271 +289 @@
     return errors
 
-
-class IObjectConverter(Interface):
-    def __init__(iface):
-        """Create an converter.
-
-        `iface` denotes the interface to which we want to turn any
-        passed object.
-
-        """
-
-    def applyRowData(data_dict, context, form_fields=None):
-        """Apply data in `data_dict` to `context`.
-
-        `data_dict` is a dict containing field names as keys and an
-        object or string as `context`.
-
-        If `context` is a string, this is understood as a factory name
-        and we will try to create a proper object calling
-        ``createObject()``.
-
-        `form_fields` are by default (``None``) buildt from the given
-        `iface` but can also be passed in to override the
-        default. This might be handy if you want to omit or select
-        certains fields from the interface.
-
-        Returns a tuple ``(<ERROR_LIST, INV_ERR_LIST, OBJ>)`` where
-        ``ERROR_DICT`` is a dict of errors for single fields (if
-        happened), ``INV_ERR_LIST`` is a list of invariant errors
-        happened (errors that apply to several fields), and ``OBJ`` is
-        the created/updated object.
-        """
-
 class DefaultObjectConverter(grok.Adapter):
-    """An object converter can apply CSV data to objects.
-
-    Thus, in a way, it can turn CSV data into real objects.
+    """Turn string values into real values.
+
+    A converter can convert string values for objects that implement a
+    certain interface into real values based on the given interface.
     """
 
@@ -314 +301 @@
     def __init__(self, iface):
         self.iface = iface
-        self.form_fields = form.Fields(iface)
+        self.default_form_fields = form.Fields(iface)
         return
 
-    def applyRowData(self, data_dict, context, form_fields=None):
+    def fromStringDict(self, data_dict, context, form_fields=None):
+        """Convert values in `data_dict`.
+
+        Converts data in `data_dict` into real values based on
+        `context` and `form_fields`.
+
+        `data_dict` is a mapping (dict) from field names to values
+        represented as strings.
+
+        The fields (keys) to convert can be given in optional
+        `form_fields`. If given, form_fields should be an instance of
+        :class:`zope.formlib.form.Fields`. Suitable instances are for
+        example created by :class:`grok.AutoFields`.
+
+        If no `form_fields` are given, a default is computed from the
+        associated interface.
+
+        The `context` can be an existing object (implementing the
+        associated interface) or a factory name. If it is a string, we
+        try to create an object using
+        :func:`zope.component.createObject`.
+
+        Returns a tuple ``(<FIELD_ERRORS>, <INVARIANT_ERRORS>,
+        <DATA_DICT>)`` where
+
+        ``<FIELD_ERRORS>``
+           is a list of tuples ``(<FIELD_NAME>, <ERROR>)`` for each
+           error that happened when validating the input data in
+           `data_dict`
+
+        ``<INVARIANT_ERRORS>``
+           is a list of invariant errors concerning several fields
+
+        ``<DATA_DICT>``
+           is a dict with the values from input dict converted.
+
+        If errors happen, i.e. the error lists are not empty, always
+        an empty ``<DATA_DICT>`` is returned.
+
+        If ``<DATA_DICT>` is non-empty, there were no errors.
+        """
         if form_fields is None:
-            form_fields = self.form_fields
+            form_fields = self.default_form_fields
+
+        request = TestRequest(form={})
+        for key, val in data_dict.items():
+            request.form['form.%s' % key] = val
 
         obj = context
         if isinstance(context, basestring):
             obj = createObject(context)
-        request = TestRequest(form={})
-        for key, val in data_dict.items():
-            request.form['form.%s' % key] = val
-        widgets = form.setUpWidgets(
+
+        widgets = form.setUpInputWidgets(
             form_fields, 'form', obj, request)
-        errors = getWidgetsData(widgets, 'form', data_dict)
-        err_messages = []
-        if errors:
-            for key, error in errors:
-                message = error.args[0]
-                err_messages.append((key, message))
-        invariant_errors = form.checkInvariants(form_fields, data_dict)
-        invariant_errors = [err.message for err in invariant_errors]
-        if not errors and not invariant_errors:
-            changed = form.applyChanges(
-                obj, form_fields, data_dict)
-        return err_messages, invariant_errors, obj
+
+        new_data = dict()
+        errors = getWidgetsData(widgets, 'form', new_data)
+
+        invariant_errors = form.checkInvariants(form_fields, new_data)
+        if errors or invariant_errors:
+            err_messages = [(key, err.args[0]) for key, err in errors]
+            invariant_errors = [err.message for err in invariant_errors]
+            return err_messages, invariant_errors, {}
+
+        return errors, invariant_errors, new_data

main/waeup.sirp/trunk/src/waeup/sirp/utils/tests/test_converters.py

@@ -61 +61 @@
     name = schema.TextLine(
         title = u'Name',
-        default = u'Manfred'
+        default = u'Manfred',
+        readonly = True,
         )
     age = schema.Int(
@@ -151 +152 @@
         input_data = dict(name='Rudi', age='99')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
-            input_data, contact)
-        assert new_contact is contact
-        assert contact.name == 'Rudi'
-        assert contact.age == 99
+        err, inv_err, data = converter.fromStringDict(
+            input_data, contact)
+        assert data['name'] == 'Rudi'
+        assert data['age'] == 99
         return
 
@@ -164 +164 @@
         input_data2 = dict(vip='')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err1, inv_err1, new_contact1 = converter.applyRowData(
+        err1, inv_err1, data1 = converter.fromStringDict(
             input_data1, contact1)
-        err2, inv_err2, new_contact2 = converter.applyRowData(
+        err2, inv_err2, data2 = converter.fromStringDict(
             input_data2, contact2)
-        assert contact1.vip is True
-        assert contact2.vip is False
+        assert data1['vip'] is True
+        assert data2['vip'] is False
 
     def test_int(self):
@@ -175 +175 @@
         input_data = dict(age='99')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
-            input_data, contact)
-        assert contact.age == 99
+        err, inv_err, data = converter.fromStringDict(
+            input_data, contact)
+        assert data['age'] == 99
         return
 
@@ -184 +184 @@
         input_data = dict(age='sweet sixteen')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, new_contact = converter.fromStringDict(
             input_data, contact)
         self.assertEqual(err, [('age', u'Invalid integer data')])
@@ -193 +193 @@
         input_data = dict(name='Rudi')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
-            input_data, contact)
-        self.assertEqual(contact.name, u'Rudi')
-        assert isinstance(contact.name, unicode)
+        err, inv_err, data = converter.fromStringDict(
+            input_data, contact)
+        self.assertEqual(data['name'], u'Rudi')
+        assert isinstance(data['name'], unicode)
         return
 
@@ -203 +203 @@
         input_data = dict(name='Kevin', age='22')
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, new_contact = converter.fromStringDict(
             input_data, contact)
         self.assertEqual(inv_err, ['Kevins are age 16 or below.'])
@@ -211 +211 @@
         contact = Contact()
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             dict(birthday='1945/12/23'), contact)
-        assert contact.birthday == datetime.date(1945, 12, 23)
-        assert isinstance(contact.birthday, datetime.date)
-
-        err, inv_err, new_contact = converter.applyRowData(
+        assert data['birthday'] == datetime.date(1945, 12, 23)
+        assert isinstance(data['birthday'], datetime.date)
+
+        err, inv_err, data = converter.fromStringDict(
             dict(birthday='1945/23/12'), contact)
-        assert contact.birthday == datetime.date(1945, 12, 23)
-
-        err, inv_err, new_contact = converter.applyRowData(
-            dict(birthday='23/12/1945'), contact)
-        assert contact.birthday == datetime.date(1945, 12, 23)
-
-        err, inv_err, new_contact = converter.applyRowData(
-            dict(birthday='23.12.1945'), contact)
-        assert contact.birthday == datetime.date(1945, 12, 23)
-
-        err, inv_err, new_contact = converter.applyRowData(
-            dict(birthday='23-12-1945'), contact)
-        assert contact.birthday == datetime.date(1945, 12, 23)
+        assert data['birthday'] == datetime.date(1945, 12, 23)
+
+        #err, inv_err, data = converter.fromStringDict(
+        #    dict(birthday='23/12/1945'), contact)
+        #assert data['birthday'] == datetime.date(1945, 12, 23)
+
+        #err, inv_err, data = converter.fromStringDict(
+        #    dict(birthday='23.12.1945'), contact)
+        #assert data['birthday'] == datetime.date(1945, 12, 23)
+
+        #err, inv_err, data = converter.fromStringDict(
+        #    dict(birthday='23-12-1945'), contact)
+        #assert data['birthday'] == datetime.date(1945, 12, 23)
         return
 
@@ -236 +236 @@
         contact = Contact()
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             dict(birthday='not-a-date'), contact)
         self.assertEqual(err, [('birthday', u'Invalid datetime data')])
@@ -245 +245 @@
         converter = IObjectConverter(IContact) # a converter to IContact
         input_data = dict(name='Bruno', age='99', vip='on')
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             input_data, contact, form_fields=form_fields_select)
-        self.assertEqual(contact.name, 'Bruno')
-        self.assertEqual(contact.age, 23)
-        self.assertEqual(contact.vip, True)
+        self.assertEqual(data['name'], 'Bruno')
+        assert 'age' not in data.keys()
+        assert data['vip'] is True
         return
 
@@ -257 +257 @@
         converter = IObjectConverter(IContact) # a converter to IContact
         input_data = dict(name='Bruno', age='99', vip='on')
-        err, inv_err, new_contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             input_data, contact, form_fields=form_fields_omit)
-        self.assertEqual(contact.name, 'Manfred')
-        self.assertEqual(contact.age, 99)
-        self.assertEqual(contact.vip, False)
+        self.assertEqual(data['age'], 99)
+        assert 'name' not in data.keys()
+        assert 'vip' not in data.keys()
         return
 
     def test_factory(self):
-        # We can use factories to create a new object
-        #
-        # This way we turn a dict of strings and some string denoting
-        # the kind of object to be created (factory name) into a real object.
+        # We can use factories to convert values
         converter = IObjectConverter(IContact) # a converter to IContact
         # pass string ``contact`` instead of a real object
-        err, inv_err, contact = converter.applyRowData(
-            dict(name='Gabi'), 'contact')
-        # we get an object...
-        assert isinstance(contact, Contact)
-        # ...with the values from the dict set.
-        self.assertEqual(contact.name, 'Gabi')
+        err, inv_err, data = converter.fromStringDict(
+            dict(name='Gabi', age='23'), 'contact')
+        self.assertEqual(data['age'], 23)
+        self.assertEqual(data['name'], u'Gabi')
         return
 
@@ -282 +277 @@
         # We can handle vocabularies
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             dict(fav_color='blue'), 'contact')
-        assert contact.fav_color == u'blue'
-        assert isinstance(contact.fav_color, unicode)
+        assert data['fav_color'] == u'blue'
+        assert isinstance(data['fav_color'], unicode)
         return
 
@@ -291 +286 @@
         # We can handle vocabularies
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             dict(fav_color='magenta'), 'contact')
         self.assertEqual(err, [('fav_color', u'Invalid value')])
-        assert contact.fav_color == u'red'
+        assert 'fav_color' not in data.keys()
         return
 
@@ -300 +295 @@
         # We can handle vocabs with non-string values
         converter = IObjectConverter(IContact) # a converter to IContact
-        err, inv_err, contact = converter.applyRowData(
+        err, inv_err, data = converter.fromStringDict(
             dict(num_cars='1'), 'contact')
-        assert contact.num_cars == 1
-        return
+        assert data['num_cars'] == 1
+        return
+
 
 def test_suite():