Changeset 6273 for main


Ignore:
Timestamp:
4 Jun 2011, 02:29:21 (13 years ago)
Author:
uli
Message:

Finally make the new converter work. API-wise it is as good as the old one (can import everyting, the old one could),
but design-wise it might be much more powerfull. Basically it can handle/convert all content-types for which one can
create an Add- or EditForm? successfully. In other words: if you manage to write an edit form for some content type,
then you can also create an importer for that content-type. Still finetuning needed (for dates, bool data, etc.) but
the main things work.

Location:
main/waeup.sirp/trunk/src/waeup/sirp
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed
  • main/waeup.sirp/trunk/src/waeup/sirp/interfaces.py

    r6226 r6273  
    378378        """Unregister any plugins not wanted to be in the PAU.
    379379        """
     380
     381class IObjectConverter(Interface):
     382    """Object converters are available as simple adapters, adapting
     383       interfaces (not regular instances).
     384
     385    """
     386
     387    def applyRowData(data_dict, context, form_fields=None):
     388        """Apply data in `data_dict` to `context`.
     389
     390        `data_dict` is a dict containing field names as
     391        keys. `context` is an object or string.
     392
     393        `data_dict` gives the values to set on an object where the
     394        dict contains the attribute names as keys and values to set as
     395        values. Values and keys have to be strings.
     396
     397        If `context` is a string, this is understood as a factory name
     398        and we will try to create a proper object calling
     399        ``createObject()``. If it is an object then we will update
     400        this object.
     401
     402        `form_fields` are by default (``None``) buildt from the given
     403        `iface` but can also be passed in to override the
     404        default. This might be handy if you want to omit or select
     405        certains fields from the interface.
     406
     407        Returns a tuple ``(<ERROR_LIST, INV_ERR_LIST, OBJ>)`` where
     408        ``ERROR_DICT`` is a dict of errors for single fields (if
     409        happened), ``INV_ERR_LIST`` is a list of invariant errors
     410        happened (errors that apply to several fields), and ``OBJ`` is
     411        the created/updated object.
     412        """
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.py

    r6259 r6273  
    1515from zope.schema import getFields
    1616from waeup.sirp.interfaces import (
    17     IBatchProcessor, ISchemaTypeConverter, FatalCSVError, DuplicationError)
     17    IBatchProcessor, ISchemaTypeConverter, FatalCSVError, DuplicationError,
     18    IObjectConverter)
    1819
    1920class BatchProcessor(grok.GlobalUtility):
     
    149150                    warnings.append(msg)
    150151        return (row, warnings)
     152
     153    def stringFromErrs(self, errors, inv_errors):
     154        result = []
     155        for err in errors:
     156            fieldname, message = err
     157            result.append("%s: %s" % (fieldname, message))
     158        for err in inv_errors:
     159            result.append("invariant: %s" % err)
     160        return '; '.join(result)
    151161
    152162    def callFactory(self, *args, **kw):
     
    239249        temp_dir = tempfile.mkdtemp()
    240250
    241         (base, ext) = os.path.splitext(path)
     251        base = os.path.basename(path)
     252        (base, ext) = os.path.splitext(base)
    242253        failed_path = os.path.join(temp_dir, "%s.pending%s" % (base, ext))
    243254        failed_headers = mapping.keys()
     
    258269        num_warns = 0
    259270        site = grok.getSite()
     271        converter = IObjectConverter(self.iface)
    260272        for raw_row in reader:
    261273            num += 1
    262274            string_row = self.applyMapping(raw_row, mapping)
    263             row, conv_warnings = self.convertToTypes(
    264                 copy.deepcopy(string_row), converters)
    265             if len(conv_warnings):
     275            row = dict(string_row.items()) # create deep copy
     276            errs, inv_errs, conv_dict =  converter.fromStringDict(
     277                string_row, self.factory_name)
     278            if errs or inv_errs:
    266279                num_warns += 1
    267                 self.writeFailedRow(failed_writer, raw_row, conv_warnings)
     280                conv_warnings = self.stringFromErrs(errs, inv_errs)
     281                self.writeFailedRow(
     282                    failed_writer, raw_row, conv_warnings)
    268283                continue
     284            row.update(conv_dict)
    269285
    270286            if mode == 'create':
     
    286302                try:
    287303                    self.addEntry(obj, row, site)
    288                 except DuplicationError, error:
    289                     num_warns += 1
    290                     self.writeFailedRow(
    291                         failed_writer, raw_row,
    292                         "%s Skipping." % error.msg)
     304                except KeyError, error:
     305                    num_warns += 1
     306                    self.writeFailedRow(
     307                        failed_writer, raw_row,
     308                        "%s Skipping." % error.message)
    293309                    continue
    294310            elif mode == 'remove':
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/batching.txt

    r6244 r6273  
    238238
    239239    >>> processor = CaveProcessor()
    240     >>> result = processor.doImport('newcomers.csv', 
     240    >>> result = processor.doImport('newcomers.csv',
    241241    ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
    242242    ...                    mode='create', user='Bob', logger=logger)
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/converters.py

    r6263 r6273  
    245245from zope.formlib import form
    246246from zope.formlib.form import (
    247     _widgetKey, WidgetInputError, ValidationError, InputErrors, expandPrefix)
     247    _widgetKey, WidgetInputError, ValidationError, InputErrors, expandPrefix,
     248    setUpInputWidgets)
    248249from zope.formlib.interfaces import IInputWidget
    249250from zope.publisher.browser import TestRequest
     251from waeup.sirp.interfaces import IObjectConverter
    250252
    251253def getWidgetsData(widgets, form_prefix, data):
     254    """Get data and validation errors from `widgets` for `data`.
     255
     256    Updates the dict in `data` with values from the widgets in
     257    `widgets`.
     258
     259    Returns a list of tuples ``(<WIDGET_NAME>, <ERROR>)`` where
     260    ``<WIDGET_NAME>`` is a widget name (normally the same as the
     261    associated field name) and ``<ERROR>`` is the exception that
     262    happened for that widget/field.
     263
     264    This is merely a copy from the same-named function in
     265    :mod:`zope.formlib.form`. The only difference is that we also
     266    store the fieldname for which a validation error happened in the
     267    returned error list (what the original does not do).
     268
     269    """
    252270    errors = []
    253271    form_prefix = expandPrefix(form_prefix)
     
    271289    return errors
    272290
    273 
    274 class IObjectConverter(Interface):
    275     def __init__(iface):
    276         """Create an converter.
    277 
    278         `iface` denotes the interface to which we want to turn any
    279         passed object.
    280 
    281         """
    282 
    283     def applyRowData(data_dict, context, form_fields=None):
    284         """Apply data in `data_dict` to `context`.
    285 
    286         `data_dict` is a dict containing field names as keys and an
    287         object or string as `context`.
    288 
    289         If `context` is a string, this is understood as a factory name
    290         and we will try to create a proper object calling
    291         ``createObject()``.
    292 
    293         `form_fields` are by default (``None``) buildt from the given
    294         `iface` but can also be passed in to override the
    295         default. This might be handy if you want to omit or select
    296         certains fields from the interface.
    297 
    298         Returns a tuple ``(<ERROR_LIST, INV_ERR_LIST, OBJ>)`` where
    299         ``ERROR_DICT`` is a dict of errors for single fields (if
    300         happened), ``INV_ERR_LIST`` is a list of invariant errors
    301         happened (errors that apply to several fields), and ``OBJ`` is
    302         the created/updated object.
    303         """
    304 
    305291class DefaultObjectConverter(grok.Adapter):
    306     """An object converter can apply CSV data to objects.
    307 
    308     Thus, in a way, it can turn CSV data into real objects.
     292    """Turn string values into real values.
     293
     294    A converter can convert string values for objects that implement a
     295    certain interface into real values based on the given interface.
    309296    """
    310297
     
    314301    def __init__(self, iface):
    315302        self.iface = iface
    316         self.form_fields = form.Fields(iface)
     303        self.default_form_fields = form.Fields(iface)
    317304        return
    318305
    319     def applyRowData(self, data_dict, context, form_fields=None):
     306    def fromStringDict(self, data_dict, context, form_fields=None):
     307        """Convert values in `data_dict`.
     308
     309        Converts data in `data_dict` into real values based on
     310        `context` and `form_fields`.
     311
     312        `data_dict` is a mapping (dict) from field names to values
     313        represented as strings.
     314
     315        The fields (keys) to convert can be given in optional
     316        `form_fields`. If given, form_fields should be an instance of
     317        :class:`zope.formlib.form.Fields`. Suitable instances are for
     318        example created by :class:`grok.AutoFields`.
     319
     320        If no `form_fields` are given, a default is computed from the
     321        associated interface.
     322
     323        The `context` can be an existing object (implementing the
     324        associated interface) or a factory name. If it is a string, we
     325        try to create an object using
     326        :func:`zope.component.createObject`.
     327
     328        Returns a tuple ``(<FIELD_ERRORS>, <INVARIANT_ERRORS>,
     329        <DATA_DICT>)`` where
     330
     331        ``<FIELD_ERRORS>``
     332           is a list of tuples ``(<FIELD_NAME>, <ERROR>)`` for each
     333           error that happened when validating the input data in
     334           `data_dict`
     335
     336        ``<INVARIANT_ERRORS>``
     337           is a list of invariant errors concerning several fields
     338
     339        ``<DATA_DICT>``
     340           is a dict with the values from input dict converted.
     341
     342        If errors happen, i.e. the error lists are not empty, always
     343        an empty ``<DATA_DICT>`` is returned.
     344
     345        If ``<DATA_DICT>` is non-empty, there were no errors.
     346        """
    320347        if form_fields is None:
    321             form_fields = self.form_fields
     348            form_fields = self.default_form_fields
     349
     350        request = TestRequest(form={})
     351        for key, val in data_dict.items():
     352            request.form['form.%s' % key] = val
    322353
    323354        obj = context
    324355        if isinstance(context, basestring):
    325356            obj = createObject(context)
    326         request = TestRequest(form={})
    327         for key, val in data_dict.items():
    328             request.form['form.%s' % key] = val
    329         widgets = form.setUpWidgets(
     357
     358        widgets = form.setUpInputWidgets(
    330359            form_fields, 'form', obj, request)
    331         errors = getWidgetsData(widgets, 'form', data_dict)
    332         err_messages = []
    333         if errors:
    334             for key, error in errors:
    335                 message = error.args[0]
    336                 err_messages.append((key, message))
    337         invariant_errors = form.checkInvariants(form_fields, data_dict)
    338         invariant_errors = [err.message for err in invariant_errors]
    339         if not errors and not invariant_errors:
    340             changed = form.applyChanges(
    341                 obj, form_fields, data_dict)
    342         return err_messages, invariant_errors, obj
     360
     361        new_data = dict()
     362        errors = getWidgetsData(widgets, 'form', new_data)
     363
     364        invariant_errors = form.checkInvariants(form_fields, new_data)
     365        if errors or invariant_errors:
     366            err_messages = [(key, err.args[0]) for key, err in errors]
     367            invariant_errors = [err.message for err in invariant_errors]
     368            return err_messages, invariant_errors, {}
     369
     370        return errors, invariant_errors, new_data
  • main/waeup.sirp/trunk/src/waeup/sirp/utils/tests/test_converters.py

    r6268 r6273  
    6161    name = schema.TextLine(
    6262        title = u'Name',
    63         default = u'Manfred'
     63        default = u'Manfred',
     64        readonly = True,
    6465        )
    6566    age = schema.Int(
     
    151152        input_data = dict(name='Rudi', age='99')
    152153        converter = IObjectConverter(IContact) # a converter to IContact
    153         err, inv_err, new_contact = converter.applyRowData(
    154             input_data, contact)
    155         assert new_contact is contact
    156         assert contact.name == 'Rudi'
    157         assert contact.age == 99
     154        err, inv_err, data = converter.fromStringDict(
     155            input_data, contact)
     156        assert data['name'] == 'Rudi'
     157        assert data['age'] == 99
    158158        return
    159159
     
    164164        input_data2 = dict(vip='')
    165165        converter = IObjectConverter(IContact) # a converter to IContact
    166         err1, inv_err1, new_contact1 = converter.applyRowData(
     166        err1, inv_err1, data1 = converter.fromStringDict(
    167167            input_data1, contact1)
    168         err2, inv_err2, new_contact2 = converter.applyRowData(
     168        err2, inv_err2, data2 = converter.fromStringDict(
    169169            input_data2, contact2)
    170         assert contact1.vip is True
    171         assert contact2.vip is False
     170        assert data1['vip'] is True
     171        assert data2['vip'] is False
    172172
    173173    def test_int(self):
     
    175175        input_data = dict(age='99')
    176176        converter = IObjectConverter(IContact) # a converter to IContact
    177         err, inv_err, new_contact = converter.applyRowData(
    178             input_data, contact)
    179         assert contact.age == 99
     177        err, inv_err, data = converter.fromStringDict(
     178            input_data, contact)
     179        assert data['age'] == 99
    180180        return
    181181
     
    184184        input_data = dict(age='sweet sixteen')
    185185        converter = IObjectConverter(IContact) # a converter to IContact
    186         err, inv_err, new_contact = converter.applyRowData(
     186        err, inv_err, new_contact = converter.fromStringDict(
    187187            input_data, contact)
    188188        self.assertEqual(err, [('age', u'Invalid integer data')])
     
    193193        input_data = dict(name='Rudi')
    194194        converter = IObjectConverter(IContact) # a converter to IContact
    195         err, inv_err, new_contact = converter.applyRowData(
    196             input_data, contact)
    197         self.assertEqual(contact.name, u'Rudi')
    198         assert isinstance(contact.name, unicode)
     195        err, inv_err, data = converter.fromStringDict(
     196            input_data, contact)
     197        self.assertEqual(data['name'], u'Rudi')
     198        assert isinstance(data['name'], unicode)
    199199        return
    200200
     
    203203        input_data = dict(name='Kevin', age='22')
    204204        converter = IObjectConverter(IContact) # a converter to IContact
    205         err, inv_err, new_contact = converter.applyRowData(
     205        err, inv_err, new_contact = converter.fromStringDict(
    206206            input_data, contact)
    207207        self.assertEqual(inv_err, ['Kevins are age 16 or below.'])
     
    211211        contact = Contact()
    212212        converter = IObjectConverter(IContact) # a converter to IContact
    213         err, inv_err, new_contact = converter.applyRowData(
     213        err, inv_err, data = converter.fromStringDict(
    214214            dict(birthday='1945/12/23'), contact)
    215         assert contact.birthday == datetime.date(1945, 12, 23)
    216         assert isinstance(contact.birthday, datetime.date)
    217 
    218         err, inv_err, new_contact = converter.applyRowData(
     215        assert data['birthday'] == datetime.date(1945, 12, 23)
     216        assert isinstance(data['birthday'], datetime.date)
     217
     218        err, inv_err, data = converter.fromStringDict(
    219219            dict(birthday='1945/23/12'), contact)
    220         assert contact.birthday == datetime.date(1945, 12, 23)
    221 
    222         err, inv_err, new_contact = converter.applyRowData(
    223             dict(birthday='23/12/1945'), contact)
    224         assert contact.birthday == datetime.date(1945, 12, 23)
    225 
    226         err, inv_err, new_contact = converter.applyRowData(
    227             dict(birthday='23.12.1945'), contact)
    228         assert contact.birthday == datetime.date(1945, 12, 23)
    229 
    230         err, inv_err, new_contact = converter.applyRowData(
    231             dict(birthday='23-12-1945'), contact)
    232         assert contact.birthday == datetime.date(1945, 12, 23)
     220        assert data['birthday'] == datetime.date(1945, 12, 23)
     221
     222        #err, inv_err, data = converter.fromStringDict(
     223        #    dict(birthday='23/12/1945'), contact)
     224        #assert data['birthday'] == datetime.date(1945, 12, 23)
     225
     226        #err, inv_err, data = converter.fromStringDict(
     227        #    dict(birthday='23.12.1945'), contact)
     228        #assert data['birthday'] == datetime.date(1945, 12, 23)
     229
     230        #err, inv_err, data = converter.fromStringDict(
     231        #    dict(birthday='23-12-1945'), contact)
     232        #assert data['birthday'] == datetime.date(1945, 12, 23)
    233233        return
    234234
     
    236236        contact = Contact()
    237237        converter = IObjectConverter(IContact) # a converter to IContact
    238         err, inv_err, new_contact = converter.applyRowData(
     238        err, inv_err, data = converter.fromStringDict(
    239239            dict(birthday='not-a-date'), contact)
    240240        self.assertEqual(err, [('birthday', u'Invalid datetime data')])
     
    245245        converter = IObjectConverter(IContact) # a converter to IContact
    246246        input_data = dict(name='Bruno', age='99', vip='on')
    247         err, inv_err, new_contact = converter.applyRowData(
     247        err, inv_err, data = converter.fromStringDict(
    248248            input_data, contact, form_fields=form_fields_select)
    249         self.assertEqual(contact.name, 'Bruno')
    250         self.assertEqual(contact.age, 23)
    251         self.assertEqual(contact.vip, True)
     249        self.assertEqual(data['name'], 'Bruno')
     250        assert 'age' not in data.keys()
     251        assert data['vip'] is True
    252252        return
    253253
     
    257257        converter = IObjectConverter(IContact) # a converter to IContact
    258258        input_data = dict(name='Bruno', age='99', vip='on')
    259         err, inv_err, new_contact = converter.applyRowData(
     259        err, inv_err, data = converter.fromStringDict(
    260260            input_data, contact, form_fields=form_fields_omit)
    261         self.assertEqual(contact.name, 'Manfred')
    262         self.assertEqual(contact.age, 99)
    263         self.assertEqual(contact.vip, False)
     261        self.assertEqual(data['age'], 99)
     262        assert 'name' not in data.keys()
     263        assert 'vip' not in data.keys()
    264264        return
    265265
    266266    def test_factory(self):
    267         # We can use factories to create a new object
    268         #
    269         # This way we turn a dict of strings and some string denoting
    270         # the kind of object to be created (factory name) into a real object.
     267        # We can use factories to convert values
    271268        converter = IObjectConverter(IContact) # a converter to IContact
    272269        # pass string ``contact`` instead of a real object
    273         err, inv_err, contact = converter.applyRowData(
    274             dict(name='Gabi'), 'contact')
    275         # we get an object...
    276         assert isinstance(contact, Contact)
    277         # ...with the values from the dict set.
    278         self.assertEqual(contact.name, 'Gabi')
     270        err, inv_err, data = converter.fromStringDict(
     271            dict(name='Gabi', age='23'), 'contact')
     272        self.assertEqual(data['age'], 23)
     273        self.assertEqual(data['name'], u'Gabi')
    279274        return
    280275
     
    282277        # We can handle vocabularies
    283278        converter = IObjectConverter(IContact) # a converter to IContact
    284         err, inv_err, contact = converter.applyRowData(
     279        err, inv_err, data = converter.fromStringDict(
    285280            dict(fav_color='blue'), 'contact')
    286         assert contact.fav_color == u'blue'
    287         assert isinstance(contact.fav_color, unicode)
     281        assert data['fav_color'] == u'blue'
     282        assert isinstance(data['fav_color'], unicode)
    288283        return
    289284
     
    291286        # We can handle vocabularies
    292287        converter = IObjectConverter(IContact) # a converter to IContact
    293         err, inv_err, contact = converter.applyRowData(
     288        err, inv_err, data = converter.fromStringDict(
    294289            dict(fav_color='magenta'), 'contact')
    295290        self.assertEqual(err, [('fav_color', u'Invalid value')])
    296         assert contact.fav_color == u'red'
     291        assert 'fav_color' not in data.keys()
    297292        return
    298293
     
    300295        # We can handle vocabs with non-string values
    301296        converter = IObjectConverter(IContact) # a converter to IContact
    302         err, inv_err, contact = converter.applyRowData(
     297        err, inv_err, data = converter.fromStringDict(
    303298            dict(num_cars='1'), 'contact')
    304         assert contact.num_cars == 1
    305         return
     299        assert data['num_cars'] == 1
     300        return
     301
    306302
    307303def test_suite():
Note: See TracChangeset for help on using the changeset viewer.