Index: /waeup/branches/ulif-rewrite/src/waeup/csvfile/README.txt
===================================================================
--- /waeup/branches/ulif-rewrite/src/waeup/csvfile/README.txt	(revision 4219)
+++ /waeup/branches/ulif-rewrite/src/waeup/csvfile/README.txt	(revision 4219)
@@ -0,0 +1,477 @@
+waeup.csvfile
+*************
+
+Generic support for handling CSV files.
+
+:Test-Layer: unit
+
+
+Basic example
+=============
+
+To initialize the whole framework we have to grok the `waeup` package
+first:
+
+    >>> import grok
+    >>> grok.testing.grok('waeup')
+
+Create a file:
+
+    >>> path = 'mycsvfile.csv'
+    >>> open(path, 'wb').write(
+    ... """col1,col2
+    ... dataitem1,dataitem2
+    ... item3,item4
+    ... """)
+
+A regular file path is difficult to handle in terms of a component
+framework. Therefore we get a wrapper for it, that makes a CSV file
+object out of a path string:
+
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> src = ICSVFile(path)
+    >>> src
+    <waeup.csvfile.csvfile.CSVFile object at 0x...>
+
+Create a receiver:
+
+    >>> from waeup.csvfile.interfaces import ICSVDataReceiver
+    >>> class Receiver(object):
+    ...   grok.implements(ICSVDataReceiver)
+    ...   def receive(self, data):
+    ...     print "RECEIVED: ", data
+
+    >>> recv = Receiver()
+
+Find a connector:
+
+    >>> from zope.component import getMultiAdapter
+    >>> from waeup.csvfile.interfaces import ICSVDataConnector
+    >>> conn = getMultiAdapter((src, recv), ICSVDataConnector)
+    Traceback (most recent call last):
+    ...
+    ComponentLookupError: ((<waeup.csvfile.csvfile.CSVFile object at 0x...>, 
+                            <Receiver object at 0x...>),
+        <InterfaceClass waeup.csvfile.interfaces.ICSVDataConnector>, u'')
+
+Okay, create a connector:
+
+    >>> class Connector1(grok.MultiAdapter):
+    ...   grok.adapts(ICSVFile, ICSVDataReceiver)
+    ...   grok.provides(ICSVDataConnector)
+    ...   def __init__(self, source, receiver):
+    ...     self.source = source
+    ...     self.receiver = receiver
+    ...   def doImport(self):
+    ...     self.receiver.receive(
+    ...        self.source.getData())
+
+    >>> grok.testing.grok_component('Connector1', Connector1)
+    True
+
+Try again...
+
+    >>> conn = getMultiAdapter((src, recv), ICSVDataConnector)
+    >>> conn
+    <Connector1 object at 0x...>
+
+    >>> conn.doImport()
+    RECEIVED: <generator object at 0x...>
+
+Clean up:
+
+    >>> import os
+    >>> os.unlink(path)
+
+
+CSV file wrappers
+=================
+
+CSV file wrappers can extract data from CSV files denoted by a path.
+
+We create a CSV file:
+
+    >>> path = 'mycsvfile.csv'
+    >>> open(path, 'wb').write(
+    ... """col1,col2
+    ... dataitem1,dataitem2
+    ... item3,item4
+    ... """)
+
+Now we get a CSV file wrapper for it. This is simply done by asking
+for an adapter to the path string:
+
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> wrapper = ICSVFile(path)
+    >>> wrapper
+    <waeup.csvfile.csvfile.CSVFile object at 0x...>
+
+This wrapper can return the CSV data as a sequence of dicts:
+
+    >>> wrapper.getData()
+    <generator object at 0x...>
+
+As we see, the single dicts (each representing a row) are returned as
+a generator. We can list them:
+
+    >>> list(wrapper.getData())
+    [{'col2': 'dataitem2', 'col1': 'dataitem1'}, 
+     {'col2': 'item4', 'col1': 'item3'}]
+
+We can get a list of headerfields found in the file:
+
+    >>> wrapper.getHeaderFields()
+    ['col1', 'col2']
+
+Getting a wrapper
+=================
+
+If we want to get a wrapper best suited for our purposes, we can also
+use the `getCSVFile` function:
+
+    >>> from waeup.csvfile.csvfile import getCSVFile
+    >>> wrapper = getCSVFile(path)
+    >>> wrapper
+    <waeup.csvfile.csvfile.CSVFile object at 0x...>
+
+As we currently have only one type of wrapper, we get this. Let's
+create another wrapper, that requires a column 'col1':
+
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> from waeup.csvfile import CSVFile
+    >>> class ICSVFileWithCol1(ICSVFile):
+    ...   """A CSV file that contains a 'col1' column.
+    ...   """
+
+    >>> class CSVFileWithCol1(CSVFile):
+    ...   required_fields = ['col1']
+    ...   grok.implements(ICSVFileWithCol1)
+    ...   grok.provides(ICSVFileWithCol1)
+
+We have to grok:
+
+    >>> grok.testing.grok_component('CSVFileWithCol1', CSVFileWithCol1)
+    True
+
+Now we can ask for a wrapper again, but this time we will get a
+CSVFileWithCol12 instance:
+
+    >>> getCSVFile(path)
+    <CSVFileWithCol1 object at 0x...>
+
+If we cannot get a wrapper at all, ``None`` is returned:
+
+    >>> getCSVFile('not-existent-file') is None
+    True
+
+
+How `getCSVFile()` decides which wrapper to use
+-----------------------------------------------
+
+Apparently, getCSVFile performes some magic: given a certain CSV file,
+it decides which one of all registered wrappers suits the file best.
+
+This decision is based on a score, which is computed as shown below
+for each registered wrapper.
+
+Before we can show this, we create some more CSV files.
+
+One file that does not contain valid CSV data:
+
+     >>> nocsvpath = 'nocsvfile.csv'
+     >>> open(nocsvpath, 'wb').write(
+     ... """blah blah blah.
+     ... blubb blubb.
+     ... """)
+
+One file that contains a 'col1' and a 'col3' column:
+
+    >>> path2 = 'mycsvfile2.csv'
+    >>> open(path2, 'wb').write(
+    ... """col1,col3
+    ... dataitem1b,dataitem2b
+    ... item3b,item4b
+    ... """)
+
+We create a wrapper that requires 'col1' and 'col2':
+
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> from waeup.csvfile import CSVFile
+    >>> class ICSVFile12(ICSVFile):
+    ...   """A CSV file that contains a 'special_col' column.
+    ...   """
+
+    >>> class CSVFile12(CSVFile):
+    ...   required_fields = ['col1', 'col2']
+    ...   grok.context(basestring)
+    ...   grok.implements(ICSVFile12)
+    ...   grok.provides(ICSVFile12)
+
+    >>> grok.testing.grok_component('CSVFile12',  CSVFile12)
+    True
+
+
+* If no instance of a certain wrapper can be created from the given
+  path (i.e. __init__ raises some kind of exception): score is -1:
+
+    >>> from waeup.csvfile.csvfile import getScore
+    >>> getScore('nonexistant', CSVFile)
+    -1
+
+* If a wrapper requires at least one header_field and the given file
+  does not provide all of the required fields: score is -1:
+
+    >>> getScore(path2, CSVFile12)
+    -1
+
+* If a wrapper requires no header fields at all
+  (i.e. `required_fields` equals empty list): score is 0 (zero):
+
+    >>> getScore(path, CSVFile)
+    0
+
+* If a wrapper requires at least one header_field and all header
+  fields do also appear in the file: score is number of required
+  fields.
+
+    >>> getScore(path, CSVFileWithCol1)
+    1
+
+    >>> getScore(path, CSVFile12)
+    2
+
+If several wrappers get the same score for a certain file, the result
+is not determined.
+
+
+How to build custom CSV file wrappers
+=====================================
+
+A typical CSV file wrapper can be built like this:
+
+    >>> import grok
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> from waeup.csvfile import CSVFile
+
+    >>> class ICustomCSVFile(ICSVFile):
+    ...   """A marker for custom CSV files."""
+
+    >>> class CustomCSVFile(CSVFile):
+    ...   required_fields = ['somecol', 'othercol']
+    ...   grok.implements(ICustomCSVFile)
+    ...   grok.provides(ICustomCSVFile)
+
+    >>> grok.testing.grok_component('CustomCSVFile',  CustomCSVFile)
+    True
+
+The special things here are:
+
+* Derive from `CSVFile`
+
+  `getCSVFile()` looks only for classes that are derived from
+  `CSVFile`. So if you want your wrapper to be found by this function,
+  derive from `CSVFile`.
+
+  As `CSVFile` is an adapter, also our custom wrapper will become one
+  (adapting strings):
+
+     >>> ICustomCSVFile(path)
+     Traceback (most recent call last):
+     ...
+     TypeError: Missing columns in CSV file: ['somecol', 'othercol']
+
+  If our input file provides the correct columns, it will work:
+
+     >>> custompath = 'mycustom.csv'
+     >>> open(custompath, 'wb').write(
+     ... """somecol,othercol,thirdcol
+     ... dataitem1,dataitem2,dataitem3
+     ... """)
+
+     >>> ICustomCSVFile(custompath)
+     <CustomCSVFile object at 0x...>
+
+* Provide and implement a custom interface
+
+  A custom CSV file wrapper should provide and implement an own
+  interface. Otherwise it could not be required by other components
+  explicitly.
+
+  We have to provide *and* implement the custom interface because
+  otherwise instances would not implement the required interface
+  (maybe due to a flaw in `grok`/`martian`).
+
+
+Common Use Cases
+================
+
+Get a wrapper for a certain type of CSV file
+--------------------------------------------
+
+The type of a CSVFile is determined by the interfaces it provides.
+
+If we want to get a wrapper that also guarantees to support certain
+fields (or None), then we already know about the wanted type.
+
+We create a file that does not have a 'special_col' field:
+
+    >>> path = 'mycsvfile.csv'
+    >>> open(path, 'wb').write(
+    ... """col1,col2
+    ... dataitem1,dataitem2
+    ... item3,item4
+    ... """)
+
+Now we create a wrapper, that requires that field:
+
+    >>> from waeup.csvfile.interfaces import ICSVFile
+    >>> from waeup.csvfile import CSVFile
+    >>> class ICSVFileWithSpecialCol(ICSVFile):
+    ...   """A CSV file that contains a 'special_col' column.
+    ...   """
+
+    >>> class CSVFileWithSpecialCol(CSVFile):
+    ...   required_fields = ['special_col']
+    ...   grok.provides(ICSVFileWithSpecialCol)
+
+    >>> grok.testing.grok_component('CSVFileWithSpecialCol', 
+    ...                             CSVFileWithSpecialCol)
+    True
+
+If we want to get a wrapper for that kind of file:
+
+    >>> ICSVFileWithSpecialCol(path)
+    Traceback (most recent call last):
+    ...
+    TypeError: Missing columns in CSV file: ['special_col']
+
+If the required col is available, however:
+
+    >>> path2 = 'mycsvfile2.csv'
+    >>> open(path2, 'wb').write(
+    ... """col1,col2,special_col
+    ... dataitem1,dataitem2,dataitem3
+    ... item4,item5,item6
+    ... """)
+    >>> ICSVFileWithSpecialCol(path2)
+    <CSVFileWithSpecialCol object at 0x...>
+
+Build an importer framework
+---------------------------
+
+We can also build an importer framework with CSV file support using
+the components described above.
+
+To model this, we start with two files, we will import lateron:
+
+    >>> path = 'mycsvfile.csv'
+    >>> open(path, 'wb').write(
+    ... """col1,col2
+    ... dataitem1a,dataitem2a
+    ... item3a,item4a
+    ... """)
+
+    >>> path2 = 'mycsvfile2.csv'
+    >>> open(path2, 'wb').write(
+    ... """col1,col3
+    ... dataitem1b,dataitem2b
+    ... item3b,item4b
+    ... """)
+
+Then we create two receivers for CSV file data:
+
+    >>> from zope.interface import Interface
+    >>> class IReceiver1(Interface):
+    ...   """A CSV data receiver."""
+
+    >>> class IReceiver2(Interface):
+    ...   """Another CSV data receiver."""
+
+    >>> class Receiver1(object):
+    ...   grok.implements(IReceiver1)
+    ...   def receive(self, data):
+    ...     print "Receiver1 received: ", data
+
+    >>> class Receiver2(object):
+    ...   grok.implements(IReceiver2)
+    ...   def receive(self, data):
+    ...     print "Receiver2 received: ", data
+    
+
+If we want to be sure, that a wrapper requires these fields, we ask
+for ICSVFile12:
+
+    >>> wrapper1 = ICSVFile12(path)
+    >>> wrapper1
+    <CSVFile12 object at 0x...>
+
+We could not use this interface (adapter) with the other CSV file:
+
+    >>> wrapper2 = ICSVFile12(path2)
+    Traceback (most recent call last):
+    ...
+    TypeError: Missing columns in CSV file: ['col2']
+
+The last step is to build a bridge between the receivers and the
+sources. We call it connector or importer here:
+
+    >>> class IImporter(Interface):
+    ...   """Import sources to receivers."""
+
+    >>> class IImporter12(IImporter):
+    ...   """Imports ICSVFile12 data into IReceiver1 objects."""
+
+    >>> class Importer12(grok.MultiAdapter):
+    ...   grok.adapts(ICSVFile12, IReceiver1)
+    ...   grok.implements(IImporter12)
+    ...   def __init__(self, csvfile, receiver):
+    ...     self.csvfile = csvfile
+    ...     self.receiver = receiver
+    ...   def doImport(self):
+    ...     self.receiver.receive(self.csvfile.getData())
+
+    >>> grok.testing.grok_component('Importer12',  Importer12)
+    True
+
+We can create an importer if we know the type of CSV file:
+
+    >>> myrecv = Receiver1()
+    >>> myfile = ICSVFile12(path)
+    >>> myfile
+    <CSVFile12 object at 0x...>
+
+    >>> ICSVFile12.providedBy(myfile)
+    True
+
+    >>> from zope.component import getMultiAdapter
+    >>> myimporter = getMultiAdapter((myfile, myrecv), IImporter12)
+    >>> myimporter
+    <Importer12 object at 0x...>
+
+
+We can also create an importer without knowing the type of CSV file
+before, using the `getCSVFile` function:
+
+    >>> from waeup.csvfile import getCSVFile
+    >>> myfile = getCSVFile(path)
+    >>> ICSVFile12.providedBy(myfile)
+    True
+
+Apparently, the CSVFileWrapper getter knows, that a CSVFile12 suits
+the contents of our file best. We did not specify what type of file
+wrapper we want.
+
+Getting an importer now is easy:
+
+    >>> myimporter = getMultiAdapter((myfile, myrecv), IImporter12)
+    >>> myimporter
+    <Importer12 object at 0x...>
+
+
+Clean up:
+
+    >>> import os
+    >>> os.unlink(path)
+    >>> os.unlink(path2)
+    >>> os.unlink(nocsvpath)
+    >>> os.unlink(custompath)
Index: /waeup/branches/ulif-rewrite/src/waeup/csvfile/__init__.py
===================================================================
--- /waeup/branches/ulif-rewrite/src/waeup/csvfile/__init__.py	(revision 4219)
+++ /waeup/branches/ulif-rewrite/src/waeup/csvfile/__init__.py	(revision 4219)
@@ -0,0 +1,2 @@
+# Make this a package.
+from waeup.csvfile.csvfile import getCSVFile, CSVFile
Index: /waeup/branches/ulif-rewrite/src/waeup/csvfile/csvfile.py
===================================================================
--- /waeup/branches/ulif-rewrite/src/waeup/csvfile/csvfile.py	(revision 4219)
+++ /waeup/branches/ulif-rewrite/src/waeup/csvfile/csvfile.py	(revision 4219)
@@ -0,0 +1,87 @@
+"""Support for CSV files.
+
+See README.txt and interfaces.py to learn more.
+"""
+
+import csv
+import grok
+from waeup.csvfile.interfaces import ICSVFile
+
+class CSVFile(grok.Adapter):
+    """A CSV file abstraction.
+    """
+    grok.context(basestring)
+    grok.implements(ICSVFile)
+
+    required_fields = []
+    path = None
+    
+    def __init__(self, context):
+        self.path = context
+        self.checkFields()
+
+    def checkFields(self):
+        headers = self.getHeaderFields()
+        missing = [x for x in self.required_fields
+                   if x not in headers]
+        if len(missing):
+            raise TypeError('Missing columns in CSV file: %s' % missing)
+        return
+            
+    def getDictReader(self):
+        return csv.DictReader(open(self.path, 'rb'))
+    
+    def getData(self):
+        reader = self.getDictReader()
+        for row in reader:
+            row = self.processRow(row)
+            yield row
+
+    def processRow(self, row):
+        return row
+
+    def getHeaderFields(self):
+        path = self.path
+        reader = self.getDictReader()
+        reader.next()
+        fields = reader.fieldnames
+        return sorted(reader.fieldnames)
+
+    
+# The registry for CSV file wrappers...
+csvwrappers = [CSVFile,]
+
+def getScore(path, wrapperclass):
+    """Compute a score for a certain wrapper related to a certain CSV
+    file.
+    """
+    wrapper = None
+    try:
+        wrapper = wrapperclass(path)
+    except:
+        return -1
+    required = wrapper.required_fields
+    if required == []:
+        return 0
+    available_headers = wrapper.getHeaderFields()
+    score = len([x for x in required if x in available_headers])
+    if score < len(required):
+        return -1
+    return score
+    
+    
+def getCSVFile(path):
+    """Get a CSVFile instance for a path.
+    """
+    best_score = -1
+    best_wrapper = None
+    
+    for wrapperclass in csvwrappers:
+        score = getScore(path, wrapperclass)
+        if score < best_score or score == -1:
+            continue
+        best_score, best_wrapper = score, wrapperclass
+    if best_wrapper is not None:
+        return best_wrapper(path)
+    return None
+
Index: /waeup/branches/ulif-rewrite/src/waeup/csvfile/interfaces.py
===================================================================
--- /waeup/branches/ulif-rewrite/src/waeup/csvfile/interfaces.py	(revision 4219)
+++ /waeup/branches/ulif-rewrite/src/waeup/csvfile/interfaces.py	(revision 4219)
@@ -0,0 +1,37 @@
+"""Interfaces for waeup.csvfile.
+"""
+from zope.interface import Interface
+from zope import schema
+
+class ICSVFile(Interface):
+    """A wrapper for a CSV file.
+    """
+    required_fields = schema.List(
+        title = u'Required header fields.',
+        value_type = schema.TextLine(
+            title = u'Field name',
+            )
+        )
+    
+    path = schema.TextLine(
+        title = u'Filesystem path to wrapped file.'
+        )
+
+    def getData():
+        """Get a dict of data contained in the CSV file.
+        """
+        
+    def getHeaderFields():
+        """Get a list of (supported) header fields.
+        """
+    
+class ICSVDataReceiver(Interface):
+    """A receiver for CSV data.
+    """
+    def setData(data):
+        """Set data.
+        """
+
+class ICSVDataConnector(Interface):
+    """A connector brings together a CSVDataSource and a CSVDataReceiver.
+    """
Index: /waeup/branches/ulif-rewrite/src/waeup/csvfile/meta.py
===================================================================
--- /waeup/branches/ulif-rewrite/src/waeup/csvfile/meta.py	(revision 4219)
+++ /waeup/branches/ulif-rewrite/src/waeup/csvfile/meta.py	(revision 4219)
@@ -0,0 +1,13 @@
+"""Meta configuration for waeup.csvfile components.
+"""
+import martian
+import waeup.csvfile.csvfile
+from waeup.csvfile import CSVFile
+
+# This grokker fills the registry on startup, with any class that is
+# derived from `CSVFileWrapper`...
+class CSVFileWrapperGrokker(martian.ClassGrokker):
+    martian.component(CSVFile)
+    def execute(self, klass, **kw):
+        waeup.csvfile.csvfile.csvwrappers.append(klass)
+        return True
