Index: /main/waeup.kofa/trunk/src/waeup/kofa/browser/tests/test_doctests.py
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/browser/tests/test_doctests.py	(revision 12945)
+++ /main/waeup.kofa/trunk/src/waeup/kofa/browser/tests/test_doctests.py	(revision 12946)
@@ -22,5 +22,5 @@
 def test_suite():
     suite = get_doctest_suite([
-        'doctests/batchprocessing_part2.txt',
+        'doctests/batchprocessing_browser.txt',
         'doctests/breadcrumbs.txt',
         'doctests/pages.txt',
Index: /main/waeup.kofa/trunk/src/waeup/kofa/doctests/accesscode_browser.txt
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/doctests/accesscode_browser.txt	(revision 12945)
+++ /main/waeup.kofa/trunk/src/waeup/kofa/doctests/accesscode_browser.txt	(revision 12946)
@@ -6,6 +6,4 @@
 Here we visit the access-code related parts of a Kofa site using
 a virtual browser.
-
-:NOTTest-Layer: functional
 
 Preliminaries
Index: /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing.txt
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing.txt	(revision 12946)
+++ /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing.txt	(revision 12946)
@@ -0,0 +1,554 @@
+Batch Processing
+****************
+
+Batch processing is much more than pure data import.
+
+Overview
+========
+
+Basically, it means processing CSV files in order to mass-create,
+mass-remove, or mass-update data.
+
+So you can feed CSV files to processors, that are part of
+the batch-processing mechanism.
+
+Processors
+----------
+
+Each CSV file processor
+
+* accepts a single data type identified by an interface.
+
+* knows about the places inside a site (University) where to store,
+  remove or update the data.
+
+* can check headers before processing data.
+
+* supports the mode 'create', 'update', 'remove'.
+
+* creates log entries (optional)
+
+* creates csv files containing successful and not-successful processed
+  data respectively.
+
+Output
+------
+
+The results of processing are written to loggers, if a logger was
+given. Beside this new CSV files are created during processing:
+
+* a pending CSV file, containing datasets that could not be processed
+
+* a finished CSV file, containing datasets successfully processed.
+
+The pending file is not created if everything works fine. The
+respective path returned in that case is ``None``.
+
+The pending file (if created) is a CSV file that contains the failed
+rows appended by a column ``--ERRROR--`` in which the reasons for
+processing failures are listed.
+
+The complete paths of these files are returned. They will be in a
+temporary directory created only for this purpose. It is the caller's
+responsibility to remove the temporay directories afterwards (the
+datacenters distProcessedFiles() method takes care for that).
+
+It looks like this::
+ 
+     -----+      +---------+
+    /     |      |         |              +------+
+   | .csv +----->|Batch-   |              |      |
+   |      |      |processor+----changes-->| ZODB |
+   |  +------+   |         |              |      |
+   +--|      |   |         +              +------+
+      | Mode +-->|         |                 -------+
+      |      |   |         +----outputs-+-> /       |
+      |  +----+->+---------+            |  |.pending|
+      +--|Log |  ^                      |  |        |
+         +----+  |                      |  +--------+
+           +-----++                     v
+           |Inter-|                  ----------+
+           |face  |                 /          |
+           +------+                | .finished |
+                                   |           |
+                                   +-----------+
+
+
+Creating a Batch Processor
+==========================
+
+We create an own batch processor for an own datatype. This datatype
+must be based on an interface that the batcher can use for converting
+data.
+
+Founding Stoneville
+-------------------
+
+We start with the interface:
+
+    >>> from zope.interface import Interface
+    >>> from zope import schema
+    >>> class ICave(Interface):
+    ...   """A cave."""
+    ...   name = schema.TextLine(
+    ...     title = u'Cave name',
+    ...     default = u'Unnamed',
+    ...     required = True)
+    ...   dinoports = schema.Int(
+    ...     title = u'Number of DinoPorts (tm)',
+    ...     required = False,
+    ...     default = 1)
+    ...   owner = schema.TextLine(
+    ...     title = u'Owner name',
+    ...     required = True,
+    ...     missing_value = 'Fred Estates Inc.')
+    ...   taxpayer = schema.Bool(
+    ...     title = u'Payes taxes',
+    ...     required = True,
+    ...     default = False)
+
+Now a class that implements this interface:
+
+    >>> import grok
+    >>> class Cave(object):
+    ...   grok.implements(ICave)
+    ...   def __init__(self, name=u'Unnamed', dinoports=2,
+    ...                owner='Fred Estates Inc.', taxpayer=False):
+    ...     self.name = name
+    ...     self.dinoports = 2
+    ...     self.owner = owner
+    ...     self.taxpayer = taxpayer
+
+We also provide a factory for caves. Strictly speaking, this not
+necessary but makes the batch processor we create afterwards, better
+understandable.
+
+    >>> from zope.component import getGlobalSiteManager
+    >>> from zope.component.factory import Factory
+    >>> from zope.component.interfaces import IFactory
+    >>> gsm = getGlobalSiteManager()
+    >>> cave_maker = Factory(Cave, 'A cave', 'Buy caves here!')
+    >>> gsm.registerUtility(cave_maker, IFactory, 'Lovely Cave')
+
+Now we can create caves using a factory:
+
+    >>> from zope.component import createObject
+    >>> createObject('Lovely Cave')
+    <Cave object at 0x...>
+
+This is nice, but we still lack a place, where we can place all the
+lovely caves we want to sell.
+
+Furthermore, as a replacement for a real site, we define a place where
+all caves can be stored: Stoneville! This is a lovely place for
+upperclass cavemen (which are the only ones that can afford more than
+one dinoport).
+
+We found Stoneville:
+
+    >>> stoneville = dict()
+
+Everything in place.
+
+Now, to improve local health conditions, imagine we want to populate
+Stoneville with lots of new happy dino-hunting natives that slept on
+the bare ground in former times and had no idea of
+bathrooms. Disgusting, isn't it?
+
+Lots of cavemen need lots of caves.
+
+Of course we can do something like:
+
+    >>> cave1 = createObject('Lovely Cave')
+    >>> cave1.name = "Fred's home"
+    >>> cave1.owner = "Fred"
+    >>> stoneville[cave1.name] = cave1
+
+and Stoneville has exactly
+
+    >>> len(stoneville)
+    1
+
+inhabitant. But we don't want to do this for hundreds or thousands of
+citizens-to-be, do we?
+
+It is much easier to create a simple CSV list, where we put in all the
+data and let a batch processor do the job.
+
+The list is already here:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner,taxpayer
+    ... Barneys Home,2,Barney,1
+    ... Wilmas Asylum,1,Wilma,1
+    ... Freds Dinoburgers,10,Fred,0
+    ... Joeys Drive-in,110,Joey,0
+    ... """)
+
+All we need, is a batch processor now.
+
+    >>> from waeup.kofa.utils.batching import BatchProcessor
+    >>> from waeup.kofa.interfaces import IGNORE_MARKER
+    >>> class CaveProcessor(BatchProcessor):
+    ...   util_name = 'caveprocessor'
+    ...   grok.name(util_name)
+    ...   name = 'Cave Processor'
+    ...   iface = ICave
+    ...   location_fields = ['name']
+    ...   factory_name = 'Lovely Cave'
+    ...
+    ...   def parentsExist(self, row, site):
+    ...     return True
+    ...
+    ...   def getParent(self, row, site):
+    ...     return stoneville
+    ...
+    ...   def entryExists(self, row, site):
+    ...     return row['name'] in stoneville.keys()
+    ...
+    ...   def getEntry(self, row, site):
+    ...     if not self.entryExists(row, site):
+    ...       return None
+    ...     return stoneville[row['name']]
+    ...
+    ...   def delEntry(self, row, site):
+    ...     del stoneville[row['name']]
+    ...
+    ...   def addEntry(self, obj, row, site):
+    ...     stoneville[row['name']] = obj
+    ...
+    ...   def updateEntry(self, obj, row, site, filename):
+    ...     # This is not strictly necessary, as the default
+    ...     # updateEntry method does exactly the same
+    ...     for key, value in row.items():
+    ...       if value != IGNORE_MARKER:
+    ...         setattr(obj, key, value)
+
+If we also want the results being logged, we must provide a logger
+(this is optional):
+
+    >>> import logging
+    >>> logger = logging.getLogger('stoneville')
+    >>> logger.setLevel(logging.DEBUG)
+    >>> logger.propagate = False
+    >>> handler = logging.FileHandler('stoneville.log', 'w')
+    >>> logger.addHandler(handler)
+
+Create the fellows:
+
+    >>> processor = CaveProcessor()
+    >>> result = processor.doImport('newcomers.csv',
+    ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
+    ...                    mode='create', user='Bob', logger=logger)
+    >>> result
+    (4, 0, '/.../newcomers.finished.csv', None)
+
+The result means: four entries were processed and no warnings
+occured. Furthermore we get filepath to a CSV file with successfully
+processed entries and a filepath to a CSV file with erraneous entries.
+As everything went well, the latter is ``None``. Let's check:
+
+    >>> sorted(stoneville.keys())
+    [u'Barneys Home', ..., u'Wilmas Asylum']
+
+The values of the Cave instances have correct type:
+
+    >>> barney = stoneville['Barneys Home']
+    >>> barney.dinoports
+    2
+
+which is a number, not a string.
+
+Apparently, when calling the processor, we gave some more info than
+only the CSV filepath. What does it all mean?
+
+While the first argument is the path to the CSV file, we also have to
+give an ordered list of headernames. These replace the header field
+names that are actually in the file. This way we can override faulty
+headers.
+
+The ``mode`` paramter tells what kind of operation we want to perform:
+``create``, ``update``, or ``remove`` data.
+
+The ``user`` parameter finally is optional and only used for logging.
+
+We can, by the way, see the results of our run in a logfile if we
+provided a logger during the call:
+
+    >>> print open('stoneville.log').read()
+    processed: newcomers.csv, create mode, 4 lines (4 successful/ 0 failed), ... s (... s/item)
+
+
+We cleanup the temporay dir created by doImport():
+
+    >>> import shutil
+    >>> import os
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+As we can see, the processing was successful. Otherwise, all problems
+could be read here as we can see, if we do the same operation again:
+
+    >>> result = processor.doImport('newcomers.csv',
+    ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
+    ...                    mode='create', user='Bob', logger=logger)
+    >>> result
+    (4, 4, '/.../newcomers.finished.csv', '/.../newcomers.pending.csv')
+
+This time we also get a path to a .pending file.
+
+The log file will tell us this in more detail:
+
+    >>> print open('stoneville.log').read()
+    processed: newcomers.csv, create mode, 4 lines (4 successful/ 0 failed), ... s (... s/item)
+    processed: newcomers.csv, create mode, 4 lines (0 successful/ 4 failed), ... s (... s/item)
+
+
+This time a new file was created, which keeps all the rows we could not
+process and an additional column with error messages:
+
+    >>> print open(result[3]).read()
+    owner,name,taxpayer,dinoports,--ERRORS--
+    Barney,Barneys Home,1,2,This object already exists.
+    Wilma,Wilmas Asylum,1,1,This object already exists.
+    Fred,Freds Dinoburgers,0,10,This object already exists.
+    Joey,Joeys Drive-in,0,110,This object already exists.
+
+This way we can correct the faulty entries and afterwards retry without
+having the already processed rows in the way.
+
+We also notice, that the values of the taxpayer column are returned as
+in the input file. There we wrote '1' for ``True`` and '0' for
+``False`` (which is accepted by the converters).
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+We can also tell to ignore some cols from input by passing
+``--IGNORE--`` as col name:
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             '--IGNORE--', '--IGNORE--'],
+    ...                    mode='update', user='Bob')
+    >>> result
+    (4, 0, '...', None)
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+If something goes wrong during processing, the respective --IGNORE--
+cols won't be populated  in the resulting pending file:
+
+    >>> result = processor.doImport('newcomers.csv', ['name', 'dinoports',
+    ...                             '--IGNORE--', '--IGNORE--'],
+    ...                    mode='create', user='Bob')
+    >>> result
+    (4, 4, '...', '...')
+
+    >>> print open(result[3], 'rb').read()
+    name,dinoports,--ERRORS--
+    Barneys Home,2,This object already exists.
+    Wilmas Asylum,1,This object already exists.
+    Freds Dinoburgers,10,This object already exists.
+    Joeys Drive-in,110,This object already exists.
+
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+Updating Entries
+----------------
+
+To update entries, we just call the batchprocessor in a different
+mode:
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='update', user='Bob')
+    >>> result
+    (4, 0, '...', None)
+
+Now we want to tell, that Wilma got an extra port for her second dino:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... Wilmas Asylum,2,Wilma
+    ... """)
+
+    >>> wilma = stoneville['Wilmas Asylum']
+    >>> wilma.dinoports
+    1
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+We start the processor:
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                    'dinoports', 'owner'], mode='update', user='Bob')
+    >>> result
+    (1, 0, '...', None)
+
+    >>> wilma = stoneville['Wilmas Asylum']
+    >>> wilma.dinoports
+    2
+
+Wilma's number of dinoports raised.
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+If we try to update an unexisting entry, an error occurs:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... NOT-WILMAS-ASYLUM,2,Wilma
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='update', user='Bob')
+    >>> result
+    (1, 1, '/.../newcomers.finished.csv', '/.../newcomers.pending.csv')
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+    
+Also invalid values will be spotted:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... Wilmas Asylum,NOT-A-NUMBER,Wilma
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='update', user='Bob')
+    >>> result
+    (1, 1, '...', '...')
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+We can also update only some cols, leaving some out. We skip the
+'dinoports' column in the next run:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,owner
+    ... Wilmas Asylum,Barney
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name', 'owner'],
+    ...                             mode='update', user='Bob')
+    >>> result
+    (1, 0, '...', None)
+
+    >>> wilma.owner
+    u'Barney'
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+We can however, not leave out the 'location field' ('name' in our
+case), as this one tells us which entry to update:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... 2,Wilma
+    ... """)
+
+    >>> processor.doImport('newcomers.csv', ['dinoports', 'owner'],
+    ...                    mode='update', user='Bob')
+    Traceback (most recent call last):
+    ...
+    FatalCSVError: Need at least columns 'name' for import!
+
+This time we get even an exception!
+
+Generally, empty strings are considered as ``None``:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... "Wilmas Asylum","","Wilma"
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='update', user='Bob')
+    >>> result
+    (1, 0, '...', None)
+
+    >>> wilma.dinoports
+    2
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+We can tell to set dinoports to ``None`` although this is not a
+number, as we declared the field not required in the interface:
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... "Wilmas Asylum","XXX","Wilma"
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='update', user='Bob', ignore_empty=False)
+    >>> result
+    (1, 0, '...', None)
+
+    >>> wilma.dinoports is None
+    True
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+Removing Entries
+----------------
+
+In 'remove' mode we can delete entries. Here validity of values in
+non-location fields doesn't matter because those fields are ignored.
+
+    >>> open('newcomers.csv', 'wb').write(
+    ... """name,dinoports,owner
+    ... "Wilmas Asylum","ILLEGAL-NUMBER",""
+    ... """)
+
+    >>> result = processor.doImport('newcomers.csv', ['name',
+    ...                             'dinoports', 'owner'],
+    ...                    mode='remove', user='Bob')
+    >>> result
+    (1, 0, '...', None)
+
+    >>> sorted(stoneville.keys())
+    [u'Barneys Home', "Fred's home", u'Freds Dinoburgers', u'Joeys Drive-in']
+
+Oops! Wilma is gone.
+
+Clean up:
+
+    >>> shutil.rmtree(os.path.dirname(result[2]))
+
+
+Clean up:
+
+    >>> import os
+    >>> os.unlink('newcomers.csv')
+    >>> os.unlink('stoneville.log')
Index: /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_browser.txt
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_browser.txt	(revision 12946)
+++ /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_browser.txt	(revision 12946)
@@ -0,0 +1,675 @@
+Batch Processing via Browser
+****************************
+
+Preliminaries:
+
+We define a function that looks up a form with several submit buttons
+for the one with a given value (this functionality is missing in
+zope.testbrowser):
+
+    >>> def lookup_submit_value(name, value, browser):
+    ...   """Find a button with a certain value."""
+    ...   for num in range(0, 100):
+    ...     try:
+    ...       button = browser.getControl(name=name, index=num)
+    ...       if button.value.endswith(value):
+    ...         return button
+    ...     except IndexError:
+    ...       break
+    ...   return None
+
+Create a site:
+
+    >>> from waeup.kofa.app import University
+    >>> getRootFolder()['app'] = University()
+    >>> from zope.component.hooks import setSite
+    >>> setSite(getRootFolder()['app'])
+
+Create a datacenter storage path:
+
+    >>> import os
+    >>> import tempfile
+    >>> dc_path = tempfile.mkdtemp()
+
+Log in:
+
+    >>> from zope.testbrowser.testing import Browser
+    >>> browser = Browser()
+    >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw')
+    >>> browser.handleErrors = False
+
+Set datacenter path and deselect moving old data:
+
+    >>> browser.open('http://localhost/app')
+    >>> browser.getLink('Data Center').click()
+    >>> browser.getLink('Edit settings').click()
+    >>> browser.getControl(name='newpath').value = dc_path
+    >>> browser.getControl(name='move').value = False
+    >>> browser.getControl(name='save').click()
+
+Set non-usable datacenter path:
+
+    >>> browser.getLink('Edit settings').click()
+    >>> browser.getControl(name='newpath').value = '/'
+    >>> browser.getControl(name='save').click()
+    >>> 'Given storage path cannot be used.' in browser.contents
+    True
+    >>> browser.getControl('Back to Data Center').click()
+
+
+Batch Processing Faculties
+==========================
+
+Go to datacenter page:
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for faculties (extended ascii values are accepted):
+
+    >>> open('faculties.csv', 'wb').write(
+    ... """code,title,title_prefix
+    ... FAC1,Faculty 1,faculty
+    ... FAC2,Faculty 2,institute
+    ... FAC3,Fäcülty 3,school
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('faculties.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'faculties.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'faculties_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.displayOptions
+    ['AccessCodeBatch Processor',
+    'AccessCode Processor',
+    'Applicant Processor',
+    'ApplicantsContainer Processor',
+    'CertificateCourse Processor',
+    'Certificate Processor',
+    'Course Processor',
+    'CourseTicket Processor',
+    'Department Processor',
+    'Faculty Processor',
+    'Hostel Processor',
+    'Public HTML Document Processor',
+    'StudentOnlinePayment Processor',
+    'Public PDF Document Processor',
+    'Public REST Document Processor',
+    'Student Processor',
+    'StudentStudyCourse Processor (update only)',
+    'StudentStudyLevel Processor',
+    'User Processor',
+    'Verdict Processor (special processor, update only)']
+
+    >>> importerselect.getControl('Faculty Processor').selected = True
+
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.options
+    ['create', 'update', 'remove']
+
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 3 rows...
+    ...Batch processing finished...
+    ...File:...faculties_zope.mgr.csv...
+
+We can grep the entries generated in logfile:
+
+    >>> browser.open('http://localhost/app/datacenter/logs')
+    >>> browser.getControl('Show', index=0).click()
+    >>> print browser.contents
+    <!DOCTYPE ...
+    ...<h1 class="kofa-content-label">Logfile datacenter.log</h1>...
+
+    >>> browser.getControl(name='query').value = "zope.mgr"
+    >>> browser.getControl('Search').click()
+    >>> 'zope.mgr - processed' in browser.contents
+    True
+
+
+Batch Processing Departments
+============================
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for departments:
+
+    >>> open('departments.csv', 'wb').write(
+    ... """code,faculty_code,title,title_prefix
+    ... DEP1,FAC1,Department 1,department
+    ... DEP2,FAC2,Department 2,centre
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('departments.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'departments.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'departments_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('Department Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 2 rows...
+    ...Batch processing finished...
+    ...File:...departments_zope.mgr.csv...
+
+Batch Processing Courses
+========================
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for courses:
+
+    >>> open('courses.csv', 'wb').write(
+    ... """code,faculty_code,department_code,title,level,passmark,credits,semester
+    ... CRS1,FAC1,DEP1,Course 1,100,40,2,1
+    ... CRS2,FAC1,DEP1,Course 2,100,40,2,2
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('courses.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'courses.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'courses_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('Course Processor', index=1).selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 2 rows...
+    ...Batch processing finished...
+    ...File:...courses_zope.mgr.csv...
+
+Batch Processing Certificates
+=============================
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for certificates:
+
+    >>> open('certificates.csv', 'wb').write(
+    ... """code,faculty_code,department_code,title,study_mode,start_level,end_level,application_category
+    ... CERT1,FAC1,DEP1,Certificate 1,pg_ft,999,999,basic
+    ... CERT2,FAC1,DEP1,Certificate 2,ug_ft,200,300,cest
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('certificates.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'certificates.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'certificates_zope.mgr.csv', browser)
+    >>> button.click()
+
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('Certificate Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 2 rows...
+    ...Batch processing finished...
+    ...File:...certificates_zope.mgr.csv...
+
+Batch Processing Certificate Courses
+====================================
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for certificate courses:
+
+    >>> open('mycertcourses.csv', 'wb').write(
+    ... """course,faculty_code,department_code,certificate_code,level,mandatory
+    ... CRS1,FAC1,DEP1,CERT1,100,True
+    ... CRS2,FAC1,DEP1,CERT1,100,True
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('mycertcourses.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'mycertcourses.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'mycertcourses_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('CertificateCourse Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 2 rows...
+    ...Batch processing finished...
+    ...File:...mycertcourses_zope.mgr.csv...
+
+Batch Processing Users
+======================
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for users:
+
+    >>> open('users.csv', 'wb').write(
+    ... """name,title,public_name,email,phone,roles
+    ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567,[]
+    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
+    ... anne,Anne Palina,,anne@abc.de,+49-234-567,"['waeup.Nonsense']"
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('users.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'users_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('User Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+We get informed that there are no problems with the current header:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    Header fields OK
+    ...
+
+The submit button is enabled:
+
+    >>> browser.getControl('Perform import').disabled
+    False
+
+    >>> browser.getControl('Perform import').click()
+
+Step 4: See import results
+
+The import was successful:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...Successfully processed 2 rows...
+    ...Batch processing finished...
+    ...File:...users_zope.mgr.csv...
+
+User henrik has got the global roles:
+
+   >>> henrik = getRootFolder()['app']['users']['henrik']
+   >>> henrik.roles
+   ['waeup.PortalManager', 'waeup.AcademicsOfficer', 'waeup.ImportManager']
+
+Pending Files
+=============
+
+When an error occurs during an import, two files are generated: a CSV
+file with finished files and a CSV file with pending data. Both are
+stored in the appropriate subdirectories in datacenter. We try to
+create faculties, from which one already exists.
+
+Go to datacenter page:
+
+    >>> browser.open('http://localhost/app/datacenter')
+
+Prepare a CSV file for faculties:
+
+    >>> open('newfaculties.csv', 'wb').write(
+    ... """code,title,title_prefix
+    ... FAC1,Faculty 1,faculty
+    ... FAC4,Faculty 4,school
+    ... FAC 5,Faculty 5,faculty
+    ... """)
+
+Upload the file:
+
+    >>> import cStringIO
+    >>> browser.getLink('Upload data').click()
+    >>> filecontents = cStringIO.StringIO(
+    ...   open('newfaculties.csv', 'rb').read())
+    >>> filewidget = browser.getControl(name='uploadfile:file')
+    >>> filewidget.add_file(filecontents, 'text/plain', 'newfaculties.csv')
+    >>> browser.getControl(name='SUBMIT').click()
+
+Since we now have a user with waeup.ImportManager role, an email has been sent:
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    ...
+    ...All import managers have been notified by email...
+    ...
+
+Step 1: start batch processing:
+
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'newfaculties_zope.mgr.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('Faculty Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3: Fix headerlines
+
+As there should be no problem with the headers, we can immediately
+perfom the import:
+
+    >>> browser.getControl('Perform import').click()
+
+Two lines could not be imported:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC...
+    ...
+    ...Processing of 2 rows failed...
+    ...Successfully processed 1 rows...
+    ...
+
+Now there are two files as a result in datacenter storage's root and
+``finished`` dirs:
+
+    >>> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv'
+    >>> print open(pending_file).read()
+    title_prefix,code,title,--ERRORS--
+    faculty,FAC1,Faculty 1,This object already exists.
+    faculty,FAC 5,Faculty 5,code: Invalid input
+
+    >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv'
+    >>> print open(finished_file).read()
+    title_prefix,code,title
+    school,FAC4,Faculty 4
+
+The finished-file contains the dataset we could import, while the
+pending file contains the dataset that failed, appended by an error
+message.
+
+
+Fixing the Pending File
+-----------------------
+
+We 'edit' the pending file (setting code to ``FAC5`` and title
+appropriately, and removing the --ERROR-- column) and finish the
+import this way:
+
+    >>> open(dc_path + '/newfaculties_zope.mgr.create.pending.csv', 'wb').write(
+    ... """title_prefix,--IGNORE--,code,title
+    ... faculty,,FAC5,Faculty 5
+    ... """)
+
+Step 1: start batch processing:
+
+    >>> browser.open('http://localhost/app/datacenter')
+    >>> browser.getLink('Process data').click()
+    >>> button = lookup_submit_value(
+    ...   'select', 'newfaculties_zope.mgr.create.pending.csv', browser)
+    >>> button.click()
+
+Step 2: select a processor and mode:
+
+    >>> importerselect = browser.getControl(name='importer')
+    >>> importerselect.getControl('Faculty Processor').selected = True
+    >>> modeselect = browser.getControl(name='mode')
+    >>> modeselect.getControl(value='create').selected = True
+    >>> browser.getControl('Proceed to step 3').click()
+
+Step 3/4: Fix headerlines and import:
+
+As there should be no problem with the headers, we can immediately
+perfom the import:
+
+    >>> browser.getControl('Perform import').click()
+
+This time everything should work:
+
+    >>> print browser.contents
+    <!DOCTYPE html PUBLIC...
+    ...
+    ...Successfully processed 1 rows...
+    ...
+
+Oh no, we forgot Anne Palina. Her user record was not imported because
+she has a non-existent role:
+
+    >>> sorted(os.listdir(dc_path))
+    ['deleted', 'finished', 'logs', 'unfinished', 'users_zope.mgr.create.pending.csv']
+
+    >>> os.listdir(dc_path + '/unfinished')
+    ['users_zope.mgr.csv']
+
+    >>> pending_file = dc_path + '/users_zope.mgr.create.pending.csv'
+    >>> print open(pending_file).read()
+    name,roles,title,public_name,phone,email,--ERRORS--
+    anne,['waeup.Nonsense'],Anne Palina,,+49-234-567,anne@abc.de,roles: invalid role
+
+There are many finished-files:
+
+    >>> sorted(os.listdir(dc_path + '/finished'))
+    ['certificates_zope.mgr.create.finished.csv', ...,
+    'users_zope.mgr.create.finished.csv']
+
+Processed (finished) Files
+==========================
+
+    >>> browser.open('http://localhost/app/datacenter/processed')
+    >>> 'download?filename=finished/certificates_zope.mgr.create.finished.csv' in browser.contents
+    True
+
+Log Files
+=========
+
+    >>> browser.open('http://localhost/app/datacenter/logs')
+    >>> 'datacenter.log' in browser.contents
+    True
+    >>> browser.getControl('Show', index=0).click()
+    >>> browser.getControl('Back', index=0).click()
+    >>> browser.getControl('Back to Data Center').click()
+    >>> 'Storage path:' in browser.contents
+    True
+
+
+Clean up:
+
+    >>> import shutil
+    >>> shutil.rmtree(dc_path)
Index: in/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_part1.txt
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_part1.txt	(revision 12945)
+++ 	(revision )
@@ -1,554 +1,0 @@
-Batch Processing
-****************
-
-Batch processing is much more than pure data import.
-
-Overview
-========
-
-Basically, it means processing CSV files in order to mass-create,
-mass-remove, or mass-update data.
-
-So you can feed CSV files to processors, that are part of
-the batch-processing mechanism.
-
-Processors
-----------
-
-Each CSV file processor
-
-* accepts a single data type identified by an interface.
-
-* knows about the places inside a site (University) where to store,
-  remove or update the data.
-
-* can check headers before processing data.
-
-* supports the mode 'create', 'update', 'remove'.
-
-* creates log entries (optional)
-
-* creates csv files containing successful and not-successful processed
-  data respectively.
-
-Output
-------
-
-The results of processing are written to loggers, if a logger was
-given. Beside this new CSV files are created during processing:
-
-* a pending CSV file, containing datasets that could not be processed
-
-* a finished CSV file, containing datasets successfully processed.
-
-The pending file is not created if everything works fine. The
-respective path returned in that case is ``None``.
-
-The pending file (if created) is a CSV file that contains the failed
-rows appended by a column ``--ERRROR--`` in which the reasons for
-processing failures are listed.
-
-The complete paths of these files are returned. They will be in a
-temporary directory created only for this purpose. It is the caller's
-responsibility to remove the temporay directories afterwards (the
-datacenters distProcessedFiles() method takes care for that).
-
-It looks like this::
- 
-     -----+      +---------+
-    /     |      |         |              +------+
-   | .csv +----->|Batch-   |              |      |
-   |      |      |processor+----changes-->| ZODB |
-   |  +------+   |         |              |      |
-   +--|      |   |         +              +------+
-      | Mode +-->|         |                 -------+
-      |      |   |         +----outputs-+-> /       |
-      |  +----+->+---------+            |  |.pending|
-      +--|Log |  ^                      |  |        |
-         +----+  |                      |  +--------+
-           +-----++                     v
-           |Inter-|                  ----------+
-           |face  |                 /          |
-           +------+                | .finished |
-                                   |           |
-                                   +-----------+
-
-
-Creating a Batch Processor
-==========================
-
-We create an own batch processor for an own datatype. This datatype
-must be based on an interface that the batcher can use for converting
-data.
-
-Founding Stoneville
--------------------
-
-We start with the interface:
-
-    >>> from zope.interface import Interface
-    >>> from zope import schema
-    >>> class ICave(Interface):
-    ...   """A cave."""
-    ...   name = schema.TextLine(
-    ...     title = u'Cave name',
-    ...     default = u'Unnamed',
-    ...     required = True)
-    ...   dinoports = schema.Int(
-    ...     title = u'Number of DinoPorts (tm)',
-    ...     required = False,
-    ...     default = 1)
-    ...   owner = schema.TextLine(
-    ...     title = u'Owner name',
-    ...     required = True,
-    ...     missing_value = 'Fred Estates Inc.')
-    ...   taxpayer = schema.Bool(
-    ...     title = u'Payes taxes',
-    ...     required = True,
-    ...     default = False)
-
-Now a class that implements this interface:
-
-    >>> import grok
-    >>> class Cave(object):
-    ...   grok.implements(ICave)
-    ...   def __init__(self, name=u'Unnamed', dinoports=2,
-    ...                owner='Fred Estates Inc.', taxpayer=False):
-    ...     self.name = name
-    ...     self.dinoports = 2
-    ...     self.owner = owner
-    ...     self.taxpayer = taxpayer
-
-We also provide a factory for caves. Strictly speaking, this not
-necessary but makes the batch processor we create afterwards, better
-understandable.
-
-    >>> from zope.component import getGlobalSiteManager
-    >>> from zope.component.factory import Factory
-    >>> from zope.component.interfaces import IFactory
-    >>> gsm = getGlobalSiteManager()
-    >>> cave_maker = Factory(Cave, 'A cave', 'Buy caves here!')
-    >>> gsm.registerUtility(cave_maker, IFactory, 'Lovely Cave')
-
-Now we can create caves using a factory:
-
-    >>> from zope.component import createObject
-    >>> createObject('Lovely Cave')
-    <Cave object at 0x...>
-
-This is nice, but we still lack a place, where we can place all the
-lovely caves we want to sell.
-
-Furthermore, as a replacement for a real site, we define a place where
-all caves can be stored: Stoneville! This is a lovely place for
-upperclass cavemen (which are the only ones that can afford more than
-one dinoport).
-
-We found Stoneville:
-
-    >>> stoneville = dict()
-
-Everything in place.
-
-Now, to improve local health conditions, imagine we want to populate
-Stoneville with lots of new happy dino-hunting natives that slept on
-the bare ground in former times and had no idea of
-bathrooms. Disgusting, isn't it?
-
-Lots of cavemen need lots of caves.
-
-Of course we can do something like:
-
-    >>> cave1 = createObject('Lovely Cave')
-    >>> cave1.name = "Fred's home"
-    >>> cave1.owner = "Fred"
-    >>> stoneville[cave1.name] = cave1
-
-and Stoneville has exactly
-
-    >>> len(stoneville)
-    1
-
-inhabitant. But we don't want to do this for hundreds or thousands of
-citizens-to-be, do we?
-
-It is much easier to create a simple CSV list, where we put in all the
-data and let a batch processor do the job.
-
-The list is already here:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner,taxpayer
-    ... Barneys Home,2,Barney,1
-    ... Wilmas Asylum,1,Wilma,1
-    ... Freds Dinoburgers,10,Fred,0
-    ... Joeys Drive-in,110,Joey,0
-    ... """)
-
-All we need, is a batch processor now.
-
-    >>> from waeup.kofa.utils.batching import BatchProcessor
-    >>> from waeup.kofa.interfaces import IGNORE_MARKER
-    >>> class CaveProcessor(BatchProcessor):
-    ...   util_name = 'caveprocessor'
-    ...   grok.name(util_name)
-    ...   name = 'Cave Processor'
-    ...   iface = ICave
-    ...   location_fields = ['name']
-    ...   factory_name = 'Lovely Cave'
-    ...
-    ...   def parentsExist(self, row, site):
-    ...     return True
-    ...
-    ...   def getParent(self, row, site):
-    ...     return stoneville
-    ...
-    ...   def entryExists(self, row, site):
-    ...     return row['name'] in stoneville.keys()
-    ...
-    ...   def getEntry(self, row, site):
-    ...     if not self.entryExists(row, site):
-    ...       return None
-    ...     return stoneville[row['name']]
-    ...
-    ...   def delEntry(self, row, site):
-    ...     del stoneville[row['name']]
-    ...
-    ...   def addEntry(self, obj, row, site):
-    ...     stoneville[row['name']] = obj
-    ...
-    ...   def updateEntry(self, obj, row, site, filename):
-    ...     # This is not strictly necessary, as the default
-    ...     # updateEntry method does exactly the same
-    ...     for key, value in row.items():
-    ...       if value != IGNORE_MARKER:
-    ...         setattr(obj, key, value)
-
-If we also want the results being logged, we must provide a logger
-(this is optional):
-
-    >>> import logging
-    >>> logger = logging.getLogger('stoneville')
-    >>> logger.setLevel(logging.DEBUG)
-    >>> logger.propagate = False
-    >>> handler = logging.FileHandler('stoneville.log', 'w')
-    >>> logger.addHandler(handler)
-
-Create the fellows:
-
-    >>> processor = CaveProcessor()
-    >>> result = processor.doImport('newcomers.csv',
-    ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
-    ...                    mode='create', user='Bob', logger=logger)
-    >>> result
-    (4, 0, '/.../newcomers.finished.csv', None)
-
-The result means: four entries were processed and no warnings
-occured. Furthermore we get filepath to a CSV file with successfully
-processed entries and a filepath to a CSV file with erraneous entries.
-As everything went well, the latter is ``None``. Let's check:
-
-    >>> sorted(stoneville.keys())
-    [u'Barneys Home', ..., u'Wilmas Asylum']
-
-The values of the Cave instances have correct type:
-
-    >>> barney = stoneville['Barneys Home']
-    >>> barney.dinoports
-    2
-
-which is a number, not a string.
-
-Apparently, when calling the processor, we gave some more info than
-only the CSV filepath. What does it all mean?
-
-While the first argument is the path to the CSV file, we also have to
-give an ordered list of headernames. These replace the header field
-names that are actually in the file. This way we can override faulty
-headers.
-
-The ``mode`` paramter tells what kind of operation we want to perform:
-``create``, ``update``, or ``remove`` data.
-
-The ``user`` parameter finally is optional and only used for logging.
-
-We can, by the way, see the results of our run in a logfile if we
-provided a logger during the call:
-
-    >>> print open('stoneville.log').read()
-    processed: newcomers.csv, create mode, 4 lines (4 successful/ 0 failed), ... s (... s/item)
-
-
-We cleanup the temporay dir created by doImport():
-
-    >>> import shutil
-    >>> import os
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-As we can see, the processing was successful. Otherwise, all problems
-could be read here as we can see, if we do the same operation again:
-
-    >>> result = processor.doImport('newcomers.csv',
-    ...                   ['name', 'dinoports', 'owner', 'taxpayer'],
-    ...                    mode='create', user='Bob', logger=logger)
-    >>> result
-    (4, 4, '/.../newcomers.finished.csv', '/.../newcomers.pending.csv')
-
-This time we also get a path to a .pending file.
-
-The log file will tell us this in more detail:
-
-    >>> print open('stoneville.log').read()
-    processed: newcomers.csv, create mode, 4 lines (4 successful/ 0 failed), ... s (... s/item)
-    processed: newcomers.csv, create mode, 4 lines (0 successful/ 4 failed), ... s (... s/item)
-
-
-This time a new file was created, which keeps all the rows we could not
-process and an additional column with error messages:
-
-    >>> print open(result[3]).read()
-    owner,name,taxpayer,dinoports,--ERRORS--
-    Barney,Barneys Home,1,2,This object already exists.
-    Wilma,Wilmas Asylum,1,1,This object already exists.
-    Fred,Freds Dinoburgers,0,10,This object already exists.
-    Joey,Joeys Drive-in,0,110,This object already exists.
-
-This way we can correct the faulty entries and afterwards retry without
-having the already processed rows in the way.
-
-We also notice, that the values of the taxpayer column are returned as
-in the input file. There we wrote '1' for ``True`` and '0' for
-``False`` (which is accepted by the converters).
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-We can also tell to ignore some cols from input by passing
-``--IGNORE--`` as col name:
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             '--IGNORE--', '--IGNORE--'],
-    ...                    mode='update', user='Bob')
-    >>> result
-    (4, 0, '...', None)
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-If something goes wrong during processing, the respective --IGNORE--
-cols won't be populated  in the resulting pending file:
-
-    >>> result = processor.doImport('newcomers.csv', ['name', 'dinoports',
-    ...                             '--IGNORE--', '--IGNORE--'],
-    ...                    mode='create', user='Bob')
-    >>> result
-    (4, 4, '...', '...')
-
-    >>> print open(result[3], 'rb').read()
-    name,dinoports,--ERRORS--
-    Barneys Home,2,This object already exists.
-    Wilmas Asylum,1,This object already exists.
-    Freds Dinoburgers,10,This object already exists.
-    Joeys Drive-in,110,This object already exists.
-
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-Updating Entries
-----------------
-
-To update entries, we just call the batchprocessor in a different
-mode:
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='update', user='Bob')
-    >>> result
-    (4, 0, '...', None)
-
-Now we want to tell, that Wilma got an extra port for her second dino:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... Wilmas Asylum,2,Wilma
-    ... """)
-
-    >>> wilma = stoneville['Wilmas Asylum']
-    >>> wilma.dinoports
-    1
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-We start the processor:
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                    'dinoports', 'owner'], mode='update', user='Bob')
-    >>> result
-    (1, 0, '...', None)
-
-    >>> wilma = stoneville['Wilmas Asylum']
-    >>> wilma.dinoports
-    2
-
-Wilma's number of dinoports raised.
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-If we try to update an unexisting entry, an error occurs:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... NOT-WILMAS-ASYLUM,2,Wilma
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='update', user='Bob')
-    >>> result
-    (1, 1, '/.../newcomers.finished.csv', '/.../newcomers.pending.csv')
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-    
-Also invalid values will be spotted:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... Wilmas Asylum,NOT-A-NUMBER,Wilma
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='update', user='Bob')
-    >>> result
-    (1, 1, '...', '...')
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-We can also update only some cols, leaving some out. We skip the
-'dinoports' column in the next run:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,owner
-    ... Wilmas Asylum,Barney
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name', 'owner'],
-    ...                             mode='update', user='Bob')
-    >>> result
-    (1, 0, '...', None)
-
-    >>> wilma.owner
-    u'Barney'
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-We can however, not leave out the 'location field' ('name' in our
-case), as this one tells us which entry to update:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... 2,Wilma
-    ... """)
-
-    >>> processor.doImport('newcomers.csv', ['dinoports', 'owner'],
-    ...                    mode='update', user='Bob')
-    Traceback (most recent call last):
-    ...
-    FatalCSVError: Need at least columns 'name' for import!
-
-This time we get even an exception!
-
-Generally, empty strings are considered as ``None``:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... "Wilmas Asylum","","Wilma"
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='update', user='Bob')
-    >>> result
-    (1, 0, '...', None)
-
-    >>> wilma.dinoports
-    2
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-We can tell to set dinoports to ``None`` although this is not a
-number, as we declared the field not required in the interface:
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... "Wilmas Asylum","XXX","Wilma"
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='update', user='Bob', ignore_empty=False)
-    >>> result
-    (1, 0, '...', None)
-
-    >>> wilma.dinoports is None
-    True
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-Removing Entries
-----------------
-
-In 'remove' mode we can delete entries. Here validity of values in
-non-location fields doesn't matter because those fields are ignored.
-
-    >>> open('newcomers.csv', 'wb').write(
-    ... """name,dinoports,owner
-    ... "Wilmas Asylum","ILLEGAL-NUMBER",""
-    ... """)
-
-    >>> result = processor.doImport('newcomers.csv', ['name',
-    ...                             'dinoports', 'owner'],
-    ...                    mode='remove', user='Bob')
-    >>> result
-    (1, 0, '...', None)
-
-    >>> sorted(stoneville.keys())
-    [u'Barneys Home', "Fred's home", u'Freds Dinoburgers', u'Joeys Drive-in']
-
-Oops! Wilma is gone.
-
-Clean up:
-
-    >>> shutil.rmtree(os.path.dirname(result[2]))
-
-
-Clean up:
-
-    >>> import os
-    >>> os.unlink('newcomers.csv')
-    >>> os.unlink('stoneville.log')
Index: in/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_part2.txt
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/doctests/batchprocessing_part2.txt	(revision 12945)
+++ 	(revision )
@@ -1,675 +1,0 @@
-Batch Processing via Browser
-****************************
-
-Preliminaries:
-
-We define a function that looks up a form with several submit buttons
-for the one with a given value (this functionality is missing in
-zope.testbrowser):
-
-    >>> def lookup_submit_value(name, value, browser):
-    ...   """Find a button with a certain value."""
-    ...   for num in range(0, 100):
-    ...     try:
-    ...       button = browser.getControl(name=name, index=num)
-    ...       if button.value.endswith(value):
-    ...         return button
-    ...     except IndexError:
-    ...       break
-    ...   return None
-
-Create a site:
-
-    >>> from waeup.kofa.app import University
-    >>> getRootFolder()['app'] = University()
-    >>> from zope.component.hooks import setSite
-    >>> setSite(getRootFolder()['app'])
-
-Create a datacenter storage path:
-
-    >>> import os
-    >>> import tempfile
-    >>> dc_path = tempfile.mkdtemp()
-
-Log in:
-
-    >>> from zope.testbrowser.testing import Browser
-    >>> browser = Browser()
-    >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw')
-    >>> browser.handleErrors = False
-
-Set datacenter path and deselect moving old data:
-
-    >>> browser.open('http://localhost/app')
-    >>> browser.getLink('Data Center').click()
-    >>> browser.getLink('Edit settings').click()
-    >>> browser.getControl(name='newpath').value = dc_path
-    >>> browser.getControl(name='move').value = False
-    >>> browser.getControl(name='save').click()
-
-Set non-usable datacenter path:
-
-    >>> browser.getLink('Edit settings').click()
-    >>> browser.getControl(name='newpath').value = '/'
-    >>> browser.getControl(name='save').click()
-    >>> 'Given storage path cannot be used.' in browser.contents
-    True
-    >>> browser.getControl('Back to Data Center').click()
-
-
-Batch Processing Faculties
-==========================
-
-Go to datacenter page:
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for faculties (extended ascii values are accepted):
-
-    >>> open('faculties.csv', 'wb').write(
-    ... """code,title,title_prefix
-    ... FAC1,Faculty 1,faculty
-    ... FAC2,Faculty 2,institute
-    ... FAC3,Fäcülty 3,school
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('faculties.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'faculties.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'faculties_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.displayOptions
-    ['AccessCodeBatch Processor',
-    'AccessCode Processor',
-    'Applicant Processor',
-    'ApplicantsContainer Processor',
-    'CertificateCourse Processor',
-    'Certificate Processor',
-    'Course Processor',
-    'CourseTicket Processor',
-    'Department Processor',
-    'Faculty Processor',
-    'Hostel Processor',
-    'Public HTML Document Processor',
-    'StudentOnlinePayment Processor',
-    'Public PDF Document Processor',
-    'Public REST Document Processor',
-    'Student Processor',
-    'StudentStudyCourse Processor (update only)',
-    'StudentStudyLevel Processor',
-    'User Processor',
-    'Verdict Processor (special processor, update only)']
-
-    >>> importerselect.getControl('Faculty Processor').selected = True
-
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.options
-    ['create', 'update', 'remove']
-
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 3 rows...
-    ...Batch processing finished...
-    ...File:...faculties_zope.mgr.csv...
-
-We can grep the entries generated in logfile:
-
-    >>> browser.open('http://localhost/app/datacenter/logs')
-    >>> browser.getControl('Show', index=0).click()
-    >>> print browser.contents
-    <!DOCTYPE ...
-    ...<h1 class="kofa-content-label">Logfile datacenter.log</h1>...
-
-    >>> browser.getControl(name='query').value = "zope.mgr"
-    >>> browser.getControl('Search').click()
-    >>> 'zope.mgr - processed' in browser.contents
-    True
-
-
-Batch Processing Departments
-============================
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for departments:
-
-    >>> open('departments.csv', 'wb').write(
-    ... """code,faculty_code,title,title_prefix
-    ... DEP1,FAC1,Department 1,department
-    ... DEP2,FAC2,Department 2,centre
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('departments.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'departments.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'departments_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Department Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 2 rows...
-    ...Batch processing finished...
-    ...File:...departments_zope.mgr.csv...
-
-Batch Processing Courses
-========================
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for courses:
-
-    >>> open('courses.csv', 'wb').write(
-    ... """code,faculty_code,department_code,title,level,passmark,credits,semester
-    ... CRS1,FAC1,DEP1,Course 1,100,40,2,1
-    ... CRS2,FAC1,DEP1,Course 2,100,40,2,2
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('courses.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'courses.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'courses_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Course Processor', index=1).selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 2 rows...
-    ...Batch processing finished...
-    ...File:...courses_zope.mgr.csv...
-
-Batch Processing Certificates
-=============================
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for certificates:
-
-    >>> open('certificates.csv', 'wb').write(
-    ... """code,faculty_code,department_code,title,study_mode,start_level,end_level,application_category
-    ... CERT1,FAC1,DEP1,Certificate 1,pg_ft,999,999,basic
-    ... CERT2,FAC1,DEP1,Certificate 2,ug_ft,200,300,cest
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('certificates.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'certificates.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'certificates_zope.mgr.csv', browser)
-    >>> button.click()
-
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Certificate Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 2 rows...
-    ...Batch processing finished...
-    ...File:...certificates_zope.mgr.csv...
-
-Batch Processing Certificate Courses
-====================================
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for certificate courses:
-
-    >>> open('mycertcourses.csv', 'wb').write(
-    ... """course,faculty_code,department_code,certificate_code,level,mandatory
-    ... CRS1,FAC1,DEP1,CERT1,100,True
-    ... CRS2,FAC1,DEP1,CERT1,100,True
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('mycertcourses.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'mycertcourses.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'mycertcourses_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('CertificateCourse Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 2 rows...
-    ...Batch processing finished...
-    ...File:...mycertcourses_zope.mgr.csv...
-
-Batch Processing Users
-======================
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for users:
-
-    >>> open('users.csv', 'wb').write(
-    ... """name,title,public_name,email,phone,roles
-    ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567,[]
-    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
-    ... anne,Anne Palina,,anne@abc.de,+49-234-567,"['waeup.Nonsense']"
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('users.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'users_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('User Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-We get informed that there are no problems with the current header:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    Header fields OK
-    ...
-
-The submit button is enabled:
-
-    >>> browser.getControl('Perform import').disabled
-    False
-
-    >>> browser.getControl('Perform import').click()
-
-Step 4: See import results
-
-The import was successful:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...Successfully processed 2 rows...
-    ...Batch processing finished...
-    ...File:...users_zope.mgr.csv...
-
-User henrik has got the global roles:
-
-   >>> henrik = getRootFolder()['app']['users']['henrik']
-   >>> henrik.roles
-   ['waeup.PortalManager', 'waeup.AcademicsOfficer', 'waeup.ImportManager']
-
-Pending Files
-=============
-
-When an error occurs during an import, two files are generated: a CSV
-file with finished files and a CSV file with pending data. Both are
-stored in the appropriate subdirectories in datacenter. We try to
-create faculties, from which one already exists.
-
-Go to datacenter page:
-
-    >>> browser.open('http://localhost/app/datacenter')
-
-Prepare a CSV file for faculties:
-
-    >>> open('newfaculties.csv', 'wb').write(
-    ... """code,title,title_prefix
-    ... FAC1,Faculty 1,faculty
-    ... FAC4,Faculty 4,school
-    ... FAC 5,Faculty 5,faculty
-    ... """)
-
-Upload the file:
-
-    >>> import cStringIO
-    >>> browser.getLink('Upload data').click()
-    >>> filecontents = cStringIO.StringIO(
-    ...   open('newfaculties.csv', 'rb').read())
-    >>> filewidget = browser.getControl(name='uploadfile:file')
-    >>> filewidget.add_file(filecontents, 'text/plain', 'newfaculties.csv')
-    >>> browser.getControl(name='SUBMIT').click()
-
-Since we now have a user with waeup.ImportManager role, an email has been sent:
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    ...
-    ...All import managers have been notified by email...
-    ...
-
-Step 1: start batch processing:
-
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'newfaculties_zope.mgr.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Faculty Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3: Fix headerlines
-
-As there should be no problem with the headers, we can immediately
-perfom the import:
-
-    >>> browser.getControl('Perform import').click()
-
-Two lines could not be imported:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC...
-    ...
-    ...Processing of 2 rows failed...
-    ...Successfully processed 1 rows...
-    ...
-
-Now there are two files as a result in datacenter storage's root and
-``finished`` dirs:
-
-    >>> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv'
-    >>> print open(pending_file).read()
-    title_prefix,code,title,--ERRORS--
-    faculty,FAC1,Faculty 1,This object already exists.
-    faculty,FAC 5,Faculty 5,code: Invalid input
-
-    >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv'
-    >>> print open(finished_file).read()
-    title_prefix,code,title
-    school,FAC4,Faculty 4
-
-The finished-file contains the dataset we could import, while the
-pending file contains the dataset that failed, appended by an error
-message.
-
-
-Fixing the Pending File
------------------------
-
-We 'edit' the pending file (setting code to ``FAC5`` and title
-appropriately, and removing the --ERROR-- column) and finish the
-import this way:
-
-    >>> open(dc_path + '/newfaculties_zope.mgr.create.pending.csv', 'wb').write(
-    ... """title_prefix,--IGNORE--,code,title
-    ... faculty,,FAC5,Faculty 5
-    ... """)
-
-Step 1: start batch processing:
-
-    >>> browser.open('http://localhost/app/datacenter')
-    >>> browser.getLink('Process data').click()
-    >>> button = lookup_submit_value(
-    ...   'select', 'newfaculties_zope.mgr.create.pending.csv', browser)
-    >>> button.click()
-
-Step 2: select a processor and mode:
-
-    >>> importerselect = browser.getControl(name='importer')
-    >>> importerselect.getControl('Faculty Processor').selected = True
-    >>> modeselect = browser.getControl(name='mode')
-    >>> modeselect.getControl(value='create').selected = True
-    >>> browser.getControl('Proceed to step 3').click()
-
-Step 3/4: Fix headerlines and import:
-
-As there should be no problem with the headers, we can immediately
-perfom the import:
-
-    >>> browser.getControl('Perform import').click()
-
-This time everything should work:
-
-    >>> print browser.contents
-    <!DOCTYPE html PUBLIC...
-    ...
-    ...Successfully processed 1 rows...
-    ...
-
-Oh no, we forgot Anne Palina. Her user record was not imported because
-she has a non-existent role:
-
-    >>> sorted(os.listdir(dc_path))
-    ['deleted', 'finished', 'logs', 'unfinished', 'users_zope.mgr.create.pending.csv']
-
-    >>> os.listdir(dc_path + '/unfinished')
-    ['users_zope.mgr.csv']
-
-    >>> pending_file = dc_path + '/users_zope.mgr.create.pending.csv'
-    >>> print open(pending_file).read()
-    name,roles,title,public_name,phone,email,--ERRORS--
-    anne,['waeup.Nonsense'],Anne Palina,,+49-234-567,anne@abc.de,roles: invalid role
-
-There are many finished-files:
-
-    >>> sorted(os.listdir(dc_path + '/finished'))
-    ['certificates_zope.mgr.create.finished.csv', ...,
-    'users_zope.mgr.create.finished.csv']
-
-Processed (finished) Files
-==========================
-
-    >>> browser.open('http://localhost/app/datacenter/processed')
-    >>> 'download?filename=finished/certificates_zope.mgr.create.finished.csv' in browser.contents
-    True
-
-Log Files
-=========
-
-    >>> browser.open('http://localhost/app/datacenter/logs')
-    >>> 'datacenter.log' in browser.contents
-    True
-    >>> browser.getControl('Show', index=0).click()
-    >>> browser.getControl('Back', index=0).click()
-    >>> browser.getControl('Back to Data Center').click()
-    >>> 'Storage path:' in browser.contents
-    True
-
-
-Clean up:
-
-    >>> import shutil
-    >>> shutil.rmtree(dc_path)
Index: /main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_doctests.py
===================================================================
--- /main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_doctests.py	(revision 12945)
+++ /main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_doctests.py	(revision 12946)
@@ -21,4 +21,4 @@
 
 def test_suite():
-    suite = get_doctest_suite(['doctests/batchprocessing_part1.txt'])
+    suite = get_doctest_suite(['doctests/batchprocessing.txt'])
     return suite
