Index: /waeup/branches/ulif-importers/src/waeup/utils/batching.py
===================================================================
--- /waeup/branches/ulif-importers/src/waeup/utils/batching.py	(revision 4806)
+++ /waeup/branches/ulif-importers/src/waeup/utils/batching.py	(revision 4806)
@@ -0,0 +1,270 @@
+"""WAeUP components for batch processing.
+
+Batch processors eat CSV files to add, update or remove large numbers
+of certain kinds of objects at once.
+"""
+import grok
+import csv
+from zope.component import createObject
+from zope.interface import Interface
+from zope.schema import getFields
+from zope.schema.interfaces import IText
+from waeup.interfaces import (IImporter, IFacultyContainer, IFaculty,
+                              IDepartment, ISchemaTypeConverter, FatalCSVError)
+
+class BatchProcessor(grok.GlobalUtility):
+    """A processor to add, update, or remove data.
+
+    This is a non-active baseclass.
+    """
+    grok.provides(IImporter)
+    grok.context(Interface)
+    grok.baseclass()
+
+    # Name used in pages and forms...
+    name = u'Department importer'
+    
+    # Internal name...
+    util_name = 'departmentimporter'
+    
+    # Items for this processor need an interface with zope.schema fields.
+    iface = IDepartment
+    
+    # The name must be the same as the util_name attribute in order to
+    # register this utility correctly.
+    grok.name(util_name)
+
+    # Headers needed to locate items...
+    location_fields = ['code', 'faculty_code']
+    
+    # A factory with this name must be registered...
+    factory_name = 'waeup.Department'
+
+    @property
+    def required_fields(self):
+        result = []
+        for key, field in getFields(self.iface).items():
+            if key in self.location_fields:
+                continue
+            if field.required:
+                result.append(key)
+        return result
+        
+    @property
+    def req(self):
+        result = dict(
+            create = self.location_fields + self.required_fields,
+            update = self.location_fields,
+            remove = self.location_fields,
+        )
+        return result
+
+    @property
+    def available_fields(self):
+        result = []
+        return sorted(list(set(
+                    self.location_fields + getFields(self.iface).keys())))
+    
+    def getHeaders(self, mode='create'):
+        return self.available_fields
+
+    def checkHeaders(self, headerfields, mode='create'):
+        req = self.req[mode]
+        # Check for required fields...
+        for field in req:
+            if not field in headerfields:
+                raise FatalCSVError(
+                    "Need at least columns %s for import!" %
+                    ', '.join(["'%s'" % x for x in req]))
+        # Check for double fields...
+        not_ignored_fields = [x for x in headerfields
+                              if not x.startswith('--')]
+        if len(set(not_ignored_fields)) < len(not_ignored_fields):
+            raise FatalCSVError(
+                "Double headers: each column name may only appear once.")
+        return True
+
+    def applyMapping(self, row, mapping):
+        result = dict()
+        for key, replacement in mapping.items():
+            result[replacement] = row[key]
+        return result
+        
+    def getMapping(self, path, headerfields):
+        reader = csv.reader(open(path, 'rb'))
+        raw_header = reader.next()
+        return dict([(x, headerfields[raw_header.index(x)])
+                     for x in raw_header])
+
+    def getFieldConverters(self, fieldnames):
+        result = dict()
+        for key, field in getFields(self.iface).items():
+            if key not in fieldnames:
+                continue
+            converter = ISchemaTypeConverter(field)
+            result[key] = converter
+        return result
+    
+    def convertToTypes(self, row, converter_dict):
+        if '--IGNORE--' in row.keys():
+            del row['--IGNORE--']
+        for key, value in row.items():
+            converter = converter_dict.get(key, None)
+            if converter:
+                row.update({key:converter_dict[key].convert(value)})
+        return row
+
+    def callFactory(self, *args, **kw):
+        return createObject(self.factory_name)
+
+    def parentsExist(self, row, site):
+        raise NotImplementedError('method not implemented')
+
+    def entryExists(self, row, site):
+        raise NotImplementedError('method not implemented')
+
+    def getParent(self, row, site):
+        raise NotImplementedError('method not implemented')
+    
+    def addEntry(self, obj, row, site):
+        raise NotImplementedError('method not implemented')
+
+    def delEntry(self, row, site):
+        raise NotImplementedError('method not implemented')        
+
+    def updateEntry(self, obj, row, site):
+        raise NotImplementedError('method not implemented')
+    
+    def doImport(self, path, headerfields, mode='create'):
+        self.checkHeaders(headerfields, mode)
+        mapping = self.getMapping(path, headerfields)
+        converters = self.getFieldConverters(headerfields)
+        reader = csv.DictReader(open(path, 'rb'))
+        warnings = dict()
+        num =0
+        site = grok.getSite()
+        for raw_row in reader:
+            num += 1
+            string_row = self.applyMapping(raw_row, mapping)
+            row = self.convertToTypes(string_row, converters)
+            if mode == 'create':
+                if not self.parentsExist(row, site):
+                    warnings[num] = "Not all parents do exist yet. Skipping"
+                    continue
+                if self.entryExists(row, site):
+                    warnings[num] = 'This object already exists. Skipping.'
+                    continue
+                obj = self.callFactory()
+                for key, value in row.items():
+                    setattr(obj, key, value)
+                self.addEntry(obj, row, site)
+            elif mode == 'remove':
+                if not self.entryExists(row, site):
+                    warnings[num] = "Cannot remove: no such entry."
+                    continue
+                self.delEntry(row, site)
+            elif mode == 'update':
+                obj = self.getEntry(row, site)
+                if obj is None:
+                    warnings[num] = "Cannot update: no such entry."
+                    continue
+                self.updateEntry(obj, row, site)
+                
+        #for linenum, text in warnings.items():
+        #    print "Line %s: %s" % (linenum, text)
+        return (num, warnings)
+
+class FacultyImporter(BatchProcessor):
+    grok.provides(IImporter)
+    grok.context(Interface)
+    util_name = 'facultyimporter'
+    grok.name(util_name)
+
+    name = u'Faculty importer'
+    iface = IFaculty
+
+    location_fields = ['code',]
+    factory_name = 'waeup.Faculty'
+
+    def parentsExist(self, row, site):
+        return 'faculties' in site.keys()
+
+    def entryExists(self, row, site):
+        return row['code'] in site['faculties'].keys()
+
+    def getParent(self, row, site):
+        return site['faculties']
+
+    def getEntry(self, row, site):
+        if not self.entryExists(row, site):
+            return None
+        parent = self.getParent(row, site)
+        return parent.get(row['code'])
+    
+    def addEntry(self, obj, row, site):
+        parent = self.getParent(row, site)
+        parent.addFaculty(obj)
+        print "ADD: ", obj
+        return
+
+    def delEntry(self, row, site):
+        parent = self.getParent(row, site)
+        del parent[row['code']]
+        print "REMOVE: ", row['code']
+        pass
+
+    def updateEntry(self, obj, row, site):
+        for key, value in row.items():
+            setattr(obj, key, value)
+        print "UPDATE: ", obj
+        pass
+
+class DepartmentImporter(BatchProcessor):
+    grok.provides(IImporter)
+    grok.context(Interface)
+    util_name = 'departmentimporter'
+    grok.name(util_name)
+
+    name = u'Department importer'
+    iface = IDepartment
+
+    location_fields = ['code', 'faculty_code']
+    factory_name = 'waeup.Department'
+
+    def parentsExist(self, row, site):
+        if not 'faculties' in site.keys():
+            return False
+        return row['faculty_code'] in site['faculties']
+
+    def entryExists(self, row, site):
+        if not self.parentsExist(row, site):
+            return False
+        parent = self.getParent(row, site)
+        return row['code'] in parent.keys()
+
+    def getParent(self, row, site):
+        return site['faculties'][row['faculty_code']]
+
+    def getEntry(self, row, site):
+        if not self.entryExists(row, site):
+            return None
+        parent = self.getParent(row, site)
+        return parent.get(row['code'])
+    
+    def addEntry(self, obj, row, site):
+        parent = self.getParent(row, site)
+        parent.addDepartment(obj)
+        print "ADD: ", obj
+        return
+
+    def delEntry(self, row, site):
+        parent = self.getParent(row, site)
+        del parent[row['code']]
+        print "REMOVE: ", row['code']
+        pass
+
+    def updateEntry(self, obj, row, site):
+        for key, value in row.items():
+            setattr(obj, key, value)
+        print "UPDATE: ", obj
+        pass
