Changeset 4821 for waeup/branches/ulif-importers

Timestamp:

15 Jan 2010, 23:35:40 (15 years ago)

Author:

uli

Message:

• Catch exceptions when converter fails.

• Output log of operation.

• Add converter for courses.

• Clean up.

File:

• waeup/branches/ulif-importers/src/waeup/utils/batching.py (modified) (12 diffs)

waeup/branches/ulif-importers/src/waeup/utils/batching.py

@@ -6 +6 @@
 import grok
 import csv
+import os
+import sys
+import time
 from zope.component import createObject
 from zope.interface import Interface
 from zope.schema import getFields
 from zope.schema.interfaces import IText
-from waeup.interfaces import (IImporter, IFacultyContainer, IFaculty,
+from waeup.interfaces import (IImporter, IFacultyContainer, IFaculty, ICourse,
                               IDepartment, ISchemaTypeConverter, FatalCSVError)
 
@@ -118 +121 @@
         if '--IGNORE--' in row.keys():
             del row['--IGNORE--']
+        warnings = []
         for key, value in row.items():
             converter = converter_dict.get(key, None)
             if converter:
-                row.update({key:converter_dict[key].fromString(value)})
-        return row
+                try:
+                    row.update({key:converter_dict[key].fromString(value)})
+                except:
+                    msg = "conversion error: field %s: %r" % (
+                        key, sys.exc_info()[1])
+                    warnings.append(msg)
+        return (row, warnings)
 
     def callFactory(self, *args, **kw):
@@ -156 +165 @@
         """
         raise NotImplementedError('method not implemented')
-    
-    def doImport(self, path, headerfields, mode='create'):
+
+    def createLogfile(self, path, fail_path, num, warnings, mode, user):
+        """Write a log file.
+        """
+        failmsg_path = "%s.%s.msg" % (path, mode)
+        fd = open(failmsg_path, 'wb')
+        fd.write("Source: %s\n" % path)
+        fd.write("Mode: %s\n" % mode)
+        fd.write("Date: %s\n" % time.ctime())
+        fd.write("User: %s\n" % user)
+        fd.write("Failed datasets: %s\n" % fail_path)
+        fd.write("Processed: %s lines (%s successful/ %s failed)\n" % (
+                num, num - len(warnings), len(warnings)
+                ))
+        fd.write("-" * 70 + '\n')
+        failnum = 1
+        for linenum, text in sorted(warnings.items()):
+            fd.write("line %s (%s): %s\n" % (failnum, linenum, text))
+            failnum += 1
+        fd.close()
+        return
+    
+    def doImport(self, path, headerfields, mode='create', user='Unknown'):
         """Perform actual import.
         """
@@ -164 +194 @@
         converters = self.getFieldConverters(headerfields)
         reader = csv.DictReader(open(path, 'rb'))
+        
+        failed_path = "%s.%s.pending" % (path, mode, )
+        failed_headers = [x for x in mapping.values()
+               if x != '--IGNORE--']
+        failed_writer = csv.DictWriter(open(failed_path, 'wb'),
+                                       failed_headers)
+        failed_writer.writerow(dict([(x,x) for x in failed_headers]))
+        
         warnings = dict()
         num =0
@@ -170 +208 @@
             num += 1
             string_row = self.applyMapping(raw_row, mapping)
-            row = self.convertToTypes(string_row, converters)
+            row, conv_warnings = self.convertToTypes(string_row, converters)
+            if len(conv_warnings):
+                warnings[num] = ', '.join(conv_warnings)
+                failed_writer.writerow(row)
+                continue
+                
             if mode == 'create':
                 if not self.parentsExist(row, site):
                     warnings[num] = "Not all parents do exist yet. Skipping"
+                    failed_writer.writerow(row)
                     continue
                 if self.entryExists(row, site):
                     warnings[num] = 'This object already exists. Skipping.'
+                    failed_writer.writerow(row)
                     continue
                 obj = self.callFactory()
@@ -185 +230 @@
                 if not self.entryExists(row, site):
                     warnings[num] = "Cannot remove: no such entry."
+                    failed_writer.writerow(row)
                     continue
                 self.delEntry(row, site)
@@ -191 +237 @@
                 if obj is None:
                     warnings[num] = "Cannot update: no such entry."
+                    failed_writer.writerow(row)
                     continue
                 self.updateEntry(obj, row, site)
-                
-        #for linenum, text in warnings.items():
-        #    print "Line %s: %s" % (linenum, text)
+
+        if path.endswith('.pending'):
+            os.unlink(path)
+            os.rename(failed_path, path)
+            failed_path = path
+        self.createLogfile(path, failed_path, num, warnings, mode, user)
+        if len(warnings) == 0:
+            del failed_writer
+            os.unlink(failed_path)
         return (num, warnings)
 
@@ -228 +281 @@
         parent = self.getParent(row, site)
         parent.addFaculty(obj)
-        print "ADD: ", obj
         return
 
@@ -234 +286 @@
         parent = self.getParent(row, site)
         del parent[row['code']]
-        print "REMOVE: ", row['code']
         pass
 
@@ -240 +291 @@
         for key, value in row.items():
             setattr(obj, key, value)
-        print "UPDATE: ", obj
         pass
 
@@ -278 +328 @@
         parent = self.getParent(row, site)
         parent.addDepartment(obj)
-        print "ADD: ", obj
         return
 
@@ -284 +333 @@
         parent = self.getParent(row, site)
         del parent[row['code']]
-        print "REMOVE: ", row['code']
-        pass
+        return
 
     def updateEntry(self, obj, row, site):
         for key, value in row.items():
             setattr(obj, key, value)
-        print "UPDATE: ", obj
-        pass
+        return
+
+class CourseImporter(BatchProcessor):
+    grok.provides(IImporter)
+    grok.context(Interface)
+    util_name = 'courseimporter'
+    grok.name(util_name)
+
+    name = u'Course importer'
+    iface = ICourse
+
+    location_fields = ['code', 'faculty_code', 'department_code']
+    factory_name = 'waeup.Course'
+
+    def parentsExist(self, row, site):
+        if not 'faculties' in site.keys():
+            return False
+        if not row['faculty_code'] in site['faculties'].keys():
+            return False
+        faculty = site['faculties'][row['faculty_code']]
+        return row['department_code'] in faculty.keys()
+
+    def entryExists(self, row, site):
+        if not self.parentsExist(row, site):
+            return False
+        parent = self.getParent(row, site)
+        return row['code'] in parent.keys()
+
+    def getParent(self, row, site):
+        dept = site['faculties'][row['faculty_code']][row['department_code']]
+        return dept.courses
+
+    def getEntry(self, row, site):
+        if not self.entryExists(row, site):
+            return None
+        parent = self.getParent(row, site)
+        return parent.get(row['code'])
+    
+    def addEntry(self, obj, row, site):
+        parent = self.getParent(row, site)
+        parent.addCourse(obj)
+        return
+
+    def delEntry(self, row, site):
+        parent = self.getParent(row, site)
+        del parent[row['code']]
+        return
+
+    def updateEntry(self, obj, row, site):
+        for key, value in row.items():
+            setattr(obj, key, value)
+        return
+