Changeset 4802 for waeup/branches/ulif-importers

Timestamp:

14 Jan 2010, 09:07:41 (15 years ago)

Author:

uli

Message:

Add pages for batch processing.

File:

• waeup/branches/ulif-importers/src/waeup/browser/pages.py (modified) (5 diffs)

waeup/branches/ulif-importers/src/waeup/browser/pages.py

@@ -3 +3 @@
 """
 import copy
+import csv
 import grok
 import os
@@ -28 +29 @@
 from zope.component.interfaces import Invalid
 from zope.exceptions import DuplicationError
+from zope.session.interfaces import ISession
+
+from waeup.interfaces import IImporter
+from zope.component import getAllUtilitiesRegisteredFor
+
 
 grok.context(IWAeUPObject)
@@ -245 +251 @@
     grok.context(IDataCenter)
     grok.name('index')
+    grok.require('waeup.manageUniversity')
     title = u'Data Center'
     pnav = 0
@@ -251 +258 @@
     grok.context(IDataCenter)
     grok.name('upload')
+    grok.require('waeup.manageUniversity')
     title = u'Data Center Upload'
     pnav = 0
@@ -310 +318 @@
         return
 
+class DatacenterImportStep1(WAeUPPage):
+    """Manual import step 1: choose file
+    """
+    grok.context(IDataCenter)
+    grok.name('import1')
+    grok.template('datacenterimport1page')
+    grok.require('waeup.manageUniversity')
+    title = u'Import CSV file manually'
+    pnav = 0
+
+    def update(self, filename=None, select=None, cancel=None):
+        if cancel is not None:
+            self.flash('Import aborted')
+            self.redirect(self.url(self.context))
+            return
+        if select is not None:
+            # A filename was selected
+            session = ISession(self.request)['waeup.sirp']
+            session['import_filename'] = select
+            self.redirect(self.url(self.context, '@@import2'))
+
+class DatacenterImportStep2(WAeUPPage):
+    """Manual import step 2: choose importer
+    """
+    grok.context(IDataCenter)
+    grok.name('import2')
+    grok.template('datacenterimport2page')
+    grok.require('waeup.manageUniversity')
+    title = u'Import CSV file manually'
+    pnav = 0
+
+    filename = None
+    mode = 'create'
+    importer = None
+
+    def getPreviewHeader(self):
+        """Get the header fields of attached CSV file.
+        """
+        reader = csv.reader(open(self.fullpath, 'rb'))
+        return reader.next()
+    
+    def getPreviewBody(self):
+        """Get the first 5 rows of attached CSV file.
+        """
+        result = []
+        num = 0
+        for row in self.reader:
+            if num > 4:
+                break
+            num += 1
+            row = row.items()
+            # Sort fields in headerfield order
+            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
+            row = [x[1] for x in row]
+            result.append(row)
+        result.append(len(result[0]) * ['...'])
+        return result
+
+    def getImporters(self):
+        importers = getAllUtilitiesRegisteredFor(IImporter)
+        importers = [
+            dict(title=x.name, name=x.util_name) for x in importers]
+        return importers
+        
+    
+    def update(self, mode=None, importer=None,
+               back1=None, cancel=None, proceed=None):
+        session = ISession(self.request)['waeup.sirp']
+        self.filename = session.get('import_filename', None)
+        
+        if self.filename is None or back1 is not None:
+            self.redirect(self.url(self.context, '@@import1'))
+            return
+        if cancel is not None:
+            self.flash('Import aborted')
+            self.redirect(self.url(self.context))
+            return
+        self.mode = mode or session.get('import_mode', self.mode)
+        self.importer = importer or session.get('import_importer', None)
+        session['import_mode'] = self.mode
+        session['import_importer'] = self.importer
+        if proceed is not None:
+            self.redirect(self.url(self.context, '@@import3'))
+            return
+        self.fullpath = os.path.join(self.context.storage, self.filename)
+        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
+
+class DatacenterImportStep3(WAeUPPage):
+    """Manual import step 3: modify header
+    """
+    grok.context(IDataCenter)
+    grok.name('import3')
+    grok.template('datacenterimport3page')
+    grok.require('waeup.manageUniversity')
+    title = u'Import CSV file manually'
+    pnav = 0
+
+    filename = None
+    mode = None
+    importername = None
+    
+    @property
+    def nextstep(self):
+        return self.url(self.context, '@@import4')
+
+    def getPreviewHeader(self):
+        """Get the header fields of attached CSV file.
+        """
+        reader = csv.reader(open(self.fullpath, 'rb'))
+        return reader.next()
+    
+    def getPreviewBody(self):
+        """Get the first 5 rows of attached CSV file.
+        """
+        result = []
+        num = 0
+        for row in self.reader:
+            if num > 4:
+                break
+            num += 1
+            row = row.items()
+            # Sort fields in headerfield order
+            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
+            row = [x[1] for x in row]
+            result.append(row)
+        result.append(len(result[0]) * ['...'])
+        return result
+
+    def getPossibleHeaders(self):
+        """Get the possible headers.
+
+        The headers are described as dicts {value:internal_name,
+        title:displayed_name}
+        """
+        result = [dict(title='<IGNORE COL>', value='--IGNORE--')]
+        headers = self.importer.getHeaders()
+        result.extend([dict(title=x, value=x) for x in headers])
+        return result
+
+    def getWarnings(self):
+        import sys
+        result = []
+        try:
+            self.importer.checkHeaders(self.headerfields, mode=self.mode)
+        except:
+            fatal = '%s' % sys.exc_info()[1]
+            result.append(fatal)
+        return result
+    
+    @property
+    def nextstep(self):
+        return self.url(self.context, '@@import4')
+
+    def update(self, headerfield=None, back2=None, cancel=None, proceed=None):
+        session = ISession(self.request)['waeup.sirp']
+        self.filename = session.get('import_filename', None)
+        self.mode = session.get('import_mode', None)
+        self.importername = session.get('import_importer', None)
+        
+        if None in (self.filename, self.mode, self.importername):
+            self.redirect(self.url(self.context, '@@import2'))
+            return
+        if back2 is not None:
+            self.redirect(self.url(self.context ,'@@import2'))
+            return
+        if cancel is not None:
+            self.flash('Import aborted.')
+            self.redirect(self.url(self.context))
+            return
+
+        self.fullpath = os.path.join(self.context.storage, self.filename)
+        self.headerfields = headerfield or self.getPreviewHeader()
+        session['import_headerfields'] = self.headerfields
+
+        if proceed is not None:
+            self.redirect(self.url(self.context, '@@import4'))
+            return
+        
+        self.importer = getUtility(IImporter, name=self.importername)
+        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
+
+class DatacenterImportStep4(WAeUPPage):
+    """Manual import step 4: do actual import
+    """
+    grok.context(IDataCenter)
+    grok.name('import4')
+    grok.template('datacenterimport4page')
+    grok.require('waeup.manageUniversity')
+    title = u'Import CSV file manually'
+    pnav = 0
+
+    filename = None
+    mode = None
+    importername = None
+    headerfields = None
+    warnnum = None
+
+    def update(self, back=None, finish=None):
+        if finish is not None:
+            self.redirect(self.url(self.context))
+            return
+        if back is not None:
+            self.redirect(self.url(self.context, '@@import3'))
+            return
+        session = ISession(self.request)['waeup.sirp']
+        self.filename = session.get('import_filename', None)
+        self.mode = session.get('import_mode', None)
+        self.importername = session.get('import_importer', None)
+        self.headerfields = session.get('import_headerfields', None)
+        
+        if None in (self.filename, self.mode, self.importername,
+                    self.headerfields):
+            self.redirect(self.url(self.context, '@@import3'))
+
+        self.fullpath = os.path.join(self.context.storage, self.filename)
+        self.importer = getUtility(IImporter, name=self.importername)
+        (linenum, warnings) = self.importer.doImport(
+            self.fullpath, self.headerfields, self.mode)
+        self.warn_num = len(warnings)
+        if self.warn_num:
+            self.flash('Processing of %d rows failed!' % self.warn_num)
+        self.flash('Successfully processed %s rows' % (
+                linenum - (self.warn_num)))
+
+        
 class DatacenterSettings(WAeUPPage):
     grok.context(IDataCenter)