Ignore:
Timestamp:
14 Jan 2010, 09:07:41 (15 years ago)
Author:
uli
Message:

Add pages for batch processing.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • waeup/branches/ulif-importers/src/waeup/browser/pages.py

    r4789 r4802  
    33"""
    44import copy
     5import csv
    56import grok
    67import os
     
    2829from zope.component.interfaces import Invalid
    2930from zope.exceptions import DuplicationError
     31from zope.session.interfaces import ISession
     32
     33from waeup.interfaces import IImporter
     34from zope.component import getAllUtilitiesRegisteredFor
     35
    3036
    3137grok.context(IWAeUPObject)
     
    245251    grok.context(IDataCenter)
    246252    grok.name('index')
     253    grok.require('waeup.manageUniversity')
    247254    title = u'Data Center'
    248255    pnav = 0
     
    251258    grok.context(IDataCenter)
    252259    grok.name('upload')
     260    grok.require('waeup.manageUniversity')
    253261    title = u'Data Center Upload'
    254262    pnav = 0
     
    310318        return
    311319
     320class DatacenterImportStep1(WAeUPPage):
     321    """Manual import step 1: choose file
     322    """
     323    grok.context(IDataCenter)
     324    grok.name('import1')
     325    grok.template('datacenterimport1page')
     326    grok.require('waeup.manageUniversity')
     327    title = u'Import CSV file manually'
     328    pnav = 0
     329
     330    def update(self, filename=None, select=None, cancel=None):
     331        if cancel is not None:
     332            self.flash('Import aborted')
     333            self.redirect(self.url(self.context))
     334            return
     335        if select is not None:
     336            # A filename was selected
     337            session = ISession(self.request)['waeup.sirp']
     338            session['import_filename'] = select
     339            self.redirect(self.url(self.context, '@@import2'))
     340
     341class DatacenterImportStep2(WAeUPPage):
     342    """Manual import step 2: choose importer
     343    """
     344    grok.context(IDataCenter)
     345    grok.name('import2')
     346    grok.template('datacenterimport2page')
     347    grok.require('waeup.manageUniversity')
     348    title = u'Import CSV file manually'
     349    pnav = 0
     350
     351    filename = None
     352    mode = 'create'
     353    importer = None
     354
     355    def getPreviewHeader(self):
     356        """Get the header fields of attached CSV file.
     357        """
     358        reader = csv.reader(open(self.fullpath, 'rb'))
     359        return reader.next()
     360   
     361    def getPreviewBody(self):
     362        """Get the first 5 rows of attached CSV file.
     363        """
     364        result = []
     365        num = 0
     366        for row in self.reader:
     367            if num > 4:
     368                break
     369            num += 1
     370            row = row.items()
     371            # Sort fields in headerfield order
     372            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
     373            row = [x[1] for x in row]
     374            result.append(row)
     375        result.append(len(result[0]) * ['...'])
     376        return result
     377
     378    def getImporters(self):
     379        importers = getAllUtilitiesRegisteredFor(IImporter)
     380        importers = [
     381            dict(title=x.name, name=x.util_name) for x in importers]
     382        return importers
     383       
     384   
     385    def update(self, mode=None, importer=None,
     386               back1=None, cancel=None, proceed=None):
     387        session = ISession(self.request)['waeup.sirp']
     388        self.filename = session.get('import_filename', None)
     389       
     390        if self.filename is None or back1 is not None:
     391            self.redirect(self.url(self.context, '@@import1'))
     392            return
     393        if cancel is not None:
     394            self.flash('Import aborted')
     395            self.redirect(self.url(self.context))
     396            return
     397        self.mode = mode or session.get('import_mode', self.mode)
     398        self.importer = importer or session.get('import_importer', None)
     399        session['import_mode'] = self.mode
     400        session['import_importer'] = self.importer
     401        if proceed is not None:
     402            self.redirect(self.url(self.context, '@@import3'))
     403            return
     404        self.fullpath = os.path.join(self.context.storage, self.filename)
     405        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
     406
     407class DatacenterImportStep3(WAeUPPage):
     408    """Manual import step 3: modify header
     409    """
     410    grok.context(IDataCenter)
     411    grok.name('import3')
     412    grok.template('datacenterimport3page')
     413    grok.require('waeup.manageUniversity')
     414    title = u'Import CSV file manually'
     415    pnav = 0
     416
     417    filename = None
     418    mode = None
     419    importername = None
     420   
     421    @property
     422    def nextstep(self):
     423        return self.url(self.context, '@@import4')
     424
     425    def getPreviewHeader(self):
     426        """Get the header fields of attached CSV file.
     427        """
     428        reader = csv.reader(open(self.fullpath, 'rb'))
     429        return reader.next()
     430   
     431    def getPreviewBody(self):
     432        """Get the first 5 rows of attached CSV file.
     433        """
     434        result = []
     435        num = 0
     436        for row in self.reader:
     437            if num > 4:
     438                break
     439            num += 1
     440            row = row.items()
     441            # Sort fields in headerfield order
     442            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
     443            row = [x[1] for x in row]
     444            result.append(row)
     445        result.append(len(result[0]) * ['...'])
     446        return result
     447
     448    def getPossibleHeaders(self):
     449        """Get the possible headers.
     450
     451        The headers are described as dicts {value:internal_name,
     452        title:displayed_name}
     453        """
     454        result = [dict(title='<IGNORE COL>', value='--IGNORE--')]
     455        headers = self.importer.getHeaders()
     456        result.extend([dict(title=x, value=x) for x in headers])
     457        return result
     458
     459    def getWarnings(self):
     460        import sys
     461        result = []
     462        try:
     463            self.importer.checkHeaders(self.headerfields, mode=self.mode)
     464        except:
     465            fatal = '%s' % sys.exc_info()[1]
     466            result.append(fatal)
     467        return result
     468   
     469    @property
     470    def nextstep(self):
     471        return self.url(self.context, '@@import4')
     472
     473    def update(self, headerfield=None, back2=None, cancel=None, proceed=None):
     474        session = ISession(self.request)['waeup.sirp']
     475        self.filename = session.get('import_filename', None)
     476        self.mode = session.get('import_mode', None)
     477        self.importername = session.get('import_importer', None)
     478       
     479        if None in (self.filename, self.mode, self.importername):
     480            self.redirect(self.url(self.context, '@@import2'))
     481            return
     482        if back2 is not None:
     483            self.redirect(self.url(self.context ,'@@import2'))
     484            return
     485        if cancel is not None:
     486            self.flash('Import aborted.')
     487            self.redirect(self.url(self.context))
     488            return
     489
     490        self.fullpath = os.path.join(self.context.storage, self.filename)
     491        self.headerfields = headerfield or self.getPreviewHeader()
     492        session['import_headerfields'] = self.headerfields
     493
     494        if proceed is not None:
     495            self.redirect(self.url(self.context, '@@import4'))
     496            return
     497       
     498        self.importer = getUtility(IImporter, name=self.importername)
     499        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
     500
     501class DatacenterImportStep4(WAeUPPage):
     502    """Manual import step 4: do actual import
     503    """
     504    grok.context(IDataCenter)
     505    grok.name('import4')
     506    grok.template('datacenterimport4page')
     507    grok.require('waeup.manageUniversity')
     508    title = u'Import CSV file manually'
     509    pnav = 0
     510
     511    filename = None
     512    mode = None
     513    importername = None
     514    headerfields = None
     515    warnnum = None
     516
     517    def update(self, back=None, finish=None):
     518        if finish is not None:
     519            self.redirect(self.url(self.context))
     520            return
     521        if back is not None:
     522            self.redirect(self.url(self.context, '@@import3'))
     523            return
     524        session = ISession(self.request)['waeup.sirp']
     525        self.filename = session.get('import_filename', None)
     526        self.mode = session.get('import_mode', None)
     527        self.importername = session.get('import_importer', None)
     528        self.headerfields = session.get('import_headerfields', None)
     529       
     530        if None in (self.filename, self.mode, self.importername,
     531                    self.headerfields):
     532            self.redirect(self.url(self.context, '@@import3'))
     533
     534        self.fullpath = os.path.join(self.context.storage, self.filename)
     535        self.importer = getUtility(IImporter, name=self.importername)
     536        (linenum, warnings) = self.importer.doImport(
     537            self.fullpath, self.headerfields, self.mode)
     538        self.warn_num = len(warnings)
     539        if self.warn_num:
     540            self.flash('Processing of %d rows failed!' % self.warn_num)
     541        self.flash('Successfully processed %s rows' % (
     542                linenum - (self.warn_num)))
     543
     544       
    312545class DatacenterSettings(WAeUPPage):
    313546    grok.context(IDataCenter)
Note: See TracChangeset for help on using the changeset viewer.