Changeset 4858


Ignore:
Timestamp:
19 Jan 2010, 17:39:32 (15 years ago)
Author:
uli
Message:

Merge changes from ulif-importers branch back into trunk.

Location:
waeup/trunk/src/waeup
Files:
7 edited
13 copied

Legend:

Unmodified
Added
Removed

  • waeup/trunk/src/waeup/browser.txt

    r4789 r4858  
    797797  >>> browser.getControl(name='SUBMIT').click()
    798798
    799 The file was indeed uploaded:
     799The file was indeed uploaded, with the current userid inserted:
    800800
    801801  >>> os.listdir(uploadpath)
    802   ['myfaculties.csv']
     802  ['myfaculties_zope.mgr.csv']
    803803
    804804We create and upload also a CSV file containing departments:
    … …  
    880880  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    881881  ...The following files are available for import:...
    882   ...mycertcourses.csv...Certificate Course Importer...
    883   ...mycertificates.csv...Certificate Importer...
    884   ...mycourses.csv...Course Importer...
    885   ...mydepartments.csv...Department Importer...
    886   ...myfaculties.csv...Faculty Importer...
     882  ...mycertcourses_zope.mgr.csv...Certificate Course Importer...
     883  ...mycertificates_zope.mgr.csv...Certificate Importer...
     884  ...mycourses_zope.mgr.csv...Course Importer...
     885  ...mydepartments_zope.mgr.csv...Department Importer...
     886  ...myfaculties_zope.mgr.csv...Faculty Importer...
    887887  ...
    888888
    … …  
    898898  >>> print browser.contents
    899899  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    900   ...Successfully imported: myfaculties.csv...
     900  ...Successfully imported: myfaculties_zope.mgr.csv...
    901901  ...
    902902
    … …  
    919919  >>> print browser.contents
    920920  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    921   ...Successfully imported: mydepartments.csv...
     921  ...Successfully imported: mydepartments_zope.mgr.csv...
    922922  ...
    923923
    … …  
    930930  >>> print browser.contents
    931931  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    932   ...Successfully imported: mycourses.csv...
     932  ...Successfully imported: mycourses_zope.mgr.csv...
    933933  ...
    934934
    … …  
    941941  >>> print browser.contents
    942942  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    943   ...Successfully imported: mycertificates.csv...
     943  ...Successfully imported: mycertificates_zope.mgr.csv...
    944944  ...
    945945
    … …  
    952952  >>> print browser.contents
    953953  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"...
    954   ...Successfully imported: mycertcourses.csv...
     954  ...Successfully imported: mycertcourses_zope.mgr.csv...
    955955  ...
    956956

  • waeup/trunk/src/waeup/browser/pages.py

    r4789 r4858  
    33"""
    44import copy
     5import csv
    56import grok
    67import os
     8import re
    79import sys
    810from hurry import yui
    … …  
    2830from zope.component.interfaces import Invalid
    2931from zope.exceptions import DuplicationError
     32from zope.session.interfaces import ISession
     33
     34from waeup.interfaces import IBatchProcessor
     35from zope.component import getAllUtilitiesRegisteredFor
     36
    3037
    3138grok.context(IWAeUPObject)
    … …  
    245252    grok.context(IDataCenter)
    246253    grok.name('index')
     254    grok.require('waeup.manageUniversity')
    247255    title = u'Data Center'
    248256    pnav = 0
    … …  
    251259    grok.context(IDataCenter)
    252260    grok.name('upload')
     261    grok.require('waeup.manageUniversity')
    253262    title = u'Data Center Upload'
    254263    pnav = 0
    … …  
    262271        try:
    263272            filename = uploadfile.filename
    264             target = os.path.join(self.context.storage, filename)
     273            target = os.path.join(self.context.storage,
     274                                  self.getNormalizedFileName(filename))
    265275            open(target, 'wb').write(uploadfile.read())
    266276        except IOError:
    … …  
    270280        self.redirect(self.url(self.context))
    271281
     282    def getNormalizedFileName(self, filename):
     283        """Build sane filename.
     284
     285        An uploaded file foo.csv will be stored as foo_USERNAME.csv
     286        where username is the principal id of the currently logged in
     287        user.
     288
     289        Spaces in filename are replaced by underscore.
     290        """
     291        username = self.request.principal.id
     292        filename = filename.replace(' ', '_')
     293        # Only accept typical filname chars...
     294        filtered_username = ''.join(re.findall('[a-zA-Z0-9_\.\-]', username))
     295        base, ext = os.path.splitext(filename)
     296        return '%s_%s%s' % (base, filtered_username, ext.lower())
    272297
    273298class DataCenterImportCSVPage(WAeUPPage):
    … …  
    310335        return
    311336
     337class DatacenterImportStep1(WAeUPPage):
     338    """Manual import step 1: choose file
     339    """
     340    grok.context(IDataCenter)
     341    grok.name('import1')
     342    grok.template('datacenterimport1page')
     343    grok.require('waeup.manageUniversity')
     344    title = u'Process CSV file'
     345    pnav = 0
     346
     347    def getFiles(self):
     348        files = self.context.getFiles(sort='date')
     349        for file in files:
     350            name = file.name
     351            if not name.endswith('.csv') and not name.endswith('.pending'):
     352                continue
     353            yield file
     354   
     355    def update(self, filename=None, select=None, cancel=None):
     356        if cancel is not None:
     357            self.flash('Import aborted')
     358            self.redirect(self.url(self.context))
     359            return
     360        if select is not None:
     361            # A filename was selected
     362            session = ISession(self.request)['waeup.sirp']
     363            session['import_filename'] = select
     364            self.redirect(self.url(self.context, '@@import2'))
     365
     366class DatacenterImportStep2(WAeUPPage):
     367    """Manual import step 2: choose importer
     368    """
     369    grok.context(IDataCenter)
     370    grok.name('import2')
     371    grok.template('datacenterimport2page')
     372    grok.require('waeup.manageUniversity')
     373    title = u'Process CSV file'
     374    pnav = 0
     375
     376    filename = None
     377    mode = 'create'
     378    importer = None
     379
     380    def getPreviewHeader(self):
     381        """Get the header fields of attached CSV file.
     382        """
     383        reader = csv.reader(open(self.fullpath, 'rb'))
     384        return reader.next()
     385   
     386    def getPreviewBody(self):
     387        """Get the first 5 rows of attached CSV file.
     388        """
     389        result = []
     390        num = 0
     391        for row in self.reader:
     392            if num > 4:
     393                break
     394            num += 1
     395            row = row.items()
     396            # Sort fields in headerfield order
     397            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
     398            row = [x[1] for x in row]
     399            result.append(row)
     400        result.append(len(result[0]) * ['...'])
     401        return result
     402
     403    def getImporters(self):
     404        importers = getAllUtilitiesRegisteredFor(IBatchProcessor)
     405        importers = [
     406            dict(title=x.name, name=x.util_name) for x in importers]
     407        return importers
     408       
     409   
     410    def update(self, mode=None, importer=None,
     411               back1=None, cancel=None, proceed=None):
     412        session = ISession(self.request)['waeup.sirp']
     413        self.filename = session.get('import_filename', None)
     414       
     415        if self.filename is None or back1 is not None:
     416            self.redirect(self.url(self.context, '@@import1'))
     417            return
     418        if cancel is not None:
     419            self.flash('Import aborted')
     420            self.redirect(self.url(self.context))
     421            return
     422        self.mode = mode or session.get('import_mode', self.mode)
     423        self.importer = importer or session.get('import_importer', None)
     424        session['import_mode'] = self.mode
     425        session['import_importer'] = self.importer
     426        if proceed is not None:
     427            self.redirect(self.url(self.context, '@@import3'))
     428            return
     429        self.fullpath = os.path.join(self.context.storage, self.filename)
     430        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
     431
     432class DatacenterImportStep3(WAeUPPage):
     433    """Manual import step 3: modify header
     434    """
     435    grok.context(IDataCenter)
     436    grok.name('import3')
     437    grok.template('datacenterimport3page')
     438    grok.require('waeup.manageUniversity')
     439    title = u'Process CSV file'
     440    pnav = 0
     441
     442    filename = None
     443    mode = None
     444    importername = None
     445   
     446    @property
     447    def nextstep(self):
     448        return self.url(self.context, '@@import4')
     449
     450    def getPreviewHeader(self):
     451        """Get the header fields of attached CSV file.
     452        """
     453        reader = csv.reader(open(self.fullpath, 'rb'))
     454        return reader.next()
     455   
     456    def getPreviewBody(self):
     457        """Get the first 5 rows of attached CSV file.
     458        """
     459        result = []
     460        num = 0
     461        for row in self.reader:
     462            if num > 4:
     463                break
     464            num += 1
     465            row = row.items()
     466            # Sort fields in headerfield order
     467            row = sorted(row, key=lambda k: self.reader.fieldnames.index(k[0]))
     468            row = [x[1] for x in row]
     469            result.append(row)
     470        result.append(len(result[0]) * ['...'])
     471        return result
     472
     473    def getPossibleHeaders(self):
     474        """Get the possible headers.
     475
     476        The headers are described as dicts {value:internal_name,
     477        title:displayed_name}
     478        """
     479        result = [dict(title='<IGNORE COL>', value='--IGNORE--')]
     480        headers = self.importer.getHeaders()
     481        result.extend([dict(title=x, value=x) for x in headers])
     482        return result
     483
     484    def getWarnings(self):
     485        import sys
     486        result = []
     487        try:
     488            self.importer.checkHeaders(self.headerfields, mode=self.mode)
     489        except:
     490            fatal = '%s' % sys.exc_info()[1]
     491            result.append(fatal)
     492        return result
     493   
     494    @property
     495    def nextstep(self):
     496        return self.url(self.context, '@@import4')
     497
     498    def update(self, headerfield=None, back2=None, cancel=None, proceed=None):
     499        session = ISession(self.request)['waeup.sirp']
     500        self.filename = session.get('import_filename', None)
     501        self.mode = session.get('import_mode', None)
     502        self.importername = session.get('import_importer', None)
     503       
     504        if None in (self.filename, self.mode, self.importername):
     505            self.redirect(self.url(self.context, '@@import2'))
     506            return
     507        if back2 is not None:
     508            self.redirect(self.url(self.context ,'@@import2'))
     509            return
     510        if cancel is not None:
     511            self.flash('Import aborted.')
     512            self.redirect(self.url(self.context))
     513            return
     514
     515        self.fullpath = os.path.join(self.context.storage, self.filename)
     516        self.headerfields = headerfield or self.getPreviewHeader()
     517        session['import_headerfields'] = self.headerfields
     518
     519        if proceed is not None:
     520            self.redirect(self.url(self.context, '@@import4'))
     521            return
     522       
     523        self.importer = getUtility(IBatchProcessor, name=self.importername)
     524        self.reader = csv.DictReader(open(self.fullpath, 'rb'))
     525
     526class DatacenterImportStep4(WAeUPPage):
     527    """Manual import step 4: do actual import
     528    """
     529    grok.context(IDataCenter)
     530    grok.name('import4')
     531    grok.template('datacenterimport4page')
     532    grok.require('waeup.manageUniversity')
     533    title = u'Process CSV file'
     534    pnav = 0
     535
     536    filename = None
     537    mode = None
     538    importername = None
     539    headerfields = None
     540    warnnum = None
     541
     542    def update(self, back=None, finish=None, showlog=None):
     543        if finish is not None:
     544            self.redirect(self.url(self.context))
     545            return
     546        if back is not None:
     547            self.redirect(self.url(self.context, '@@import3'))
     548            return
     549        session = ISession(self.request)['waeup.sirp']
     550        self.filename = session.get('import_filename', None)
     551        self.mode = session.get('import_mode', None)
     552        self.importername = session.get('import_importer', None)
     553        self.headerfields = session.get('import_headerfields', None)
     554       
     555        if None in (self.filename, self.mode, self.importername,
     556                    self.headerfields):
     557            self.redirect(self.url(self.context, '@@import3'))
     558            return
     559
     560        if showlog is not None:
     561            logfilename = "%s.%s.msg" % (self.filename, self.mode)
     562            session['logname'] = logfilename
     563            self.redirect(self.url(self.context, '@@show'))
     564            return
     565           
     566        self.fullpath = os.path.join(self.context.storage, self.filename)
     567        self.importer = getUtility(IBatchProcessor, name=self.importername)
     568        (linenum, warnings) = self.importer.doImport(
     569            self.fullpath, self.headerfields, self.mode,
     570            self.request.principal.id)
     571        self.warn_num = len(warnings)
     572        if self.warn_num:
     573            self.flash('Processing of %d rows failed!' % self.warn_num)
     574        self.flash('Successfully processed %s rows' % (
     575                linenum - (self.warn_num)))
     576
     577class DatacenterLogsOverview(WAeUPPage):
     578    grok.context(IDataCenter)
     579    grok.name('logs')
     580    grok.template('datacenterlogspage')
     581    grok.require('waeup.manageUniversity')
     582    title = u'Data Center Logs'
     583    pnav = 0
     584
     585    def update(self, show=None, remove=None, logname=None, back=None):
     586        session = ISession(self.request)['waeup.sirp']
     587        if back is not None:
     588            self.redirect(self.url(self.context))
     589            return
     590        if logname is not None:
     591            session['logname'] = logname
     592            if remove is not None:
     593                fullpath = os.path.join(self.context.storage, logname)
     594                try:
     595                    os.unlink(fullpath)
     596                    self.flash('File %s deleted.' % logname)
     597                except:
     598                    self.flash("Could not delete %s: " % logname)
     599                    self.flash("Problem: %s" % sys.exc_info()[1])
     600
     601        if show is not None:
     602            self.redirect(self.url(self.context, '@@show'))
     603            return
     604        self.files = self.context.getLogFiles()
     605
     606class DatacenterLogsFileview(WAeUPPage):
     607    grok.context(IDataCenter)
     608    grok.name('show')
     609    grok.template('datacenterlogsshowfilepage')
     610    grok.require('waeup.manageUniversity')
     611    title = u'Show file'
     612    pnav = 0
     613
     614    def update(self, show=None, remove=None, back=None):
     615        session = ISession(self.request)['waeup.sirp']
     616        logname = session.get('logname', None)
     617        if back is not None or logname is None:
     618            self.redirect(self.url(self.context, '@@logs'))
     619            return
     620        self.filename = logname
     621        self.files = self.context.getLogFiles()
     622        fullpath = os.path.join(self.context.storage, logname)
     623        self.filecontents = open(fullpath, 'rb').read()
     624
    312625class DatacenterSettings(WAeUPPage):
    313626    grok.context(IDataCenter)
    … …  
    317630    title = u'Data Center Settings'
    318631    pnav = 0
    319     #grok.template('master')
    320632
    321633    def update(self, newpath=None, move=False, overwrite=False,

  • waeup/trunk/src/waeup/browser/static/purple.css

    r4789 r4858  
    1414
    1515/***** Uli Stuff *****/
     16
     17/* Make YUI button icons appear vertically centered... */
     18.yui-skin-sam .yui-button a img {
     19  vertical-align: middle;
     20  margin-right: 3px;
     21  margin-top: -1px;
     22  margin-left: -2px;
     23}
    1624
    1725/* Fix logo image height: as image is taller than h1 text, header area

  • waeup/trunk/src/waeup/browser/viewlets.py

    r4789 r4858  
    369369    target = 'addcertificatecourse'
    370370
    371 
     371#
     372# Actions with a 'browse' icon...
     373#
     374class BrowseActionButton(ActionButton):
     375    grok.baseclass()
     376    grok.context(IWAeUPObject)
     377    grok.template('actionbutton')
     378    grok.viewletmanager(ActionBar)
     379    grok.require('waeup.manageUniversity')
     380    icon = 'actionicon_manage.png' # File must exist in static/
     381    target = '@@show' # link to this viewname.
     382    text = 'Show batch logs' # Text to display on the button
     383
     384class BrowseDatacenterLogs(BrowseActionButton):
     385    grok.context(IDataCenter)
     386    grok.view(DatacenterPage)
     387    grok.order(4)
     388    icon = 'documentinfo_templet.png'
     389    target = '@@logs'
     390    text = 'Show batch logs'
     391
     392class BatchOpButton(ActionButton):
     393    grok.context(IDataCenter)
     394    grok.view(DatacenterPage)
     395    grok.order(6)
     396    icon = 'actionbox_templet.png'
     397    target = '@@import1'
     398    text = 'Batch processing'
     399   
    372400#
    373401# Primary navigation tabs (in upper left navigation bar)...

  • waeup/trunk/src/waeup/datacenter.py

    r4789 r4858  
    6363        return result
    6464   
    65     def getFiles(self):
     65    def getFiles(self, sort='name'):
    6666        """Get a list of files stored in `storage`.
    6767
    … …  
    7676                continue
    7777            result.append(DataCenterFile(fullpath))
    78         return result
    79 
     78        if sort == 'date':
     79            # sort results in newest-first order...
     80            result = sorted(result, key=lambda x: x.getTimeStamp(),
     81                            reverse=True)
     82        return result
     83
     84    def getLogFiles(self):
     85        """Get a list of .msg files.
     86        """
     87        result = []
     88        files = self.getFiles()
     89        for file in files:
     90            if not file.name.endswith('.msg'):
     91                continue
     92            result.append(
     93                LogFile(os.path.join(self.storage, file.name)))
     94        return result
     95           
    8096    def setStoragePath(self, path, move=False, overwrite=False):
    8197        """Set the path where to store files.
    … …  
    179195        return
    180196
     197
    181198class DataCenterFile(object):
    182199    """A description of a file stored in data center.
    … …  
    189206        self.size = self.getSize()
    190207        self.uploaddate = self.getDate()
     208        self.lines = self.getLinesNumber()
    191209
    192210    def getDate(self):
    … …  
    196214        return date.strftime('%c')
    197215
     216    def getTimeStamp(self):
     217        """Get a (machine readable) timestamp.
     218        """
     219        return os.path.getctime(self.context)
     220   
    198221    def getSize(self):
    199222        """Get a human readable file size.
    … …  
    209232        return size
    210233
     234    def getLinesNumber(self):
     235        """Get number of lines.
     236        """
     237        num = 0
     238        for line in open(self.context, 'rb'):
     239            num += 1
     240        return num
     241   
     242class LogFile(DataCenterFile):
     243    """A description of a log file.
     244    """
     245    def __init__(self, context):
     246        super(LogFile, self).__init__(context)
     247        self._markers = dict()
     248        self._parsed = False
     249        self.userid = self.getUserId()
     250        self.mode = self.getMode()
     251        self.stats = self.getStats()
     252        self.source = self.getSourcePath()
     253
     254    def _parseFile(self, maxline=10):
     255        """Find markers in a file.
     256        """
     257        if self._parsed:
     258            return
     259        for line in open(self.context, 'rb'):
     260            line = line.strip()
     261            if not ':' in line:
     262                continue
     263            name, text = line.split(':', 1)
     264            self._markers[name.lower()] = text
     265        self._parsed = True
     266        return
     267
     268    def _getMarker(self, marker):
     269        marker = marker.lower()
     270        if not self._parsed:
     271            self._parseFile()
     272        if marker in self._markers.keys():
     273            return self._markers[marker]
     274   
     275    def getUserId(self):
     276        return self._getMarker('user') or '<UNKNOWN>'
     277
     278    def getMode(self):
     279        return self._getMarker('mode') or '<NOT SET>'
     280
     281    def getStats(self):
     282        return self._getMarker('processed') or '<Info not avail.>'
     283
     284    def getSourcePath(self):
     285        return self._getMarker('source') or None
     286   
    211287class Import(object):
    212288    """Helper class to aggregate imports and their data.

  • waeup/trunk/src/waeup/interfaces.py

    r4789 r4858  
    99from waeup.permissions import RoleSource
    1010
     11class FatalCSVError(Exception):
     12    """Some row could not be processed.
     13    """
     14    pass
     15
    1116def SimpleWAeUPVocabulary(*terms):
    1217    """A well-buildt vocabulary provides terms with a value, token and
    … …  
    2126       up a catalog.
    2227    """
    23     catalog = None
    2428    def getValues(self):
    25         if self.catalog is None:
    26             self.catalog = getUtility(ICatalog, name='courses_catalog')
    27         return list(self.catalog.searchResults(code=('', 'z*')))
     29        catalog = getUtility(ICatalog, name='courses_catalog')
     30        return list(catalog.searchResults(code=('', 'z*')))
     31
     32    def getToken(self, value):
     33        return value.code
    2834       
    2935    def getTitle(self, value):
    … …  
    201207    review_state = schema.Choice(
    202208        title = u'review state',
     209        default = 'unchecked',
    203210        values = ['unchecked', 'checked']
    204211        )
    … …  
    348355        `clear_old_data` is set to True.
    349356        """
    350        
     357
     358class IBatchProcessor(Interface):
     359    """A batch processor that handles mass-operations.
     360    """
     361    name = schema.TextLine(
     362        title = u'Importer name'
     363        )
     364
     365    mode = schema.Choice(
     366        title = u'Import mode',
     367        values = ['create', 'update', 'remove']
     368        )
     369   
     370    def doImport(path):
     371        """Read data from ``path`` and update connected object.
     372        """
     373
     374class ISchemaTypeConverter(Interface):
     375    """A converter for zope.schema.types.
     376    """
     377    def convert(string):
     378        """Convert string to certain schema type.
     379        """
     380
     381   
    351382class IUserAccount(IWAeUPObject):
    352383    """A user account.
    … …  
    403434    """A data center file.
    404435    """
     436
     437    name = schema.TextLine(
     438        title = u'Filename')
     439
     440    size = schema.TextLine(
     441        title = u'Human readable file size')
     442
     443    uploaddate = schema.TextLine(
     444        title = u'Human readable upload datetime')
     445
     446    lines = schema.Int(
     447        title = u'Number of lines in file')
     448       
    405449    def getDate():
    406450        """Get creation timestamp from file in human readable form.
    … …  
    410454        """Get human readable size of file.
    411455        """
     456
     457    def getLinesNumber():
     458        """Get number of lines of file.
     459        """

  • waeup/trunk/src/waeup/utils/api.txt

    r4789 r4858  
    1515
    1616   helpers.txt
     17   batching.txt
    1718   importexport.txt
    1819   csvimport.txt
     20   converters.txt
Note: See TracChangeset for help on using the changeset viewer.