source: waeup/branches/ulif-layout/src/waeup/datacenter.py @ 4574

Last change on this file since 4574 was 4574, checked in by uli, 15 years ago

Apply last fixes to ulif-rewrite to this branch as well.
File size: 11.0 KB
Line 
1"""WAeUP data center.
2
3The waeup data center cares for management of upload data and provides
4tools for importing/exporting CSV data.
5"""
6import os
7import shutil
8import struct
9import sys
10import grok
11from datetime import datetime
12from zope.component import getUtility, getMultiAdapter
13from zope.interface import Interface
14from waeup.csvfile import getCSVFile
15from waeup.interfaces import (IWAeUPObject, IWAeUPCSVImporter,
16                              ICSVDataReceivers, IDataCenterFile)
17from waeup.utils.helpers import copyFileSystemTree
18from waeup.viewlets import Index, MainArea, LeftSidebar
19
20class DataCenter(grok.Container):
21    """A data center contains CSV files.
22    """
23    storage = os.path.join(os.path.dirname(__file__), 'files')
24
25    def getReceivers(self):
26        receivers = []
27        curr_obj = getattr(self, '__parent__', None)
28        while curr_obj is not None:
29            if ICSVDataReceivers.providedBy(curr_obj):
30                receivers = self.getCSVDataReceivers(curr_obj)
31                break
32            curr_obj = getattr(curr_obj, '__parent__', None)
33        return receivers
34
35    def getReceiverIds(self):
36        """Get a dict of available receivers.
37
38        The keys of the result are the receiver ids.
39        """
40        receivers = self.getReceivers()
41        return dict([(self.getReceiverId(x), x) for x in receivers])
42   
43    def getImporters(self):
44        """Get a list of all importers available.
45
46        The search for available importers is done in two steps:
47
48        1) Look for a utility providing ICSVDataReceiver,
49
50        2) For every item of that utility: try to get an adapter
51           providing IWAeUPCSVImporter.
52        """
53        result = []
54        receivers = self.getReceivers()
55        files = self.getFiles()
56        for receiver in receivers:
57            for file in files:
58                wrapped_file = getCSVFile(file.context)
59                try:
60                    importer = getMultiAdapter((wrapped_file, receiver),
61                                           IWAeUPCSVImporter)
62                    result.append(importer)
63                except:
64                    # No multi-adapter for this combination available...
65                    pass
66        return result
67   
68    def getFiles(self):
69        """Get a list of files stored in `storage`.
70
71        Files are sorted by basename.
72        """
73        result = []
74        if not os.path.exists(self.storage):
75            return result
76        for filename in sorted(os.listdir(self.storage)):
77            fullpath = os.path.join(self.storage, filename)
78            if not os.path.isfile(fullpath):
79                continue
80            result.append(DataCenterFile(fullpath))
81        return result
82
83    def setStoragePath(self, path, move=False, overwrite=False):
84        """Set the path where to store files.
85        """
86        path = os.path.abspath(path)
87        not_copied = []
88        if not os.path.exists(path):
89            raise ValueError('The path given does not exist: %s' % path)
90        if move is True:
91
92            not_copied = copyFileSystemTree(self.storage, path,
93                                            overwrite=overwrite)
94       
95        self.storage = path
96        return not_copied
97
98    def getCSVDataReceivers(self, obj):
99        """Get a list of attributes, that can receive CSV data.
100        """
101        result = []
102        for attr_name in dir(obj):
103            if attr_name.startswith('_'):
104                continue
105            try:
106                attr = getattr(obj, attr_name)
107                # This might fail...
108                #IWAeUPCSVImporter(attr)
109                result.append(attr)
110            except:
111                pass
112        return result
113
114    def getPossibleImports(self):
115        """Get list of possible imports.
116
117        A single import is defined as a tuple
118       
119          ( <file-descr>, <importers> )
120       
121        where ``<file-descr>`` is an `IDataCenterFile` object and
122        ``<importers>`` is a list of `IWAeUPCSVImporter` objects.
123        """
124        result = []
125        importers = self.getImporters()
126        for filedescr in self.getFiles():
127            possible_importers = []
128            for importer in importers:
129                if importer.csvfile.path != filedescr.context:
130                    continue
131                importer_context = (
132                    importer, self.getReceiverId(importer.receiver))
133                possible_importers.append(importer_context)
134            if len(possible_importers) == 0:
135                continue
136            result.append((filedescr, possible_importers))
137        return result
138
139    def getReceiverId(self, obj):
140        """Get a unique id for an object.
141
142        If the object is stored in ZODB it should contain a `_p_oid`
143        attribute, which is guaranteed to be unique over the ZODB.
144
145        If the object has no such attribute, then it will be held in
146        memory only and will vanish as soon as the request is over. In
147        this case we can use the memory address of it.
148
149        Both, the id() result and the ZODB oid, are actually integers
150        which we return as strings. To make the difference chrystal
151        clear, we prepend ``z`` to ids taken from ZODB oids.
152        """
153        if not hasattr(obj, '_p_oid'):
154            return str(id(obj))
155        oid = getattr(obj, '_p_oid')
156        if oid is None:
157            # The persistent object is not stored in the ZODB
158            return str(id(obj))
159        return 'z%s' % struct.unpack('>Q', oid)[0]
160
161    def doImport(self, path, receiverid, clear=None):
162        receivers = self.getReceiverIds()
163        if receiverid not in receivers.keys():
164            raise ValueError('The requested data receiver cannot be found.')
165        receiver = receivers[receiverid]
166        filewrapper = getCSVFile(path)
167        if filewrapper is None:
168            raise ValueError('Format of CSV file not supported.')
169        importer = getMultiAdapter((filewrapper, receiver), IWAeUPCSVImporter)
170        if clear is not None:
171            importer.doImport(clear_old_data=clear)
172        else:
173            importer.doImport()
174        return
175
176class DataCenterFile(object):
177    """A description of a file stored in data center.
178    """
179    grok.implements(IDataCenterFile)
180   
181    def __init__(self, context):
182        self.context = context
183        self.name = os.path.basename(self.context)
184        self.size = self.getSize()
185        self.uploaddate = self.getDate()
186
187    def getDate(self):
188        """Get a human readable datetime representation.
189        """
190        date = datetime.fromtimestamp(os.path.getctime(self.context))
191        return date.strftime('%c')
192
193    def getSize(self):
194        """Get a human readable file size.
195        """
196        bytesize = os.path.getsize(self.context)
197        size = "%s bytes" % bytesize
198        units = ['kb', 'MB', 'GB']
199        for power, unit in reversed(list(enumerate(units))):
200            power += 1
201            if bytesize >= 1024 ** power:
202                size = "%.2f %s" % (bytesize/(1024.0**power), unit)
203                break
204        return size
205
206
207class Content(grok.Viewlet):
208    grok.viewletmanager(MainArea)
209    grok.context(DataCenter)
210    grok.view(Index)
211
212class Upload(grok.View):
213    grok.context(DataCenter)
214    grok.name('upload')
215    grok.template('master')
216
217    def update(self, uploadfile=None, CANCEL=None, SUBMIT=None):
218        if CANCEL is not None:
219            self.redirect(self.url(self.context))
220            return
221        if not uploadfile:
222            return
223        try:
224            filename = uploadfile.filename
225            target = os.path.join(self.context.storage, filename)
226            open(target, 'wb').write(uploadfile.read())
227        except IOError:
228            self.flash('Error while uploading file. Please retry.')
229            self.flash('I/O error: %s' % sys.exc_info()[1])
230            return
231        self.redirect(self.url(self.context))
232   
233class UploadMain(grok.Viewlet):
234    grok.viewletmanager(MainArea)
235    grok.context(DataCenter)
236    grok.view(Upload)
237   
238
239class Settings(grok.View):
240    grok.context(DataCenter)
241    grok.name('settings')
242    grok.template('master')
243
244    def update(self, newpath=None, move=False, overwrite=False,
245               save=None, cancel=None):
246        if move:
247            move = True
248        if overwrite:
249            overwrite = True
250        if newpath is None:
251            return
252        if cancel is not None:
253            self.redirect(self.url(self.context))
254            return
255        try:
256            not_copied = self.context.setStoragePath(newpath, move=move)
257            for name in not_copied:
258                self.flash('File already existed (not copied): %s' % name)
259        except ValueError:
260            self.flash('Given storage path cannot be used.')
261            self.flash('Error: %s' %sys.exc_info()[1])
262            return
263        if newpath:
264            self.flash('New storage path succefully set.')
265            self.redirect(self.url(self.context))
266        return
267
268class SettingsMain(grok.Viewlet):
269    grok.viewletmanager(MainArea)
270    grok.context(DataCenter)
271    grok.view(Settings)
272
273class SettingsLink(grok.Viewlet):
274    grok.viewletmanager(LeftSidebar)
275    grok.context(DataCenter)
276    grok.view(Index)
277    grok.order(2)
278    grok.require('waeup.manageUniversity')
279
280    def render(self):
281        return u'<div class="portlet"><a href="@@settings">Settings</a></div>'
282
283class ImportCSV(grok.View):
284    """A view for importing things.
285
286    """
287    grok.require('waeup.manageUniversity')
288    grok.template('master')
289    grok.context(DataCenter)
290
291    def update(self, csvfile=None, clear=None, overwrite=None,
292               receiverid=None, CANCEL=None, SUBMIT=None):
293        if CANCEL is not None:
294            self.redirect(self.url(self.context))
295            return
296        if not csvfile:
297            return
298        if not SUBMIT:
299            return
300        try:
301            self.context.doImport(csvfile, receiverid, clear=clear)
302        except ValueError:
303            self.flash('Could not import: %s' % os.path.basename(csvfile))
304            return
305        except:
306            self.flash('Import failed: %s %s' % (sys.exc_info()[0],
307                                                 sys.exc_info()[1]))
308            return
309        self.flash('Successfully imported: %s' % os.path.basename(csvfile))
310        self.redirect(self.url(self.context))
311        return
312
313class Import(object):
314    """Helper class to aggregate imports and their data.
315    """
316    def __init__(self, filedescr, importers):
317        self.file = filedescr
318        self.importers = []
319        for importer, receiverid in importers:
320            importer.receiverid = receiverid
321            self.importers.append(importer)
322       
323class ImportCSVMain(grok.Viewlet):
324    grok.viewletmanager(MainArea)
325    grok.context(DataCenter)
326    grok.view(ImportCSV)
327    grok.require('waeup.manageUniversity')
328
329    def getImports(self):
330        result = []
331        imports = self.context.getPossibleImports()
332        result = [Import(x, y) for x, y in imports]
333        return result
334   
335    def update(self, *args, **kw):
336        return super(ImportCSVMain, self).update(*args, **kw)
337       
Note: See TracBrowser for help on using the repository browser.