source: waeup/branches/ulif-namespace/src/sirp/datacenter.py @ 4791

Last change on this file since 4791 was 4789, checked in by uli, 15 years ago

Merge changes from ulif-layout back into trunk (finally).

File size: 7.4 KB
Line 
1"""WAeUP data center.
2
3The waeup data center cares for management of upload data and provides
4tools for importing/exporting CSV data.
5"""
6import os
7import struct
8import grok
9from datetime import datetime
10from zope.component import getMultiAdapter
11from waeup.csvfile import getCSVFile
12from waeup.interfaces import (IDataCenter, IWAeUPCSVImporter,
13                              ICSVDataReceivers, IDataCenterFile)
14from waeup.utils.helpers import copyFileSystemTree
15
16class DataCenter(grok.Container):
17    """A data center contains CSV files.
18    """
19    grok.implements(IDataCenter)
20    storage = os.path.join(os.path.dirname(__file__), 'files')
21
22    def getReceivers(self):
23        receivers = []
24        curr_obj = getattr(self, '__parent__', None)
25        while curr_obj is not None:
26            if ICSVDataReceivers.providedBy(curr_obj):
27                receivers = self.getCSVDataReceivers(curr_obj)
28                break
29            curr_obj = getattr(curr_obj, '__parent__', None)
30        return receivers
31
32    def getReceiverIds(self):
33        """Get a dict of available receivers.
34
35        The keys of the result are the receiver ids.
36        """
37        receivers = self.getReceivers()
38        return dict([(self.getReceiverId(x), x) for x in receivers])
39   
40    def getImporters(self):
41        """Get a list of all importers available.
42
43        The search for available importers is done in two steps:
44
45        1) Look for a utility providing ICSVDataReceiver,
46
47        2) For every item of that utility: try to get an adapter
48           providing IWAeUPCSVImporter.
49        """
50        result = []
51        receivers = self.getReceivers()
52        files = self.getFiles()
53        for receiver in receivers:
54            for file in files:
55                wrapped_file = getCSVFile(file.context)
56                try:
57                    importer = getMultiAdapter((wrapped_file, receiver),
58                                           IWAeUPCSVImporter)
59                    result.append(importer)
60                except:
61                    # No multi-adapter for this combination available...
62                    pass
63        return result
64   
65    def getFiles(self):
66        """Get a list of files stored in `storage`.
67
68        Files are sorted by basename.
69        """
70        result = []
71        if not os.path.exists(self.storage):
72            return result
73        for filename in sorted(os.listdir(self.storage)):
74            fullpath = os.path.join(self.storage, filename)
75            if not os.path.isfile(fullpath):
76                continue
77            result.append(DataCenterFile(fullpath))
78        return result
79
80    def setStoragePath(self, path, move=False, overwrite=False):
81        """Set the path where to store files.
82        """
83        path = os.path.abspath(path)
84        not_copied = []
85        if not os.path.exists(path):
86            raise ValueError('The path given does not exist: %s' % path)
87        if move is True:
88
89            not_copied = copyFileSystemTree(self.storage, path,
90                                            overwrite=overwrite)
91       
92        self.storage = path
93        return not_copied
94
95    def getCSVDataReceivers(self, obj):
96        """Get a list of attributes, that can receive CSV data.
97
98        We also get a list of values, if obj is a container that
99        contains value items.
100        """
101        result = []
102        for attr_name in dir(obj):
103            if attr_name.startswith('_'):
104                continue
105            try:
106                attr = getattr(obj, attr_name)
107                # This might fail...
108                #IWAeUPCSVImporter(attr)
109                result.append(attr)
110            except:
111                pass
112        if hasattr(obj, 'values'):
113            try:
114                result.extend(obj.values())
115            except:
116                pass
117        return result
118
119    def getPossibleImports(self):
120        """Get list of possible imports.
121
122        A single import is defined as a tuple
123       
124          ( <file-descr>, <importers> )
125       
126        where ``<file-descr>`` is an `IDataCenterFile` object and
127        ``<importers>`` is a list of `IWAeUPCSVImporter` objects.
128        """
129        result = []
130        importers = self.getImporters()
131        for filedescr in self.getFiles():
132            possible_importers = []
133            for importer in importers:
134                if importer.csvfile.path != filedescr.context:
135                    continue
136                importer_context = (
137                    importer, self.getReceiverId(importer.receiver))
138                possible_importers.append(importer_context)
139            if len(possible_importers) == 0:
140                continue
141            result.append((filedescr, possible_importers))
142        return result
143
144    def getReceiverId(self, obj):
145        """Get a unique id for an object.
146
147        If the object is stored in ZODB it should contain a `_p_oid`
148        attribute, which is guaranteed to be unique over the ZODB.
149
150        If the object has no such attribute, then it will be held in
151        memory only and will vanish as soon as the request is over. In
152        this case we can use the memory address of it.
153
154        Both, the id() result and the ZODB oid, are actually integers
155        which we return as strings. To make the difference chrystal
156        clear, we prepend ``z`` to ids taken from ZODB oids.
157        """
158        if not hasattr(obj, '_p_oid'):
159            return str(id(obj))
160        oid = getattr(obj, '_p_oid')
161        if oid is None:
162            # The persistent object is not stored in the ZODB
163            return str(id(obj))
164        return 'z%s' % struct.unpack('>Q', oid)[0]
165
166    def doImport(self, path, receiverid, clear=None):
167        receivers = self.getReceiverIds()
168        if receiverid not in receivers.keys():
169            raise ValueError('The requested data receiver cannot be found.')
170        receiver = receivers[receiverid]
171        filewrapper = getCSVFile(path)
172        if filewrapper is None:
173            raise ValueError('Format of CSV file not supported.')
174        importer = getMultiAdapter((filewrapper, receiver), IWAeUPCSVImporter)
175        if clear is not None:
176            importer.doImport(clear_old_data=clear)
177        else:
178            importer.doImport()
179        return
180
181class DataCenterFile(object):
182    """A description of a file stored in data center.
183    """
184    grok.implements(IDataCenterFile)
185   
186    def __init__(self, context):
187        self.context = context
188        self.name = os.path.basename(self.context)
189        self.size = self.getSize()
190        self.uploaddate = self.getDate()
191
192    def getDate(self):
193        """Get a human readable datetime representation.
194        """
195        date = datetime.fromtimestamp(os.path.getctime(self.context))
196        return date.strftime('%c')
197
198    def getSize(self):
199        """Get a human readable file size.
200        """
201        bytesize = os.path.getsize(self.context)
202        size = "%s bytes" % bytesize
203        units = ['kb', 'MB', 'GB']
204        for power, unit in reversed(list(enumerate(units))):
205            power += 1
206            if bytesize >= 1024 ** power:
207                size = "%.2f %s" % (bytesize/(1024.0**power), unit)
208                break
209        return size
210
211class Import(object):
212    """Helper class to aggregate imports and their data.
213    """
214    def __init__(self, filedescr, importers):
215        self.file = filedescr
216        self.importers = []
217        for importer, receiverid in importers:
218            importer.receiverid = receiverid
219            self.importers.append(importer)
Note: See TracBrowser for help on using the repository browser.