1 | """WAeUP data center. |
---|
2 | |
---|
3 | The waeup data center cares for management of upload data and provides |
---|
4 | tools for importing/exporting CSV data. |
---|
5 | """ |
---|
6 | import os |
---|
7 | import struct |
---|
8 | import grok |
---|
9 | from datetime import datetime |
---|
10 | from zope.component import getMultiAdapter |
---|
11 | from waeup.csvfile import getCSVFile |
---|
12 | from waeup.interfaces import (IDataCenter, IWAeUPCSVImporter, |
---|
13 | ICSVDataReceivers, IDataCenterFile) |
---|
14 | from waeup.utils.helpers import copyFileSystemTree |
---|
15 | |
---|
16 | class DataCenter(grok.Container): |
---|
17 | """A data center contains CSV files. |
---|
18 | """ |
---|
19 | grok.implements(IDataCenter) |
---|
20 | storage = os.path.join(os.path.dirname(__file__), 'files') |
---|
21 | |
---|
22 | def getReceivers(self): |
---|
23 | receivers = [] |
---|
24 | curr_obj = getattr(self, '__parent__', None) |
---|
25 | while curr_obj is not None: |
---|
26 | if ICSVDataReceivers.providedBy(curr_obj): |
---|
27 | receivers = self.getCSVDataReceivers(curr_obj) |
---|
28 | break |
---|
29 | curr_obj = getattr(curr_obj, '__parent__', None) |
---|
30 | return receivers |
---|
31 | |
---|
32 | def getReceiverIds(self): |
---|
33 | """Get a dict of available receivers. |
---|
34 | |
---|
35 | The keys of the result are the receiver ids. |
---|
36 | """ |
---|
37 | receivers = self.getReceivers() |
---|
38 | return dict([(self.getReceiverId(x), x) for x in receivers]) |
---|
39 | |
---|
40 | def getImporters(self): |
---|
41 | """Get a list of all importers available. |
---|
42 | |
---|
43 | The search for available importers is done in two steps: |
---|
44 | |
---|
45 | 1) Look for a utility providing ICSVDataReceiver, |
---|
46 | |
---|
47 | 2) For every item of that utility: try to get an adapter |
---|
48 | providing IWAeUPCSVImporter. |
---|
49 | """ |
---|
50 | result = [] |
---|
51 | receivers = self.getReceivers() |
---|
52 | files = self.getFiles() |
---|
53 | for receiver in receivers: |
---|
54 | for file in files: |
---|
55 | wrapped_file = getCSVFile(file.context) |
---|
56 | try: |
---|
57 | importer = getMultiAdapter((wrapped_file, receiver), |
---|
58 | IWAeUPCSVImporter) |
---|
59 | result.append(importer) |
---|
60 | except: |
---|
61 | # No multi-adapter for this combination available... |
---|
62 | pass |
---|
63 | return result |
---|
64 | |
---|
65 | def getFiles(self): |
---|
66 | """Get a list of files stored in `storage`. |
---|
67 | |
---|
68 | Files are sorted by basename. |
---|
69 | """ |
---|
70 | result = [] |
---|
71 | if not os.path.exists(self.storage): |
---|
72 | return result |
---|
73 | for filename in sorted(os.listdir(self.storage)): |
---|
74 | fullpath = os.path.join(self.storage, filename) |
---|
75 | if not os.path.isfile(fullpath): |
---|
76 | continue |
---|
77 | result.append(DataCenterFile(fullpath)) |
---|
78 | return result |
---|
79 | |
---|
80 | def setStoragePath(self, path, move=False, overwrite=False): |
---|
81 | """Set the path where to store files. |
---|
82 | """ |
---|
83 | path = os.path.abspath(path) |
---|
84 | not_copied = [] |
---|
85 | if not os.path.exists(path): |
---|
86 | raise ValueError('The path given does not exist: %s' % path) |
---|
87 | if move is True: |
---|
88 | |
---|
89 | not_copied = copyFileSystemTree(self.storage, path, |
---|
90 | overwrite=overwrite) |
---|
91 | |
---|
92 | self.storage = path |
---|
93 | return not_copied |
---|
94 | |
---|
95 | def getCSVDataReceivers(self, obj): |
---|
96 | """Get a list of attributes, that can receive CSV data. |
---|
97 | |
---|
98 | We also get a list of values, if obj is a container that |
---|
99 | contains value items. |
---|
100 | """ |
---|
101 | result = [] |
---|
102 | for attr_name in dir(obj): |
---|
103 | if attr_name.startswith('_'): |
---|
104 | continue |
---|
105 | try: |
---|
106 | attr = getattr(obj, attr_name) |
---|
107 | # This might fail... |
---|
108 | #IWAeUPCSVImporter(attr) |
---|
109 | result.append(attr) |
---|
110 | except: |
---|
111 | pass |
---|
112 | if hasattr(obj, 'values'): |
---|
113 | try: |
---|
114 | result.extend(obj.values()) |
---|
115 | except: |
---|
116 | pass |
---|
117 | return result |
---|
118 | |
---|
119 | def getPossibleImports(self): |
---|
120 | """Get list of possible imports. |
---|
121 | |
---|
122 | A single import is defined as a tuple |
---|
123 | |
---|
124 | ( <file-descr>, <importers> ) |
---|
125 | |
---|
126 | where ``<file-descr>`` is an `IDataCenterFile` object and |
---|
127 | ``<importers>`` is a list of `IWAeUPCSVImporter` objects. |
---|
128 | """ |
---|
129 | result = [] |
---|
130 | importers = self.getImporters() |
---|
131 | for filedescr in self.getFiles(): |
---|
132 | possible_importers = [] |
---|
133 | for importer in importers: |
---|
134 | if importer.csvfile.path != filedescr.context: |
---|
135 | continue |
---|
136 | importer_context = ( |
---|
137 | importer, self.getReceiverId(importer.receiver)) |
---|
138 | possible_importers.append(importer_context) |
---|
139 | if len(possible_importers) == 0: |
---|
140 | continue |
---|
141 | result.append((filedescr, possible_importers)) |
---|
142 | return result |
---|
143 | |
---|
144 | def getReceiverId(self, obj): |
---|
145 | """Get a unique id for an object. |
---|
146 | |
---|
147 | If the object is stored in ZODB it should contain a `_p_oid` |
---|
148 | attribute, which is guaranteed to be unique over the ZODB. |
---|
149 | |
---|
150 | If the object has no such attribute, then it will be held in |
---|
151 | memory only and will vanish as soon as the request is over. In |
---|
152 | this case we can use the memory address of it. |
---|
153 | |
---|
154 | Both, the id() result and the ZODB oid, are actually integers |
---|
155 | which we return as strings. To make the difference chrystal |
---|
156 | clear, we prepend ``z`` to ids taken from ZODB oids. |
---|
157 | """ |
---|
158 | if not hasattr(obj, '_p_oid'): |
---|
159 | return str(id(obj)) |
---|
160 | oid = getattr(obj, '_p_oid') |
---|
161 | if oid is None: |
---|
162 | # The persistent object is not stored in the ZODB |
---|
163 | return str(id(obj)) |
---|
164 | return 'z%s' % struct.unpack('>Q', oid)[0] |
---|
165 | |
---|
166 | def doImport(self, path, receiverid, clear=None): |
---|
167 | receivers = self.getReceiverIds() |
---|
168 | if receiverid not in receivers.keys(): |
---|
169 | raise ValueError('The requested data receiver cannot be found.') |
---|
170 | receiver = receivers[receiverid] |
---|
171 | filewrapper = getCSVFile(path) |
---|
172 | if filewrapper is None: |
---|
173 | raise ValueError('Format of CSV file not supported.') |
---|
174 | importer = getMultiAdapter((filewrapper, receiver), IWAeUPCSVImporter) |
---|
175 | if clear is not None: |
---|
176 | importer.doImport(clear_old_data=clear) |
---|
177 | else: |
---|
178 | importer.doImport() |
---|
179 | return |
---|
180 | |
---|
181 | class DataCenterFile(object): |
---|
182 | """A description of a file stored in data center. |
---|
183 | """ |
---|
184 | grok.implements(IDataCenterFile) |
---|
185 | |
---|
186 | def __init__(self, context): |
---|
187 | self.context = context |
---|
188 | self.name = os.path.basename(self.context) |
---|
189 | self.size = self.getSize() |
---|
190 | self.uploaddate = self.getDate() |
---|
191 | |
---|
192 | def getDate(self): |
---|
193 | """Get a human readable datetime representation. |
---|
194 | """ |
---|
195 | date = datetime.fromtimestamp(os.path.getctime(self.context)) |
---|
196 | return date.strftime('%c') |
---|
197 | |
---|
198 | def getSize(self): |
---|
199 | """Get a human readable file size. |
---|
200 | """ |
---|
201 | bytesize = os.path.getsize(self.context) |
---|
202 | size = "%s bytes" % bytesize |
---|
203 | units = ['kb', 'MB', 'GB'] |
---|
204 | for power, unit in reversed(list(enumerate(units))): |
---|
205 | power += 1 |
---|
206 | if bytesize >= 1024 ** power: |
---|
207 | size = "%.2f %s" % (bytesize/(1024.0**power), unit) |
---|
208 | break |
---|
209 | return size |
---|
210 | |
---|
211 | class Import(object): |
---|
212 | """Helper class to aggregate imports and their data. |
---|
213 | """ |
---|
214 | def __init__(self, filedescr, importers): |
---|
215 | self.file = filedescr |
---|
216 | self.importers = [] |
---|
217 | for importer, receiverid in importers: |
---|
218 | importer.receiverid = receiverid |
---|
219 | self.importers.append(importer) |
---|