"""WAeUP data center. The waeup data center cares for management of upload data and provides tools for importing/exporting CSV data. """ import os import struct import grok from datetime import datetime from zope.component import getMultiAdapter from waeup.csvfile import getCSVFile from waeup.interfaces import (IDataCenter, IWAeUPCSVImporter, ICSVDataReceivers, IDataCenterFile) from waeup.utils.helpers import copyFileSystemTree class DataCenter(grok.Container): """A data center contains CSV files. """ grok.implements(IDataCenter) storage = os.path.join(os.path.dirname(__file__), 'files') def getReceivers(self): receivers = [] curr_obj = getattr(self, '__parent__', None) while curr_obj is not None: if ICSVDataReceivers.providedBy(curr_obj): receivers = self.getCSVDataReceivers(curr_obj) break curr_obj = getattr(curr_obj, '__parent__', None) return receivers def getReceiverIds(self): """Get a dict of available receivers. The keys of the result are the receiver ids. """ receivers = self.getReceivers() return dict([(self.getReceiverId(x), x) for x in receivers]) def getImporters(self): """Get a list of all importers available. The search for available importers is done in two steps: 1) Look for a utility providing ICSVDataReceiver, 2) For every item of that utility: try to get an adapter providing IWAeUPCSVImporter. """ result = [] receivers = self.getReceivers() files = self.getFiles() for receiver in receivers: for file in files: wrapped_file = getCSVFile(file.context) try: importer = getMultiAdapter((wrapped_file, receiver), IWAeUPCSVImporter) result.append(importer) except: # No multi-adapter for this combination available... pass return result def getFiles(self): """Get a list of files stored in `storage`. Files are sorted by basename. """ result = [] if not os.path.exists(self.storage): return result for filename in sorted(os.listdir(self.storage)): fullpath = os.path.join(self.storage, filename) if not os.path.isfile(fullpath): continue result.append(DataCenterFile(fullpath)) return result def setStoragePath(self, path, move=False, overwrite=False): """Set the path where to store files. """ path = os.path.abspath(path) not_copied = [] if not os.path.exists(path): raise ValueError('The path given does not exist: %s' % path) if move is True: not_copied = copyFileSystemTree(self.storage, path, overwrite=overwrite) self.storage = path return not_copied def getCSVDataReceivers(self, obj): """Get a list of attributes, that can receive CSV data. We also get a list of values, if obj is a container that contains value items. """ result = [] for attr_name in dir(obj): if attr_name.startswith('_'): continue try: attr = getattr(obj, attr_name) # This might fail... #IWAeUPCSVImporter(attr) result.append(attr) except: pass if hasattr(obj, 'values'): try: result.extend(obj.values()) except: pass return result def getPossibleImports(self): """Get list of possible imports. A single import is defined as a tuple ( , ) where ```` is an `IDataCenterFile` object and ```` is a list of `IWAeUPCSVImporter` objects. """ result = [] importers = self.getImporters() for filedescr in self.getFiles(): possible_importers = [] for importer in importers: if importer.csvfile.path != filedescr.context: continue importer_context = ( importer, self.getReceiverId(importer.receiver)) possible_importers.append(importer_context) if len(possible_importers) == 0: continue result.append((filedescr, possible_importers)) return result def getReceiverId(self, obj): """Get a unique id for an object. If the object is stored in ZODB it should contain a `_p_oid` attribute, which is guaranteed to be unique over the ZODB. If the object has no such attribute, then it will be held in memory only and will vanish as soon as the request is over. In this case we can use the memory address of it. Both, the id() result and the ZODB oid, are actually integers which we return as strings. To make the difference chrystal clear, we prepend ``z`` to ids taken from ZODB oids. """ if not hasattr(obj, '_p_oid'): return str(id(obj)) oid = getattr(obj, '_p_oid') if oid is None: # The persistent object is not stored in the ZODB return str(id(obj)) return 'z%s' % struct.unpack('>Q', oid)[0] def doImport(self, path, receiverid, clear=None): receivers = self.getReceiverIds() if receiverid not in receivers.keys(): raise ValueError('The requested data receiver cannot be found.') receiver = receivers[receiverid] filewrapper = getCSVFile(path) if filewrapper is None: raise ValueError('Format of CSV file not supported.') importer = getMultiAdapter((filewrapper, receiver), IWAeUPCSVImporter) if clear is not None: importer.doImport(clear_old_data=clear) else: importer.doImport() return class DataCenterFile(object): """A description of a file stored in data center. """ grok.implements(IDataCenterFile) def __init__(self, context): self.context = context self.name = os.path.basename(self.context) self.size = self.getSize() self.uploaddate = self.getDate() def getDate(self): """Get a human readable datetime representation. """ date = datetime.fromtimestamp(os.path.getctime(self.context)) return date.strftime('%c') def getSize(self): """Get a human readable file size. """ bytesize = os.path.getsize(self.context) size = "%s bytes" % bytesize units = ['kb', 'MB', 'GB'] for power, unit in reversed(list(enumerate(units))): power += 1 if bytesize >= 1024 ** power: size = "%.2f %s" % (bytesize/(1024.0**power), unit) break return size class Import(object): """Helper class to aggregate imports and their data. """ def __init__(self, filedescr, importers): self.file = filedescr self.importers = [] for importer, receiverid in importers: importer.receiverid = receiverid self.importers.append(importer)