source: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/jambtables.py @ 5263

Last change on this file since 5263 was 5254, checked in by uli, 14 years ago

Update methods to store importer_username on imports.

File size: 5.5 KB
Line 
1##
2## jambtables.py
3## Login : <uli@pu.smp.net>
4## Started on  Tue Jun 22 06:31:42 2010 Uli Fouquet
5## $Id$
6##
7## Copyright (C) 2010 Uli Fouquet
8## This program is free software; you can redistribute it and/or modify
9## it under the terms of the GNU General Public License as published by
10## the Free Software Foundation; either version 2 of the License, or
11## (at your option) any later version.
12##
13## This program is distributed in the hope that it will be useful,
14## but WITHOUT ANY WARRANTY; without even the implied warranty of
15## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16## GNU General Public License for more details.
17##
18## You should have received a copy of the GNU General Public License
19## along with this program; if not, write to the Free Software
20## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21##
22import csv
23import os
24import shutil
25import tempfile
26import grok
27
28from datetime import datetime
29from BTrees.OOBTree import OOBTree
30from BTrees.Length import Length
31
32from waeup.sirp.jambtables.interfaces import IJAMBDataTable
33
34#: The header fields required for a valid JAMB table CSV file.
35JAMB_DATA_HEADERS = [
36    'firstname', 'lastname', 'middlenames', 'screening_type',
37    'screening_venue', 'reg_no', 'sex', 'course1', 'jamb_state',
38    'screening_date', 'jamb_lga', 'fst_sit_fname', 'date_of_birth']
39
40def filter_data(datadict):
41    """Filter items whose key contains 'ignore'
42
43    The function will remove all fields whose name start with
44    ``ignore``. All data will be convertetd to unicode except
45    ``data_of_birth``, which is turned into a `dateteime` object.
46    """
47    keys = datadict.keys()
48    for key in keys:
49        if not 'ignore' in key:
50            continue
51        del datadict[key]
52    for name in JAMB_DATA_HEADERS:
53        datadict[name] = unicode(datadict[name])
54    datadict['date_of_birth'] = datetime.strptime(
55        datadict['date_of_birth'],
56        '%d/%m/%Y'
57        ).date()
58   
59    return datadict
60
61
62class JAMBDataTable(grok.Model):
63    """A data table that contains JAMB data.
64
65    JAMB data tables are plain but fast as they store nearly no data
66    inside the ZODB. All data is held on-disk in CSV tables.
67
68    As a consequence these tables are read-only.
69    """
70    grok.implements(IJAMBDataTable)
71
72    #: The datetime when data was imported.
73    import_datetime = None
74
75    #: The username of the person that initiated the last import.
76    importer_username = None
77   
78    def __init__(self):
79        super(JAMBDataTable, self).__init__()
80        self._datafile_path = None
81        self._data_len = 0
82        self._temporary = False
83        return
84
85    def __del__(self):
86        self.clear()
87
88    def __iter__(self):
89        reader = None
90        if self._datafile_path is not None:
91            reader = csv.DictReader(open(self._datafile_path, 'rb'))
92        if reader is None:
93            raise StopIteration
94        for line in reader:
95            data_dict = filter_data(line)
96            yield data_dict
97
98    def keys(self):
99        """Get iterator over all registration numbers stored in table.
100        """
101        for item in self:
102            yield item['reg_no']
103
104    def items(self):
105        """Get tuples of registration number and datasets for each entry in
106           data table.
107        """
108        for item in self:
109            yield (item['reg_no'], item)
110           
111    def clear(self):
112        """Remove all existing entries.
113
114        Unsets also the :attr:`import_datetime` and
115        :attr:`importer_username` attributes.
116        """
117        self.import_datetime = None
118        self.importer_username = None
119        if self._datafile_path is None:
120            return
121        if self._temporary:
122            if not os.path.exists(self._datafile_path):
123                return
124            shutil.rmtree(os.path.dirname(self._datafile_path))
125        self._datafile_path = None
126           
127    def importFromCSV(self, filepath, username=None):
128        """Importing data from a CSV file means to copy the source to a safe
129           location.
130
131           If the username is set, it will be stored as well in
132           :attr:`importer_username`.
133        """
134        self.clear()
135        self.import_datetime = datetime.now()
136        self.importer_username = None
137        if username is not None:
138            self.importer_username = unicode(username)
139        self._copyDataFile(filepath)
140
141    def _copyDataFile(self, path):
142        """Copy file in path to the JAMBData storage.
143
144        See :meth:`_getJAMBTableStorage`.
145        """
146        storage = self._getJAMBTableStorage()
147        self._datafile_path = os.path.join(
148            storage, os.path.basename(path)
149            )
150        shutil.copy2(path, self._datafile_path)
151        return
152
153    def _getJAMBTableStorage(self):
154        """Get a path to store copies of datatables.
155
156        We normally store data in a ``jambdata`` subdir of datacenter,
157        but if none exists, we create a temporary dir and set
158        `temporary` to ``True``.
159
160        Any not existent directory is created.
161
162        Note, that temporary dirs will be deleted when the
163        JAMBDataTable object is destroyed.
164
165        Returns absolute path to the JAMB data storage.
166        """
167        site = grok.getSite()
168        if site is None:
169            jambtable_storage = tempfile.mkdtemp()
170            self._temporary = True
171        else:
172            datacenter = site['datacenter']
173            jambtable_storage = os.path.join(datacenter.storage, 'jambdata')
174        if not os.path.isdir(jambtable_storage):
175            os.mkdir(jambtable_storage)
176        return os.path.abspath(jambtable_storage)
Note: See TracBrowser for help on using the repository browser.