Index: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/README.txt
===================================================================
--- main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/README.txt	(revision 5235)
+++ main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/README.txt	(revision 5235)
@@ -0,0 +1,42 @@
+JAMB registration numbers
+*************************
+
+:nodoctest:
+
+JAMB data is stored in a JAMBDataTable.
+
+We can import JAMB registration data from a regular CSV file:
+
+    >>> datafile = 'jambsample.csv'
+    >>> open(datafile, 'wb').write(
+    ... '''reg_no,fst_sit_fname,lastname,firstname,middlenames,sex,date_of_birth,jamb_state,jamb_lga,course1,screening_date,screening_venue,entry_session,screening_type
+    ... 91100546DD,ISUOSUO MOSES ODOMERO,ISUOSUO,MOSES,ODOMERO,M,25/5/1982,DEL,ISO-S,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde
+    ... 91111834CC,DURUIHEOMA AUGUSTINA ADANNA,DURUIHEOMA,AUGUSTINA,ADANNA,F,15/4/1986,IMO,MBAIT,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde
+    ... 91109351AC,ARISERE EBIKEBUNA COMFORT,ARISERE,EBIKEBUNA,COMFORT,F,6/1/1984,EDO,OV-SW,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde
+    ... ''')
+
+Now we create a JAMBDataTable and import the data:
+
+    >>> from waeup.sirp.jambtables import JAMBDataTable
+    >>> table = JAMBDataTable()
+    >>> len(list(table))
+    0
+
+    >>> table.importFromCSV(datafile)
+    >>> len(list(table))
+    3
+
+We can get the entries from the table as an iterator:
+
+    >>> for x in table:
+    ...   print x
+
+
+We can delete the table:
+
+    >>> del JAMBDataTable
+
+Clean up:
+
+    >>> import os
+    >>> os.unlink(datafile)
Index: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/__init__.py
===================================================================
--- main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/__init__.py	(revision 5235)
+++ main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/__init__.py	(revision 5235)
@@ -0,0 +1,2 @@
+# Make this a package.
+from waeup.sirp.jambtables.jambtables import JAMBDataTable
Index: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/jambtables.py
===================================================================
--- main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/jambtables.py	(revision 5235)
+++ main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/jambtables.py	(revision 5235)
@@ -0,0 +1,147 @@
+##
+## jambtables.py
+## Login : <uli@pu.smp.net>
+## Started on  Tue Jun 22 06:31:42 2010 Uli Fouquet
+## $Id$
+## 
+## Copyright (C) 2010 Uli Fouquet
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+## 
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+## 
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##
+import csv
+import os
+import shutil
+import tempfile
+import grok
+
+from datetime import datetime
+from BTrees.OOBTree import OOBTree
+from BTrees.Length import Length
+
+def filter_data(datadict):
+    """Filter items whose key contains 'ignore'
+    """
+    keys = datadict.keys()
+    for key in keys:
+        if not 'ignore' in key:
+            continue
+        del datadict[key]
+    for name in ['firstname', 'lastname', 'middlenames', 'screening_type',
+                 'screening_venue', 'reg_no', 'sex', 'course1', 'jamb_state',
+                 'screening_date', 'jamb_lga', 'fst_sit_fname',]:
+        datadict[name] = unicode(datadict[name])
+    datadict['date_of_birth'] = datetime.strptime(
+        datadict['date_of_birth'],
+        '%d/%m/%Y'
+        ).date()
+    
+    return datadict
+
+
+class JAMBDataTable(grok.Model):
+    """A data table that contains JAMB data.
+
+    JAMB data tables are plain but fast as they store nearly no data
+    inside the ZODB. All data is held on-disk in CSV tables.
+
+    As a consequence these tables are read-only.
+    """
+
+    import_datetime = None
+    
+    def __init__(self):
+        super(JAMBDataTable, self).__init__()
+        self._datafile_path = None
+        self._data_len = 0
+        self._temporary = False
+        return
+
+    def __del__(self):
+        self.clear()
+
+    def __iter__(self):
+        reader = None
+        if self._datafile_path is not None:
+            reader = csv.DictReader(open(self._datafile_path, 'rb'))
+        if reader is None:
+            raise StopIteration
+        for line in reader:
+            data_dict = filter_data(line)
+            yield data_dict
+
+    def clear(self):
+        """Remove all existing entries.
+        """
+        if self._datafile_path is None:
+            return
+        if self._temporary:
+            if not os.path.exists(self._datafile_path):
+                return
+            shutil.rmtree(os.path.dirname(self._datafile_path))
+            
+    def importFromCSV(self, filepath):
+        """Importing data from a CSV file means to copy the source to a safe
+           location.
+        """
+        self.clear()
+        self.import_datetime = datetime.now()
+        self._copyDataFile(filepath)
+
+    #def add(self, reg_no, data_dict):
+    #    item = filter_data(data_dict)
+    #    item.__name__ = reg_no
+    #    item.__parent__ = self
+    #    #if send_events:
+    #    #    objectEventNotify(ObjectWillBeAddedEvent(item, self, reg_no))
+    #    #self._data[reg_no] = item
+    #    self._data_len += 1
+    #    #if senf_event:
+    #    #    objectEventNotify(ObjectAddedEvent(item, self, reg_no))
+
+    def _copyDataFile(self, path):
+        """Copy file in path to the JAMBData storage.
+
+        See :meth:`_getJAMBTableStorage`.
+        """
+        storage = self._getJAMBTableStorage()
+        self._datafile_path = os.path.join(
+            storage, os.path.basename(path)
+            )
+        shutil.copy2(path, self._datafile_path)
+        return
+
+    def _getJAMBTableStorage(self):
+        """Get a path to store copies of datatables.
+
+        We normally store data in a ``jambdata`` subdir of datacenter,
+        but if none exists, we create a temporary dir and set
+        `temporary` to ``True``.
+
+        Any not existent directory is created.
+
+        Note, that temporary dirs will be deleted when the
+        JAMBDataTable object is destroyed.
+
+        Returns absolute path to the JAMB data storage.
+        """
+        site = grok.getSite()
+        if site is None:
+            jambtable_storage = tempfile.mkdtemp()
+            self._temporary = True
+        else:
+            datacenter = site['datacenter']
+            jambtable_storage = os.path.join(datacenter.storage, 'jambdata')
+        if not os.path.isdir(jambtable_storage):
+            os.mkdir(jambtable_storage)
+        return os.path.abspath(jambtable_storage)
Index: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/__init__.py
===================================================================
--- main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/__init__.py	(revision 5235)
+++ main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/__init__.py	(revision 5235)
@@ -0,0 +1,1 @@
+# Make this a package.
Index: main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/test_jambtables.py
===================================================================
--- main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/test_jambtables.py	(revision 5235)
+++ main/waeup.sirp/branches/ulif-fasttables/src/waeup/sirp/jambtables/tests/test_jambtables.py	(revision 5235)
@@ -0,0 +1,149 @@
+##
+## test_jambtables.py
+## Login : <uli@pu.smp.net>
+## Started on  Thu Jun 24 08:08:12 2010 Uli Fouquet
+## $Id$
+## 
+## Copyright (C) 2010 Uli Fouquet
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+## 
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+## 
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##
+"""
+Tests for JAMB tables.
+
+"""
+import os
+import shutil
+import tempfile
+import unittest
+
+import grok
+
+from zope.site import LocalSiteManager
+from zope.site.hooks import setSite, clearSite
+
+from waeup.sirp.app import University
+from waeup.sirp.jambtables import JAMBDataTable
+from waeup.sirp.datacenter import DataCenter
+
+class JAMBTableTestCaseBase(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.datafile = os.path.join(
+            self.tempdir, 'jambsample.csv'
+            )
+        open(self.datafile, 'wb').write(
+            'reg_no,fst_sit_fname,lastname,firstname,middlenames,sex,date_of_birth,jamb_state,jamb_lga,course1,screening_date,screening_venue,entry_session,screening_type,ignore_this_col\n' + 
+            '91100546DD,ISUOSUO MOSES ODOMERO,ISUOSUO,MOSES,ODOMERO,M,25/5/1982,DEL,ISO-S,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde,blah\n' +
+            '91111834CC,DURUIHEOMA AUGUSTINA ADANNA,DURUIHEOMA,AUGUSTINA,ADANNA,F,15/4/1986,IMO,MBAIT,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde,blah\n' +
+            '91109351AC,ARISERE EBIKEBUNA COMFORT,ARISERE,EBIKEBUNA,COMFORT,F,6/1/1984,EDO,OV-SW,BSCPOL,2009/09/25 09:00:00 GMT+1,REPRINT SLIP AS FROM WED 23/09/2009,9,pde,blah\n'
+            )
+        setSite()
+        return
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+        return
+
+class JAMBTableTestCase(JAMBTableTestCaseBase):
+    def test_createJAMBTable(self):
+        table = JAMBDataTable()
+        self.assertEqual(table._data_len, 0)
+        self.assertEqual(table._temporary, False)
+        self.assertEqual(table._datafile_path, None)
+        return
+
+    def test_iterator(self):
+        table = JAMBDataTable()
+        self.assertEqual(len(list(table)), 0)
+        table.importFromCSV(self.datafile)
+        self.assertEqual(len(list(table)), 3)
+
+    def test_import(self):
+        table = JAMBDataTable()
+        self.assertEqual(table.import_datetime, None)
+        table.importFromCSV(self.datafile)
+        self.assertNotEqual(table.import_datetime, None)
+
+    def test_clear(self):
+        table = JAMBDataTable()
+        table.importFromCSV(self.datafile)
+        path = table._datafile_path
+        table.clear()
+        exists = os.path.exists(path)
+        self.assertEqual(exists, False)
+
+
+class JAMBTableFunctionalTestCase(JAMBTableTestCaseBase):
+    """A test case where we check JAMB tables inside a site.
+    """
+
+    def setUp(self):
+        super(JAMBTableFunctionalTestCase, self).setUp()
+        
+        # We need at least these to register factories for a University.
+        grok.testing.grok('waeup.sirp.student')
+        grok.testing.grok('waeup.sirp.hostel')
+        grok.testing.grok('waeup.sirp.university')
+
+        self.site = University()
+        sm = LocalSiteManager(self.site)
+        self.site.setSiteManager(sm)
+        self.datacenter_dir = tempfile.mkdtemp()
+        self.datacenter = DataCenter()
+        self.datacenter.setStoragePath(self.datacenter_dir)
+        # If the old datacenter already had a jambtables dir, remove it...
+        if os.path.isdir(
+            os.path.join(self.datacenter_dir, 'jambdata')
+            ):
+            shutil.rmtree(os.path.join(self.datacenter_dir, 'jambdata'))
+        return
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+        shutil.rmtree(self.datacenter_dir)
+        # Unregister site...
+        setSite()
+        return
+
+    def test_createJAMBTableInSite(self):
+        table = JAMBDataTable()
+        self.assertEqual(table._data_len, 0)
+        self.assertEqual(table._temporary, False)
+        self.assertEqual(table._datafile_path, None)
+        table.importFromCSV(self.datafile)
+        return
+
+    def test_createJAMBTableOutsideSite(self):
+        clearSite(self.site)
+        table = JAMBDataTable()
+        self.assertEqual(table._data_len, 0)
+        self.assertEqual(table._temporary, False)
+        self.assertEqual(table._datafile_path, None)
+        table.importFromCSV(self.datafile)
+        return
+
+        
+def test_suite():
+    suite = unittest.TestSuite()
+    for testcase in [
+        JAMBTableTestCase,
+        JAMBTableFunctionalTestCase,
+        ]:
+        suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
+                testcase
+                )
+        )
+    return suite
