Ignore:
Timestamp:
8 Nov 2011, 18:22:43 (13 years ago)
Author:
uli
Message:

Remove old blob storage and fix related tests.

Location:
main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp/applicants/tests/test_applicant.py

    r7002 r7034  
    3939from waeup.sirp.image import WAeUPImageFile, createWAeUPImageFile
    4040from waeup.sirp.image.interfaces import IWAeUPImageFile
    41 from waeup.sirp.imagestorage import ImageStorageFileRetrieval, DefaultStorage
     41from waeup.sirp.imagestorage import DefaultStorage
    4242from waeup.sirp.interfaces import IFileStoreHandler, IFileStoreNameChooser
    4343from waeup.sirp.applicants import (
    … …  
    218218    def setUp(self):
    219219        super(ApplicantFactoryTest, self).setUp()
    220         # Install a IFileRetrieval utility that returns WAeUPImageFiles.
    221         storage = ImageStorageFileRetrieval()
    222         #provideUtility(storage, IFileRetrieval)
    223220        self.factory = ApplicantFactory()
    224221        return

  • main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp/applicants/tests/test_browser.py

    r7002 r7034  
    748748        file_ctrl.add_file(pseudo_image, filename='myphoto.jpg')
    749749        self.browser.getControl("Save").click() # submit form
    750         storage = self.app['images']
    751750        storage = getUtility(IExtFileStore)
    752751        file_id = IFileStoreNameChooser(self.applicant).chooseName()

  • main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp/imagestorage.py

    r7032 r7034  
    200200from waeup.sirp.utils.helpers import cmp_files
    201201
    202 def md5digest(fd):
    203     """Get an MD5 hexdigest for the file stored in `fd`.
    204 
    205     `fd`
    206       a file object open for reading.
    207 
    208     """
    209     return hashlib.md5(fd.read()).hexdigest()
    210 
    211202class FileStoreNameChooser(grok.Adapter):
    212203    """Default file store name chooser.
    … …  
    252243            return name
    253244        return u'unknown_file'
    254 
    255 class Basket(grok.Container):
    256     """A basket holds a set of image files with same hash.
    257 
    258     It is used for storing files as blobs in ZODB. Current sites do
    259     not use it.
    260 
    261     .. deprecated:: 0.2
    262 
    263     """
    264 
    265     def _del(self):
    266         """Remove temporary files associated with local blobs.
    267 
    268         A basket holds files as Blob objects. Unfortunately, if a
    269         basket was not committed (put into ZODB), those blobs linger
    270         around as real files in some temporary directory and won't be
    271         removed.
    272 
    273         This is a helper function to remove all those uncommitted
    274         blobs that has to be called explicitly, for instance in tests.
    275         """
    276         key_list = self.keys()
    277         for key in key_list:
    278             item = self[key]
    279             if getattr(item, '_p_oid', None):
    280                 # Don't mess around with blobs in ZODB
    281                 continue
    282             fd = item.open('r')
    283             name = getattr(fd, 'name', None)
    284             fd.close()
    285             if name is not None and os.path.exists(name):
    286                 os.unlink(name)
    287             del self[key]
    288         return
    289 
    290     def getInternalId(self, fd):
    291         """Get the basket-internal id for the file stored in `fd`.
    292 
    293         `fd` must be a file open for reading. If an (byte-wise) equal
    294         file can be found in the basket, its internal id (basket id)
    295         is returned, ``None`` otherwise.
    296         """
    297         fd.seek(0)
    298         for key, val in self.items():
    299             fd_stored = val.open('r')
    300             file_len = os.stat(fd_stored.name)[6]
    301             if file_len == 0:
    302                 # Nasty workaround. Blobs seem to suffer from being emptied
    303                 # accidentally.
    304                 site = grok.getSite()
    305                 if site is not None:
    306                     site.logger.warn(
    307                         'Empty Blob detected: %s' % fd_stored.name)
    308                 warnings.warn("EMPTY BLOB DETECTED: %s" % fd_stored.name)
    309                 fd_stored.close()
    310                 val.open('w').write(fd.read())
    311                 return key
    312             fd_stored.seek(0)
    313             if cmp_files(fd, fd_stored):
    314                 fd_stored.close()
    315                 return key
    316             fd_stored.close()
    317         return None
    318 
    319     @property
    320     def curr_id(self):
    321         """The current basket id.
    322 
    323         An integer number which is not yet in use. If there are
    324         already `maxint` entries in the basket, a :exc:`ValueError` is
    325         raised. The latter is _highly_ unlikely. It would mean to have
    326         more than 2**32 hash collisions, i.e. so many files with the
    327         same MD5 sum.
    328         """
    329         num = 1
    330         while True:
    331             if str(num) not in self.keys():
    332                 return str(num)
    333             num += 1
    334             if num <= 0:
    335                 name = getattr(self, '__name__', None)
    336                 raise ValueError('Basket full: %s' % name)
    337 
    338     def storeFile(self, fd, filename):
    339         """Store the file in `fd` into the basket.
    340 
    341         The file will be stored in a Blob.
    342         """
    343         fd.seek(0)
    344         internal_id = self.getInternalId(fd) # Moves file pointer!
    345         if internal_id is None:
    346             internal_id = self.curr_id
    347             fd.seek(0)
    348             self[internal_id] = Blob()
    349             transaction.commit() # Urgently needed to make the Blob
    350                                  # persistent. Took me ages to find
    351                                  # out that solution, which makes some
    352                                  # design flaw in ZODB Blobs likely.
    353             self[internal_id].open('w').write(fd.read())
    354             fd.seek(0)
    355             self._p_changed = True
    356         return internal_id
    357 
    358     def retrieveFile(self, basket_id):
    359         """Retrieve a file open for reading with basket id `basket_id`.
    360 
    361         If there is no such id, ``None`` is returned. It is the
    362         callers responsibility to close the open file.
    363         """
    364         if basket_id in self.keys():
    365             return self[basket_id].open('r')
    366         return None
    367 
    368 class ImageStorage(grok.Container):
    369     """A container for image files.
    370 
    371     .. deprecated:: 0.2
    372 
    373        Use :class:`waeup.sirp.ExtFileStore` instead.
    374     """
    375     def _del(self):
    376         for basket in self.values():
    377             try:
    378                 basket._del()
    379             except:
    380                 pass
    381 
    382     def storeFile(self, fd, filename):
    383         """Store contents of file addressed by `fd` under filename `filename`.
    384 
    385         Returns the internal file id (a string) for the file stored.
    386         """
    387         fd.seek(0)
    388         digest = md5digest(fd)
    389         fd.seek(0)
    390         if not digest in self.keys():
    391             self[digest] = Basket()
    392         basket_id = self[digest].storeFile(fd, filename)
    393         full_id = "%s-%s" % (digest, basket_id)
    394         return full_id
    395 
    396     def retrieveFile(self, file_id):
    397         if not '-' in file_id:
    398             return None
    399         full_id, basket_id = file_id.split('-', 1)
    400         if not full_id in self.keys():
    401             return None
    402         return self[full_id].retrieveFile(basket_id)
    403 
    404 class ImageStorageFileRetrieval(Persistent):
    405     """A persistent object to retrieve files stored in ZODB.
    406 
    407     .. deprecated:: 0.2
    408 
    409        Since we have :class:`ExtFileStore` now we do not need this
    410        class anymore.
    411     """
    412     grok.implements(IFileRetrieval)
    413 
    414     def getImageStorage(self):
    415         site = grok.getSite()
    416         if site is None:
    417             return None
    418         return site.get('images', None)
    419 
    420     def isImageStorageEnabled(self):
    421         site = grok.getSite()
    422         if site is None:
    423             return False
    424         if site.get('images', None) is None:
    425             return False
    426         return True
    427 
    428     def getFile(self, data):
    429         # ImageStorage is disabled, so give fall-back behaviour for
    430         # testing without ImageStorage
    431         if not self.isImageStorageEnabled():
    432             return StringIO(data)
    433         storage = self.getImageStorage()
    434         if storage is None:
    435             raise ValueError('Cannot find an image storage')
    436         result = storage.retrieveFile(data)
    437         if result is None:
    438             return StringIO(data)
    439         return storage.retrieveFile(data)
    440 
    441     def createFile(self, filename, f):
    442         if not self.isImageStorageEnabled():
    443             return WAeUPImageFile(filename, f.read())
    444         storage = self.getImageStorage()
    445         if storage is None:
    446             raise ValueError('Cannot find an image storage')
    447         file_id = storage.storeFile(f, filename)
    448         return WAeUPImageFile(filename, file_id)
    449 
    450245
    451246class ExtFileStore(object):
    … …  
    604399
    605400class DefaultStorage(ExtFileStore):
     401    """Default storage for files.
     402
     403    Registered globally as utility for
     404    :class:`hurry.file.interfaces.IFileRetrieval`.
     405    """
    606406    grok.provides(IFileRetrieval)
    607407
    … …  
    613413    This handler is the fallback called by external file stores when
    614414    there is no or an unknown marker in the file id.
     415
     416    Registered globally as utility for
     417    :class:`waeup.sirp.interfaces.IFileStoreHandler`.
    615418    """
    616419    grok.implements(IFileStoreHandler)

  • main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp/tests/test_app.py

    r7002 r7034  
    66from zope.interface.verify import verifyClass, verifyObject
    77from waeup.sirp.app import University
    8 from waeup.sirp.imagestorage import ImageStorageFileRetrieval
    98from waeup.sirp.interfaces import IUniversity
    109from waeup.sirp.testing import FunctionalLayer, FunctionalTestCase
    … …  
    3231        return
    3332
    34     def test_images(self):
    35         # Make sure we have a image container in a university
    36         assert 'images' in self.app.keys()
    37         return
    38 
    3933    def test_IFileRetrieval_utility(self):
    4034        # Make sure we can get a local IFileRetrieval utility

  • main/waeup.sirp/branches/ulif-extimgstore/src/waeup/sirp/tests/test_imagestorage.py

    r7032 r7034  
    1616from waeup.sirp.image import createWAeUPImageFile
    1717from waeup.sirp.imagestorage import (
    18     md5digest, Basket, ImageStorage, ImageStorageFileRetrieval,
    1918    FileStoreNameChooser, ExtFileStore, DefaultFileStoreHandler,
    2019    DefaultStorage)
    … …  
    2928    def tearDown(self):
    3029        shutil.rmtree(self.workdir)
    31 
    32     def test_md5digest(self):
    33         samplefile = os.path.join(self.workdir, 'sample')
    34         open(samplefile, 'wb').write('blah')
    35         fp = open(samplefile, 'r')
    36         digest = md5digest(fp)
    37         self.assertEqual(digest, '6f1ed002ab5595859014ebf0951522d9')
    38 
    39 class BasketTests(FunctionalTestCase):
    40 
    41     layer = FunctionalLayer
    42 
    43     def setUp(self):
    44         super(BasketTests, self).setUp()
    45         self.workdir = tempfile.mkdtemp()
    46         self.samplefile = os.path.join(self.workdir, 'sample')
    47         self.otherfile = os.path.join(self.workdir, 'other')
    48         open(self.samplefile, 'wb').write('Hi there!')
    49         open(self.otherfile, 'wb').write('Hi from other!')
    50         self.basket = Basket()
    51         self.fd = open(self.samplefile, 'r')
    52         self.fd2 = open(self.otherfile, 'r')
    53         self.stderr = StringIO()
    54         self.old_stderr = sys.stderr
    55 
    56     def tearDown(self):
    57         sys.stderr = self.old_stderr
    58         super(BasketTests, self).tearDown()
    59         self.fd.close()
    60         shutil.rmtree(self.workdir)
    61         self.basket._del() # Remove subojects explicitly
    62         del self.basket
    63         return
    64 
    65     def test_ifaces(self):
    66         pass
    67 
    68     def test_curr_id_empty(self):
    69         curr_id = self.basket.curr_id
    70         self.assertEqual(curr_id, '1')
    71 
    72     def test_getInternalId_empty(self):
    73         basket_id = self.basket.getInternalId(self.fd)
    74         self.assertTrue(basket_id is None)
    75 
    76     def test_storeFile_single(self):
    77         basket_id = self.basket.storeFile(self.fd, 'sample')
    78         self.assertEqual(basket_id, '1')
    79         contents = self.basket['1'].open('r').read()
    80         self.assertEqual(contents, 'Hi there!')
    81 
    82     def test_storeFile_double(self):
    83         basket_id1 = self.basket.storeFile(self.fd, 'sample')
    84         basket_id2 = self.basket.storeFile(self.fd, 'sample')
    85         self.assertTrue(basket_id1 == basket_id2 == '1')
    86         contents = self.basket['1'].open('r').read()
    87         self.assertEqual(contents, 'Hi there!')
    88 
    89     def test_storeFile_multiple(self):
    90         basket_id1 = self.basket.storeFile(self.fd, 'sample')
    91         basket_id2 = self.basket.storeFile(self.fd2, 'sample')
    92         self.assertEqual(basket_id1, '1')
    93         self.assertEqual(basket_id2, '2')
    94         contents1 = self.basket['1'].open('r').read()
    95         contents2 = self.basket['2'].open('r').read()
    96         self.assertEqual(contents1, 'Hi there!')
    97         self.assertEqual(contents2, 'Hi from other!')
    98 
    99     def test_retrieveFile(self):
    100         basket_id = self.basket.storeFile(self.fd, 'sample')
    101         fd = self.basket.retrieveFile(basket_id)
    102         result = fd.read()
    103         self.assertEqual(result, 'Hi there!')
    104 
    105     def test_retrieveFile_not_existent(self):
    106         result = self.basket.retrieveFile('not-a-valid-basket-id')
    107         self.assertTrue(result is None)
    108 
    109     def test_detect_zero_length_blobs(self):
    110         # Ensure we get a warning when an empty Blob is found
    111         self.basket.storeFile(self.fd, 'sample')
    112         self.basket['1'].open('w').write('')
    113         self.fd.seek(0)
    114         sys.stderr = self.stderr         # Redirect stderr
    115         self.basket.storeFile(self.fd, 'sample')
    116         sys.stderr = self.old_stderr     # Restore stderr
    117         self.stderr.seek(0)
    118         self.assertTrue(
    119             "EMPTY BLOB DETECTED" in self.stderr.read())
    120 
    121     def test_refill_zero_length_blobs(self):
    122         # When we detect an empty Blob, it will be reused
    123         self.basket.storeFile(self.fd, 'sample')
    124         self.basket['1'].open('w').write('')
    125         self.fd.seek(0)
    126         sys.stderr = self.stderr         # Redirect stderr
    127         self.basket.storeFile(self.fd, 'sample')
    128         sys.stderr = self.old_stderr     # Restore stderr
    129         contents = self.basket['1'].open('r').read()
    130         self.assertEqual(contents, 'Hi there!')
    131 
    132 
    133 class ImageStorageTests(FunctionalTestCase):
    134 
    135     layer = FunctionalLayer
    136 
    137     def setUp(self):
    138         super(ImageStorageTests, self).setUp()
    139         self.workdir = tempfile.mkdtemp()
    140         self.samplefile = os.path.join(self.workdir, 'sample')
    141         self.otherfile = os.path.join(self.workdir, 'other')
    142         open(self.samplefile, 'wb').write('Hi there!')
    143         open(self.otherfile, 'wb').write('Hi from other!')
    144         self.storage = ImageStorage()
    145         self.fd = open(self.samplefile, 'r')
    146         self.fd2 = open(self.otherfile, 'r')
    147 
    148     def tearDown(self):
    149         super(ImageStorageTests, self).tearDown()
    150         self.fd.close()
    151         self.fd2.close()
    152         shutil.rmtree(self.workdir)
    153         self.storage._del() # Remove subojects explicitly
    154         del self.storage
    155         return
    156 
    157     def test_ifaces(self):
    158         pass
    159 
    160     def test_storeFile(self):
    161         full_id = self.storage.storeFile(self.fd, 'sample.txt')
    162         self.assertEqual(full_id, '396199333edbf40ad43e62a1c1397793-1')
    163 
    164     def test_storeFile_duplicate(self):
    165         full_id1 = self.storage.storeFile(self.fd, 'sample1.txt')
    166         full_id2 = self.storage.storeFile(self.fd, 'sample2.txt')
    167         full_id3 = self.storage.storeFile(self.fd, 'sample1.txt')
    168         self.assertEqual(full_id1, '396199333edbf40ad43e62a1c1397793-1')
    169         self.assertEqual(full_id2, '396199333edbf40ad43e62a1c1397793-1')
    170         self.assertEqual(full_id3, '396199333edbf40ad43e62a1c1397793-1')
    171         contents = self.storage.retrieveFile(
    172             '396199333edbf40ad43e62a1c1397793-1').read()
    173         self.assertEqual(contents, 'Hi there!')
    174 
    175     def test_storeFile_multiple(self):
    176         full_id1 = self.storage.storeFile(self.fd, 'sample1.txt')
    177         full_id2 = self.storage.storeFile(self.fd2, 'sample1.txt')
    178         self.assertEqual(full_id1, '396199333edbf40ad43e62a1c1397793-1')
    179         self.assertEqual(full_id2, '6936fcf8d564f1c5be5a017e650c5e8f-1')
    180 
    181     def test_retrieveFile_not_existent(self):
    182         result = self.storage.retrieveFile('not-existent')
    183         self.assertTrue(result is None)
    184 
    185     def test_retrieveFile_illegal_marker(self):
    186         result1 = self.storage.retrieveFile('really-not-existent')
    187         result2 = self.storage.retrieveFile('notexistent')
    188         self.assertTrue(result1 is result2 is None)
    189 
    190     def test_retrieveFile(self):
    191         full_id = self.storage.storeFile(self.fd, 'sample.txt')
    192         result = self.storage.retrieveFile(full_id)
    193         content = result.read()
    194         self.assertEqual(content, 'Hi there!')
    195 
    196     def test_retrieveFile_multiple(self):
    197         full_id1 = self.storage.storeFile(self.fd, 'sample.txt')
    198         full_id2 = self.storage.storeFile(self.fd2, 'other.txt')
    199         result1 = self.storage.retrieveFile(full_id1)
    200         result2 = self.storage.retrieveFile(full_id2)
    201         content1 = result1.read()
    202         content2 = result2.read()
    203         self.assertEqual(content1, 'Hi there!')
    204         self.assertEqual(content2, 'Hi from other!')
    205 
    206 class ImageStorageFileRetrievalTests(FunctionalTestCase):
    207 
    208     layer = FunctionalLayer
    209 
    210     def setUp(self):
    211         super(ImageStorageFileRetrievalTests, self).setUp()
    212         self.workdir = tempfile.mkdtemp()
    213         self.samplefile = os.path.join(self.workdir, 'sample')
    214         self.otherfile = os.path.join(self.workdir, 'other')
    215         open(self.samplefile, 'wb').write('Hi there!')
    216         open(self.otherfile, 'wb').write('Hi from other!')
    217         self.storage = ImageStorage()
    218         self.fd = open(self.samplefile, 'r')
    219         self.fd2 = open(self.otherfile, 'r')
    220         # Set up a single image storage in a site
    221         self.getRootFolder()['app'] = University()
    222         self.app = self.getRootFolder()['app']
    223         if not 'images' in self.app.keys():
    224             self.app['images'] = ImageStorage()
    225         self.storage = self.app['images']
    226         return
    227 
    228 
    229     def tearDown(self):
    230         super(ImageStorageFileRetrievalTests, self).tearDown()
    231         self.fd.close()
    232         self.fd2.close()
    233         shutil.rmtree(self.workdir)
    234         self.storage._del() # Remove subojects explicitly
    235         return
    236 
    237     def test_ifaces(self):
    238         retrieval = ImageStorageFileRetrieval()
    239         assert verifyClass(IFileRetrieval, ImageStorageFileRetrieval)
    240         assert verifyObject(IFileRetrieval, retrieval)
    241         return
    242 
    243     def test_getImageStorage_nosite(self):
    244         retrieval = ImageStorageFileRetrieval()
    245         storage = retrieval.getImageStorage()
    246         self.assertTrue(storage is None)
    247         return
    248 
    249     def test_getImageStorage(self):
    250         setSite(self.app)
    251         retrieval = ImageStorageFileRetrieval()
    252         storage = retrieval.getImageStorage()
    253         self.assertTrue(storage is self.storage)
    254         return
    255 
    256     def test_isImageStorageEnabled_nosite(self):
    257         retrieval = ImageStorageFileRetrieval()
    258         self.assertTrue(retrieval.isImageStorageEnabled() is False)
    259         return
    260 
    261     def test_isImageStorageEnabled(self):
    262         setSite(self.app)
    263         retrieval = ImageStorageFileRetrieval()
    264         self.assertTrue(retrieval.isImageStorageEnabled() is True)
    265         return
    266 
    267     def test_getFile_nosite(self):
    268         retrieval = ImageStorageFileRetrieval()
    269         f = retrieval.getFile('Hi there!')
    270         self.assertEqual(f.read(), 'Hi there!')
    271         return
    272 
    273     def test_getFile(self):
    274         setSite(self.app)
    275         retrieval = ImageStorageFileRetrieval()
    276         waeup_image = retrieval.createFile('sample.txt', self.fd)
    277         full_id = waeup_image.data
    278         result = retrieval.getFile(full_id)
    279         self.assertEqual(result.read(), 'Hi there!')
    280         return
    281 
    282     def test_createFile_nosite(self):
    283         retrieval = ImageStorageFileRetrieval()
    284         waeup_image = retrieval.createFile('sample.txt', self.fd)
    285         self.assertEqual(waeup_image.data, 'Hi there!')
    286         return
    287 
    288     def test_createFile(self):
    289         # Ensure we can create WAeUPImageFiles when in site
    290         setSite(self.app)
    291         retrieval = ImageStorageFileRetrieval()
    292         waeup_image = retrieval.createFile('sample.txt', self.fd)
    293         full_id = waeup_image.data
    294         self.assertEqual(full_id, '396199333edbf40ad43e62a1c1397793-1')
    295         return
    296 
    297     def DISABLEDtest_waeupimagefile(self):
    298         # Make sure WAeUPImageFile can use our file retrieval
    299         setSite(self.app)
    300         myfile = createWAeUPImageFile('sample.jpg', self.fd)
    301         contents = myfile.file.read()
    302         self.assertEqual(contents, 'Hi there!')
    303 
    304     def DISABLEDtest_waeupimagefile_raw(self):
    305         # Make sure we can retrieve a file also if it was initialized
    306         # with no image storage available
    307         myfile = createWAeUPImageFile('sample.jpg', self.fd)
    308         setSite(self.app)
    309         contents = myfile.file.read()
    310         self.assertEqual(contents, 'Hi there!')
    31130
    31231class FileStoreNameChooserTests(FunctionalTestCase):
    … …  
    429148class CustomizedFileHandler(object):
    430149    def pathFromFileID(self, store, root, file_id):
    431         """Turn filename into path to store.
    432         """
    433150        return os.path.join(root, file_id[12:])
    434151
    … …  
    440157
    441158    layer = FunctionalLayer
    442 
    443159
    444160    def setUp(self):
    … …  
    449165        open(self.samplefile, 'wb').write('Hi there!')
    450166        open(self.otherfile, 'wb').write('Hi from other!')
    451         self.storage = ImageStorage()
    452167        self.fd = open(self.samplefile, 'r')
    453168        self.fd2 = open(self.otherfile, 'r')
Note: See TracChangeset for help on using the changeset viewer.