source: main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_batching.py @ 9747

Last change on this file since 9747 was 9739, checked in by Henrik Bettermann, 12 years ago

Put all information into a single logfile line. Otherwise we can't find the information via the UI.

  • Property svn:keywords set to Id
File size: 20.7 KB
Line 
1## $Id: test_batching.py 9739 2012-11-29 18:46:54Z henrik $
2##
3## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
4## This program is free software; you can redistribute it and/or modify
5## it under the terms of the GNU General Public License as published by
6## the Free Software Foundation; either version 2 of the License, or
7## (at your option) any later version.
8##
9## This program is distributed in the hope that it will be useful,
10## but WITHOUT ANY WARRANTY; without even the implied warranty of
11## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12## GNU General Public License for more details.
13##
14## You should have received a copy of the GNU General Public License
15## along with this program; if not, write to the Free Software
16## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17##
18import datetime
19import doctest
20import logging
21import os
22import shutil
23import tempfile
24import unittest
25from zc.async.interfaces import IJob
26from zope import schema
27from zope.component import provideUtility, getGlobalSiteManager, getUtility
28from zope.component.factory import Factory
29from zope.component.hooks import clearSite, setSite
30from zope.component.interfaces import IFactory
31from zope.interface import Interface, implements, verify
32from waeup.kofa.app import University
33from waeup.kofa.interfaces import (
34    ICSVExporter, IBatchProcessor, IExportJobContainer, IJobManager,
35    IExportJob, IExportContainerFinder)
36from waeup.kofa.testing import (
37    FunctionalLayer, FunctionalTestCase, FakeJob, FakeJobManager)
38from waeup.kofa.utils.batching import (
39    ExporterBase, BatchProcessor, export_job, AsyncExportJob,
40    ExportJobContainer, VirtualExportJobContainer, ExportContainerFinder)
41
42optionflags = (
43    doctest.REPORT_NDIFF + doctest.ELLIPSIS + doctest.NORMALIZE_WHITESPACE)
44
45
46class ICave(Interface):
47    """A cave."""
48    id_num = schema.TextLine(
49        title = u'internal id',
50        default = u'default',
51        required = True,
52        readonly = True,
53        )
54    name = schema.TextLine(
55        title = u'Cave name',
56        default = u'Unnamed',
57        required = True)
58    dinoports = schema.Int(
59        title = u'Number of DinoPorts (tm)',
60        required = False,
61        default = 1)
62    owner = schema.TextLine(
63        title = u'Owner name',
64        required = True,
65        missing_value = 'Fred Estates Inc.')
66    taxpayer = schema.Bool(
67        title = u'Payes taxes',
68        required = True,
69        default = False)
70
71class Cave(object):
72    implements(ICave)
73    def __init__(self, name=u'Unnamed', dinoports=2,
74                 owner='Fred Estates Inc.', taxpayer=False):
75        self.name = name
76        self.dinoports = 2
77        self.owner = owner
78        self.taxpayer = taxpayer
79
80stoneville = dict
81
82SAMPLE_DATA = """name,dinoports,owner,taxpayer
83Barneys Home,2,Barney,1
84Wilmas Asylum,1,Wilma,1
85Freds Dinoburgers,10,Fred,0
86Joeys Drive-in,110,Joey,0
87"""
88
89class CaveProcessor(BatchProcessor):
90    util_name = 'caveprocessor'
91    name = 'Cave Processor'
92    iface = ICave
93    location_fields = ['name']
94    factory_name = 'Lovely Cave'
95
96    def parentsExist(self, row, site):
97        return True
98
99    def getParent(self, row, site):
100        return stoneville
101
102    def entryExists(self, row, site):
103        return row['name'] in stoneville.keys()
104
105    def getEntry(self, row, site):
106        if not self.entryExists(row, site):
107            return None
108        return stoneville[row['name']]
109
110    def delEntry(self, row, site):
111        del stoneville[row['name']]
112
113    def addEntry(self, obj, row, site):
114        stoneville[row['name']] = obj
115
116class BatchProcessorTests(FunctionalTestCase):
117
118    layer = FunctionalLayer
119
120    def setupLogger(self):
121
122        self.logger = logging.getLogger('stoneville')
123        self.logger.setLevel(logging.DEBUG)
124        self.logger.propagate = False
125        self.logfile = os.path.join(self.workdir, 'stoneville.log')
126        self.handler = logging.FileHandler(self.logfile, 'w')
127        self.logger.addHandler(self.handler)
128
129    def setUp(self):
130        global stoneville
131        super(BatchProcessorTests, self).setUp()
132
133        # Setup a sample site for each test
134        app = University()
135        self.dc_root = tempfile.mkdtemp()
136        app['datacenter'].setStoragePath(self.dc_root)
137
138        # Prepopulate the ZODB...
139        self.getRootFolder()['app'] = app
140        self.app = self.getRootFolder()['app']
141
142        self.workdir = tempfile.mkdtemp()
143        factory = Factory(Cave)
144        provideUtility(factory, IFactory, 'Lovely Cave')
145
146        # Provide sample data
147        self.newcomers_csv = os.path.join(self.workdir, 'newcomers.csv')
148        open(self.newcomers_csv, 'wb').write(SAMPLE_DATA)
149        self.setupLogger()
150        self.stoneville = stoneville
151        stoneville = dict()
152        self.resultpath = None
153        return
154
155    def tearDown(self):
156        super(BatchProcessorTests, self).tearDown()
157        shutil.rmtree(self.workdir)
158        shutil.rmtree(self.dc_root)
159        self.logger.removeHandler(self.handler)
160        clearSite()
161        if not isinstance(self.resultpath, list):
162            self.resultpath = [self.resultpath]
163        for path in self.resultpath:
164            if not isinstance(path, basestring):
165                continue
166            if not os.path.isdir(path):
167                path = os.path.dirname(path)
168            if os.path.exists(path):
169                shutil.rmtree(path)
170        return
171
172    def test_iface(self):
173        # make sure we fullfill interface contracts
174        obj = BatchProcessor()
175        verify.verifyClass(IBatchProcessor, BatchProcessor)
176        verify.verifyObject(IBatchProcessor, obj)
177        return
178
179    def test_import(self):
180        processor = CaveProcessor()
181        result = processor.doImport(
182            self.newcomers_csv,
183            ['name', 'dinoports', 'owner', 'taxpayer'],
184            mode='create', user='Bob', logger=self.logger)
185        num_succ, num_fail, finished_path, failed_path = result
186        self.resultpath = [finished_path, failed_path]
187        assert num_succ == 4
188        assert num_fail == 0
189        assert finished_path.endswith('/newcomers.finished.csv')
190        assert failed_path is None
191
192    def test_import_stoneville(self):
193        processor = CaveProcessor()
194        result = processor.doImport(
195            self.newcomers_csv,
196            ['name', 'dinoports', 'owner', 'taxpayer'],
197            mode='create', user='Bob', logger=self.logger)
198        num_succ, num_fail, finished_path, failed_path = result
199        self.resultpath = [finished_path, failed_path]
200        assert len(self.stoneville) == 4
201        self.assertEqual(
202            sorted(self.stoneville.keys()),
203            [u'Barneys Home', u'Freds Dinoburgers',
204             u'Joeys Drive-in', u'Wilmas Asylum'])
205
206    def test_import_correct_type(self):
207        processor = CaveProcessor()
208        result = processor.doImport(
209            self.newcomers_csv,
210            ['name', 'dinoports', 'owner', 'taxpayer'],
211            mode='create', user='Bob', logger=self.logger)
212        num_succ, num_fail, finished_path, failed_path = result
213        self.resultpath = [finished_path, failed_path]
214        assert isinstance(self.stoneville['Barneys Home'].dinoports, int)
215
216
217    def test_log(self):
218        """
219           >>> print log_contents
220           processed: /.../newcomers.csv, create mode, 4 lines (4 successful/ 0 failed), ... s (... s/item)
221
222        """
223        processor = CaveProcessor()
224        result = processor.doImport(
225            self.newcomers_csv,
226            ['name', 'dinoports', 'owner', 'taxpayer'],
227            mode='create', user='Bob', logger=self.logger)
228        num_succ, num_fail, finished_path, failed_path = result
229        self.resultpath = [finished_path, failed_path]
230        log_contents = open(self.logfile, 'rb').read()
231        doctest.run_docstring_examples(
232            self.test_log, locals(), False, 'test_log', None, optionflags)
233        return
234
235class ExporterBaseTests(unittest.TestCase):
236
237    def setUp(self):
238        self.workdir = tempfile.mkdtemp()
239        self.workfile = os.path.join(self.workdir, 'testfile.csv')
240        return
241
242    def tearDown(self):
243        shutil.rmtree(self.workdir)
244        return
245
246    def test_iface(self):
247        # ExporterBase really implements the promised interface.
248        obj = ExporterBase()
249        verify.verifyClass(ICSVExporter, ExporterBase)
250        verify.verifyObject(ICSVExporter, obj)
251        return
252
253    def test_unimplemented(self):
254        # make sure the not implemented methods signal that.
255        exporter = ExporterBase()
256        self.assertRaises(NotImplementedError, exporter.export_all, None)
257        self.assertRaises(NotImplementedError, exporter.export, None)
258        return
259
260    def test_mangle_value(self):
261        # some basic types are mangled correctly
262        exporter = ExporterBase()
263        result1 = exporter.mangle_value(True, 'foo')
264        result2 = exporter.mangle_value(False, 'foo')
265        result3 = exporter.mangle_value('string', 'foo')
266        result4 = exporter.mangle_value(u'string', 'foo')
267        result5 = exporter.mangle_value(None, 'foo')
268        result6 = exporter.mangle_value(datetime.date(2012, 4, 1), 'foo')
269        result7 = exporter.mangle_value(
270            datetime.datetime(2012, 4, 1, 12, 1, 1), 'foo')
271        self.assertEqual(
272            (result1, result2, result3, result4, result5),
273            ('1', '0', u'string', u'string', ''))
274        self.assertEqual(type(result3), type('string'))
275        self.assertEqual(type(result4), type('string'))
276        # dates are formatted with trailing hash
277        self.assertEqual(result6, '2012-04-01#')
278        # datetimes are formatted as yyyy-mm-dd hh:mm:ss
279        self.assertEqual(result7, '2012-04-01 12:01:01')
280        return
281
282    def test_get_csv_writer(self):
283        # we can get a CSV writer to a memory file
284        exporter = ExporterBase()
285        writer, outfile = exporter.get_csv_writer()
286        writer.writerow(dict(code='A', title='B', title_prefix='C'))
287        outfile.seek(0)
288        self.assertEqual(
289            outfile.read(),
290            'code,title,title_prefix\r\nA,B,C\r\n')
291        return
292
293    def test_get_csv_writer_with_file(self):
294        # we can get CSV writer that writes to a real file
295        exporter = ExporterBase()
296        writer, outfile = exporter.get_csv_writer(filepath=self.workfile)
297        writer.writerow(dict(code='A', title='B', title_prefix='C'))
298        outfile.close()
299        resultfile = open(self.workfile, 'rb')
300        self.assertEqual(
301            resultfile.read(),
302            'code,title,title_prefix\r\nA,B,C\r\n')
303        return
304
305    def test_write_item(self):
306        # we can write items to opened exporter files.
307        exporter = ExporterBase()
308        writer, outfile = exporter.get_csv_writer()
309        class Sample(object):
310            code = 'A'
311            title = u'B'
312            title_prefix = True
313        exporter.write_item(Sample(), writer)
314        outfile.seek(0)
315        self.assertEqual(
316            outfile.read(),
317            'code,title,title_prefix\r\nA,B,1\r\n')
318        return
319
320    def test_close_outfile(self):
321        # exporters can help to close outfiles.
322        exporter = ExporterBase()
323        writer, outfile = exporter.get_csv_writer()
324        result = exporter.close_outfile(None, outfile)
325        self.assertEqual(result, 'code,title,title_prefix\r\n')
326        return
327
328    def test_close_outfile_real(self):
329        # we can also close outfiles in real files.
330        exporter = ExporterBase()
331        writer, outfile = exporter.get_csv_writer(filepath=self.workfile)
332        result = exporter.close_outfile(self.workfile, outfile)
333        self.assertEqual(result, None)
334        return
335
336
337class CaveExporter(ExporterBase):
338    # A minimal fake exporter suitable to be called by export_jobs
339    fields = ('name', 'dinoports', 'owner', 'taxpayer')
340    title = u'Dummy cave exporter'
341
342    def export_all(self, site, filepath=None):
343        if filepath is None:
344            return SAMPLE_DATA
345        open(filepath, 'wb').write(SAMPLE_DATA)
346        return
347
348class ExportJobTests(unittest.TestCase):
349    # Test asynchronous export functionality (simple cases)
350
351    def setUp(self):
352        # register a suitable ICSVExporter as named utility
353        self.exporter = CaveExporter()
354        self.gsm = getGlobalSiteManager()
355        self.gsm.registerUtility(
356            self.exporter, ICSVExporter, name='cave_exporter')
357
358    def tearDown(self):
359        self.gsm.unregisterUtility(self.exporter)
360
361    def test_export_job_func(self):
362        # the export_job func does really export data...
363        result_path = export_job(None, 'cave_exporter')
364        self.assertTrue(os.path.isfile(result_path))
365        contents = open(result_path, 'rb').read()
366        shutil.rmtree(os.path.dirname(result_path))
367        self.assertEqual(contents, SAMPLE_DATA)
368        return
369
370    def test_export_job_interfaces(self):
371        # the AsyncExportJob implements promised interfaces correctly...
372        job = AsyncExportJob(None, None)
373        verify.verifyClass(IJob, AsyncExportJob)
374        verify.verifyObject(IJob, job)
375        verify.verifyClass(IExportJob, AsyncExportJob)
376        verify.verifyObject(IExportJob, job)
377        return
378
379
380class FakeJobWithResult(FakeJob):
381
382    def __init__(self):
383        self.dir_path = tempfile.mkdtemp()
384        self.result = os.path.join(self.dir_path, 'fake.csv')
385        open(self.result, 'wb').write('a fake result')
386        return
387
388class ExportJobContainerTests(unittest.TestCase):
389    # Test ExportJobContainer
390
391    TestedClass = ExportJobContainer
392
393    def setUp(self):
394        # register a suitable ICSVExporter as named utility
395        self.exporter = CaveExporter()
396        self.job_manager = FakeJobManager()
397        self.gsm = getGlobalSiteManager()
398        self.gsm.registerUtility(
399            self.exporter, ICSVExporter, name='cave_exporter')
400        self.gsm.registerUtility(
401            self.job_manager, IJobManager)
402
403    def tearDown(self):
404        self.gsm.unregisterUtility(self.exporter)
405        self.gsm.unregisterUtility(self.job_manager, IJobManager)
406
407    def test_export_job_interfaces(self):
408        # the ExportJobContainer implements promised interfaces correctly...
409        container = self.TestedClass()
410        verify.verifyClass(IExportJobContainer, self.TestedClass)
411        verify.verifyObject(IExportJobContainer, container)
412        return
413
414    def test_start_export_job(self):
415        # we can start jobs
416        container = self.TestedClass()
417        container.start_export_job('cave_exporter', 'bob')
418        result = self.job_manager._jobs.values()[0]
419        self.assertTrue(IJob.providedBy(result))
420        self.assertEqual(
421            container.running_exports,
422            [('1', 'cave_exporter', 'bob')]
423            )
424        return
425
426    def test_get_running_export_jobs_all(self):
427        # we can get export jobs of all users
428        container = self.TestedClass()
429        container.start_export_job('cave_exporter', 'bob')
430        container.start_export_job('cave_exporter', 'alice')
431        result = container.get_running_export_jobs()
432        self.assertEqual(
433            result,
434            [('1', 'cave_exporter', 'bob'),
435             ('2', 'cave_exporter', 'alice')]
436            )
437        return
438
439    def test_get_running_export_jobs_user(self):
440        # we can get the export jobs running for a certain user
441        container = self.TestedClass()
442        container.start_export_job('cave_exporter', 'bob')
443        container.start_export_job('cave_exporter', 'alice')
444        result1 = container.get_running_export_jobs(user_id='alice')
445        result2 = container.get_running_export_jobs(user_id='foo')
446        self.assertEqual(
447            result1, [('2', 'cave_exporter', 'alice')])
448        self.assertEqual(
449            result2, [])
450        return
451
452    def test_get_running_export_jobs_only_if_exist(self):
453        # we get only jobs that are accessible through the job manager...
454        container = self.TestedClass()
455        container.start_export_job('cave_exporter', 'bob')
456        container.start_export_job('cave_exporter', 'bob')
457        self.assertTrue(
458            ('2', 'cave_exporter', 'bob') in container.running_exports)
459        # we remove the second entry from job manager
460        del self.job_manager._jobs['2']
461        result = container.get_running_export_jobs(user_id='bob')
462        self.assertEqual(
463            result, [('1', 'cave_exporter', 'bob')])
464        self.assertTrue(
465            ('2', 'cave_exporter', 'bob') not in container.running_exports)
466        return
467
468    def test_get_export_job_status(self):
469        # we can get the stati of jobs...
470        container = self.TestedClass()
471        container.start_export_job('cave_exporter', 'alice')
472        container.start_export_job('cave_exporter', 'bob')
473        container.start_export_job('cave_exporter', 'bob')
474        result = container.get_export_jobs_status(user_id='bob')
475        # we'll get the raw value, a translation and the title of the
476        # exporter
477        self.assertEqual(
478            result,
479            [('new', u'new', u'Dummy cave exporter'),
480             ('completed', u'completed', u'Dummy cave exporter')]
481            )
482        return
483
484    def test_delete_export_entry(self):
485        # we can remove export entries in local lists and the job
486        # manager as well...
487        container = self.TestedClass()
488        container.start_export_job('cave_exporter', 'bob')
489        entry = container.running_exports[0]
490        container.delete_export_entry(entry)
491        # both, running_exports list and job manager are empty now
492        self.assertEqual(
493            container.running_exports, [])
494        self.assertEqual(
495            self.job_manager._jobs, {})
496        return
497
498    def test_delete_export_entry_remove_file(self):
499        # any result files of exports are deleted as well
500        container = self.TestedClass()
501        entry = ('4', 'cave_exporter', 'bob')
502        container.running_exports = [entry]
503        fake_job = FakeJobWithResult()
504        self.job_manager._jobs['4'] = fake_job
505        self.assertTrue(os.path.isfile(fake_job.result))
506        container.delete_export_entry(entry)
507        self.assertTrue(not os.path.exists(fake_job.result))
508        return
509
510    def test_entry_from_job_id(self):
511        # we can get an entry for a job_id if the id exists
512        container = self.TestedClass()
513        entry = ('4', 'cave_exporter', 'bob')
514        container.running_exports = [entry]
515        fake_job = FakeJobWithResult()
516        self.job_manager._jobs['4'] = fake_job
517        result1 = container.entry_from_job_id(None)
518        result2 = container.entry_from_job_id('4')
519        result3 = container.entry_from_job_id('23')
520        self.assertEqual(result1, None)
521        self.assertEqual(result2, ('4', 'cave_exporter', 'bob'))
522        self.assertEqual(result3, None)
523        shutil.rmtree(fake_job.dir_path)
524        return
525
526class VirtualExportJobContainerTests(ExportJobContainerTests):
527    # VirtualExportJobContainers should provide the
528    # same functionality as regular ones.
529
530    TestedClass = VirtualExportJobContainer
531
532    def setUp(self):
533        super(VirtualExportJobContainerTests, self).setUp()
534        self.root_job_container = ExportJobContainer()
535        def fake_finder():
536            return self.root_job_container
537        self.gsm = getGlobalSiteManager()
538        self.gsm.registerUtility(fake_finder, IExportContainerFinder)
539        return
540
541class ExportContainerFinderTests(FunctionalTestCase):
542    # Tests for export container finder.
543
544    layer = FunctionalLayer
545
546    def test_get_finder_as_util(self):
547        # we can get a finder by utility lookup
548        finder = getUtility(IExportContainerFinder)
549        self.assertTrue(finder is not None)
550        self.assertEqual(
551            IExportContainerFinder.providedBy(finder),
552            True)
553        return
554
555    def test_iface(self):
556        # the finder complies with the promised interface
557        finder = ExportContainerFinder()
558        verify.verifyClass(IExportContainerFinder,
559                           ExportContainerFinder)
560        verify.verifyObject(IExportContainerFinder, finder)
561        return
562
563    def test_no_site(self):
564        # a finder returns None if no site is available
565        finder = ExportContainerFinder()
566        self.assertEqual(
567            finder(), None)
568        return
569
570    def test_active_site(self):
571        # we get the datafinder if one is installed and site set
572        self.getRootFolder()['app'] = University()
573        finder = getUtility(IExportContainerFinder)
574        setSite(self.getRootFolder()['app'])
575        container = finder()
576        self.assertTrue(container is not None)
577        return
578
579    def test_broken_site(self):
580        # if the current site has no ExportContainer, we get None
581        self.getRootFolder()['app'] = University()
582        app = self.getRootFolder()['app']
583        del app['datacenter'] # datacenter _is_ the export container
584        setSite(app)
585        finder = getUtility(IExportContainerFinder)
586        container = finder()
587        self.assertTrue(container is None)
588        return
Note: See TracBrowser for help on using the repository browser.