Changeset 4897 for waeup/trunk

Timestamp:

27 Jan 2010, 11:33:44 (15 years ago)

Author:

uli

Message:

Add tests for distributing files.

File:

• waeup/trunk/src/waeup/datacenter.txt (modified) (1 diff)

waeup/trunk/src/waeup/datacenter.txt

@@ -97 +97 @@
     >>> shutil.rmtree(verynewpath)
 
+
+Distributing processed files
+============================
+
+When files were processed by a batch processor, we can put the
+resulting files into desired destinations.
+
+We recreate the datacenter root in case it is missing:
+
+    >>> import os
+    >>> dc_root = mydatacenter.storage
+    >>> fin_dir = os.path.join(dc_root, 'finished')
+    >>> unfin_dir = os.path.join(dc_root, 'unfinished')
+
+    >>> def recreate_dc_storage():
+    ...   if os.path.exists(dc_root):
+    ...     shutil.rmtree(dc_root)
+    ...   os.mkdir(dc_root)
+    ...   mydatacenter.setStoragePath(mydatacenter.storage)
+    >>> recreate_dc_storage()
+
+We define a function that creates a set of faked result files:
+
+    >>> import os
+    >>> import tempfile
+    >>> def create_fake_results(source_basename, create_pending=True):
+    ...   tmp_dir = tempfile.mkdtemp()
+    ...   src = os.path.join(dc_root, source_basename)
+    ...   pending_src = None
+    ...   if create_pending:
+    ...     pending_src = os.path.join(tmp_dir, 'mypendingsource.csv')
+    ...   finished_src = os.path.join(tmp_dir, 'myfinishedsource.csv')
+    ...   for path in (src, pending_src, finished_src):
+    ...     if path is not None:
+    ...       open(path, 'wb').write('blah')
+    ...   return tmp_dir, src, finished_src, pending_src
+
+Now we can create the set of result files, that typically come after a
+successful processing of a regular source:
+
+Now we can try to distribute those files. Let's start with a source
+file, that was processed successfully:
+
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.csv', create_pending=False)
+    >>> mydatacenter.distProcessedFiles(True, src, finished_src,
+    ...                            pending_src)
+    >>> sorted(os.listdir(dc_root))
+    ['finished', 'logs', 'unfinished']
+
+    >>> sorted(os.listdir(fin_dir))
+    ['mysource.csv', 'mysource.finished.csv']
+
+    >>> sorted(os.listdir(unfin_dir))
+    []
+
+    >>> shutil.rmtree(tmp_dir)
+
+The root dir is empty, while the original file and the file containing
+all processed data were moved to'finished/'.
+
+Now we restart, but this time we fake an erranous action:
+
+    >>> recreate_dc_storage()
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.csv')
+    >>> mydatacenter.distProcessedFiles(False, src, finished_src,
+    ...                                 pending_src)
+    >>> sorted(os.listdir(dc_root))
+    ['finished', 'logs', 'mysource.pending.csv', 'unfinished']
+
+    >>> sorted(os.listdir(fin_dir))
+    ['mysource.finished.csv']
+
+    >>> sorted(os.listdir(unfin_dir))
+    ['mysource.csv']
+
+    >>> shutil.rmtree(tmp_dir)
+
+While the original source was moved to the 'unfinished' dir, the
+pending file went to the root and the set of already processed items
+are stored in finished/.
+
+We fake processing the pending file and assume that everything went
+well this time:
+
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.pending.csv', create_pending=False)
+    >>> mydatacenter.distProcessedFiles(True, src, finished_src,
+    ...                                 pending_src)
+
+    >>> sorted(os.listdir(dc_root))
+    ['finished', 'logs', 'unfinished']
+
+    >>> sorted(os.listdir(fin_dir))
+    ['mysource.csv', 'mysource.finished.csv']
+
+    >>> sorted(os.listdir(unfin_dir))
+    []
+
+    >>> shutil.rmtree(tmp_dir)
+
+The result is the same as in the first case shown above.
+
+We restart again, but this time we fake several non-working imports in
+a row.
+
+We start with a faulty start-import:
+
+    >>> recreate_dc_storage()
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.csv')
+    >>> mydatacenter.distProcessedFiles(False, src, finished_src,
+    ...                                 pending_src)
+    >>> shutil.rmtree(tmp_dir)
+
+We try to process the pending file, which fails again:
+
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.pending.csv')
+    >>> mydatacenter.distProcessedFiles(False, src, finished_src,
+    ...                                 pending_src)
+    >>> shutil.rmtree(tmp_dir)
+
+We try to process the new pending file:
+
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.pending.csv')
+    >>> mydatacenter.distProcessedFiles(False, src, finished_src,
+    ...                                 pending_src)
+
+    >>> sorted(os.listdir(dc_root))
+    ['finished', 'logs', 'mysource.pending.csv', 'unfinished']
+
+    >>> sorted(os.listdir(fin_dir))
+    ['mysource.finished.csv']
+
+    >>> sorted(os.listdir(unfin_dir))
+    ['mysource.csv']
+
+    >>> shutil.rmtree(tmp_dir)
+
+Finally, we process the pending file and everything works:
+
+    >>> tmp_dir, src, finished_src, pending_src = create_fake_results(
+    ...  'mysource.pending.csv', create_pending=False)
+    >>> mydatacenter.distProcessedFiles(True, src, finished_src,
+    ...                                 pending_src)
+
+    >>> sorted(os.listdir(dc_root))
+    ['finished', 'logs', 'unfinished']
+
+    >>> sorted(os.listdir(fin_dir))
+    ['mysource.csv', 'mysource.finished.csv']
+
+    >>> sorted(os.listdir(unfin_dir))
+    []
+
+The root dir is empty (contains no input files) and only the files in
+finished-subdirectory remain.
+
+Clean up:
+
+    >>> shutil.rmtree(tmp_dir)
+    >>> shutil.rmtree(verynewpath)
 
 Handling imports