Changeset 4896 for waeup/trunk

Timestamp:

27 Jan 2010, 11:32:58 (15 years ago)

Author:

uli

Message:

Add methods to store processed files into desired locations.

File:

• waeup/trunk/src/waeup/datacenter.py (modified) (3 diffs)

waeup/trunk/src/waeup/datacenter.py

@@ -6 +6 @@
 import logging
 import os
+import shutil
 import struct
 import grok
@@ -65 +66 @@
         """Create standard subdirs.
         """
-        for name in ['finished', 'pending']:
+        for name in ['finished', 'unfinished']:
             path = os.path.join(self.storage, name)
             if os.path.exists(path):
@@ -254 +255 @@
         return
 
-
+    def _moveFile(self, source, dest):
+        """Move file source to dest preserving ctime, mtime, etc.
+        """
+        if not os.path.exists(source):
+            self.logger.warn('No such source path: %s' % source)
+            return
+        if source == dest:
+            return
+        shutil.copyfile(source, dest)
+        shutil.copystat(source, dest)
+        os.unlink(source)
+    
+    def distProcessedFiles(self, successful, source_path, finished_file,
+                           pending_file, move_orig=True):
+        """Put processed files into final locations.
+
+        ``successful`` is a boolean that tells, whether processing was
+        successful.
+
+        ``source_path``: path to file that was processed.
+
+        ``finished_file``, ``pending_file``: paths to the respective
+        generated .pending and .finished file. The .pending file path
+        may be ``None``.
+
+        See datacenter.txt for more info about how this works.
+        """
+        basename = os.path.basename(source_path)
+        pending_name = basename
+        pending = False
+        finished_dir = os.path.join(self.storage, 'finished')
+        unfinished_dir = os.path.join(self.storage, 'unfinished')
+
+        if basename.endswith('.pending.csv'):
+            maybe_basename = "%s.csv" % basename.split('.', 2)[0]
+            maybe_src = os.path.join(unfinished_dir, maybe_basename)
+            if os.path.isfile(maybe_src):
+                basename = maybe_basename
+                pending = True
+                
+        base, ext = os.path.splitext(basename)
+        finished_name = "%s.finished%s" % (base, ext)
+        if not pending:
+            pending_name = "%s.pending%s" % (base, ext)
+
+        # Put .pending and .finished file into respective places...
+        pending_dest = os.path.join(self.storage, pending_name)
+        finished_dest = os.path.join(finished_dir, finished_name)
+        self._moveFile(finished_file, finished_dest)
+        if pending_file is not None:
+            self._moveFile(pending_file, pending_dest)
+
+        # Put source file into final location...
+        finished_dest = os.path.join(finished_dir, basename)
+        unfinished_dest = os.path.join(unfinished_dir, basename)
+        if successful and not pending:
+            self._moveFile(source_path, finished_dest)
+        elif successful and pending:
+            self._moveFile(unfinished_dest, finished_dest)
+            os.unlink(source_path)
+        elif not successful and not pending:
+            self._moveFile(source_path, unfinished_dest)
+        return
+
+    
 class DataCenterFile(object):
     """A description of a file stored in data center.