Changeset 8634

Timestamp:

6 Jun 2012, 01:40:08 (13 years ago)

Author:

uli

Message:

Let datacenter use the new CSV file merger from helpers.

CAUTION: this is a bit tested, but not in real use. Try
some local imports before using it in production environment!

Location:

main/waeup.kofa/trunk/src/waeup/kofa

Files:

• datacenter.py (modified) (3 diffs)
• tests/test_datacenter.py (modified) (1 diff)

main/waeup.kofa/trunk/src/waeup/kofa/datacenter.py

@@ -31 +31 @@
                                    IDataCenterStorageMovedEvent,
                                    IDataCenterConfig)
-from waeup.kofa.utils.helpers import copy_filesystem_tree
+from waeup.kofa.utils.helpers import copy_filesystem_tree, merge_csv_files
 from waeup.kofa.utils.logger import Logger
 
@@ -130 +130 @@
         os.unlink(source)
 
+    def _appendCSVFile(self, source, dest):
+        """Append data from CSV file `source` to data from CSV file `dest`.
+
+        The `source` file is deleted afterwards.
+        """
+        if not os.path.exists(dest):
+            return self._moveFile(source, dest)
+        if not os.path.exists(source):
+            self.logger.warn('No such source path: %s' % source)
+            return
+        if source == dest:
+            return
+        result_path = merge_csv_files(dest, source)
+        self._moveFile(result_path, dest)
+        os.unlink(source)
+
     def distProcessedFiles(self, successful, source_path, finished_file,
                            pending_file, mode='create', move_orig=True):
@@ -171 +187 @@
         pending_dest = os.path.join(self.storage, pending_name)
         finished_dest = os.path.join(finished_dir, finished_name)
-        self._moveFile(finished_file, finished_dest)
+        self._appendCSVFile(finished_file, finished_dest)
         if pending_file is not None:
             self._moveFile(pending_file, pending_dest)

main/waeup.kofa/trunk/src/waeup/kofa/tests/test_datacenter.py

@@ -213 +213 @@
         self.assertEqual(names, expected)
         return
+
+    def test_append_csv_file(self):
+        # we can append CSV files to others
+        datacenter = DataCenter()
+        csv_file1 = os.path.join(self.workdir, 'foo.csv')
+        csv_file2 = os.path.join(self.workdir, 'bar.csv')
+        open(csv_file1, 'wb').write('name,age\nBarney,28')
+        open(csv_file2, 'wb').write('name,age\nManfred,28')
+        datacenter._appendCSVFile(csv_file2, csv_file1)
+        result = open(csv_file1, 'rb').read()
+        self.assertEqual(result, 'age,name\r\n28,Barney\r\n28,Manfred\r\n')
+        # The source is deleted afterwards
+        self.assertEqual(os.path.exists(csv_file2), False)
+
+    def test_append_csv_file_no_dest(self):
+        # a non-existing dest CSV file will result in a simple move
+        datacenter = DataCenter()
+        csv_file1 = os.path.join(self.workdir, 'foo.csv')
+        csv_file2 = os.path.join(self.workdir, 'bar.csv')
+        # csv_file1 does not exist
+        open(csv_file2, 'wb').write('name,age\nManfred,28\n')
+        datacenter._appendCSVFile(csv_file2, csv_file1)
+        result = open(csv_file1, 'rb').read()
+        # raw input, no CSV mangling
+        self.assertEqual(result, 'name,age\nManfred,28\n')
+        # The source is deleted afterwards
+        self.assertEqual(os.path.exists(csv_file2), False)
+
+    def test_append_csv_file_no_source(self):
+        # a non existing source file will mean no changes at all
+        datacenter = DataCenter()
+        csv_file1 = os.path.join(self.workdir, 'foo.csv')
+        csv_file2 = os.path.join(self.workdir, 'bar.csv')
+        open(csv_file1, 'wb').write('name,age\nManfred,28\n')
+        # csv_file2 does not exist
+        datacenter._appendCSVFile(csv_file2, csv_file1)
+        result = open(csv_file1, 'rb').read()
+        # csv_file1 is the same as before
+        self.assertEqual(result, 'name,age\nManfred,28\n')
+
+    def test_append_csv_file_same_src_and_dest(self):
+        # if both csv files are the same file, nothing will be changed
+        datacenter = DataCenter()
+        csv_file1 = os.path.join(self.workdir, 'foo.csv')
+        csv_file2 = csv_file1
+        open(csv_file1, 'wb').write('name,age\nManfred,28\n')
+        # csv_file2 does not exist
+        datacenter._appendCSVFile(csv_file2, csv_file1)
+        result = open(csv_file1, 'rb').read()
+        # csv_file1 is the same as before
+        self.assertEqual(result, 'name,age\nManfred,28\n')
+        self.assertEqual(os.path.exists(csv_file2), True)