- Timestamp:
- 6 Jun 2012, 01:40:08 (13 years ago)
- Location:
- main/waeup.kofa/trunk/src/waeup/kofa
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
main/waeup.kofa/trunk/src/waeup/kofa/datacenter.py
r8592 r8634 31 31 IDataCenterStorageMovedEvent, 32 32 IDataCenterConfig) 33 from waeup.kofa.utils.helpers import copy_filesystem_tree 33 from waeup.kofa.utils.helpers import copy_filesystem_tree, merge_csv_files 34 34 from waeup.kofa.utils.logger import Logger 35 35 … … 130 130 os.unlink(source) 131 131 132 def _appendCSVFile(self, source, dest): 133 """Append data from CSV file `source` to data from CSV file `dest`. 134 135 The `source` file is deleted afterwards. 136 """ 137 if not os.path.exists(dest): 138 return self._moveFile(source, dest) 139 if not os.path.exists(source): 140 self.logger.warn('No such source path: %s' % source) 141 return 142 if source == dest: 143 return 144 result_path = merge_csv_files(dest, source) 145 self._moveFile(result_path, dest) 146 os.unlink(source) 147 132 148 def distProcessedFiles(self, successful, source_path, finished_file, 133 149 pending_file, mode='create', move_orig=True): … … 171 187 pending_dest = os.path.join(self.storage, pending_name) 172 188 finished_dest = os.path.join(finished_dir, finished_name) 173 self._ moveFile(finished_file, finished_dest)189 self._appendCSVFile(finished_file, finished_dest) 174 190 if pending_file is not None: 175 191 self._moveFile(pending_file, pending_dest) -
main/waeup.kofa/trunk/src/waeup/kofa/tests/test_datacenter.py
r8592 r8634 213 213 self.assertEqual(names, expected) 214 214 return 215 216 def test_append_csv_file(self): 217 # we can append CSV files to others 218 datacenter = DataCenter() 219 csv_file1 = os.path.join(self.workdir, 'foo.csv') 220 csv_file2 = os.path.join(self.workdir, 'bar.csv') 221 open(csv_file1, 'wb').write('name,age\nBarney,28') 222 open(csv_file2, 'wb').write('name,age\nManfred,28') 223 datacenter._appendCSVFile(csv_file2, csv_file1) 224 result = open(csv_file1, 'rb').read() 225 self.assertEqual(result, 'age,name\r\n28,Barney\r\n28,Manfred\r\n') 226 # The source is deleted afterwards 227 self.assertEqual(os.path.exists(csv_file2), False) 228 229 def test_append_csv_file_no_dest(self): 230 # a non-existing dest CSV file will result in a simple move 231 datacenter = DataCenter() 232 csv_file1 = os.path.join(self.workdir, 'foo.csv') 233 csv_file2 = os.path.join(self.workdir, 'bar.csv') 234 # csv_file1 does not exist 235 open(csv_file2, 'wb').write('name,age\nManfred,28\n') 236 datacenter._appendCSVFile(csv_file2, csv_file1) 237 result = open(csv_file1, 'rb').read() 238 # raw input, no CSV mangling 239 self.assertEqual(result, 'name,age\nManfred,28\n') 240 # The source is deleted afterwards 241 self.assertEqual(os.path.exists(csv_file2), False) 242 243 def test_append_csv_file_no_source(self): 244 # a non existing source file will mean no changes at all 245 datacenter = DataCenter() 246 csv_file1 = os.path.join(self.workdir, 'foo.csv') 247 csv_file2 = os.path.join(self.workdir, 'bar.csv') 248 open(csv_file1, 'wb').write('name,age\nManfred,28\n') 249 # csv_file2 does not exist 250 datacenter._appendCSVFile(csv_file2, csv_file1) 251 result = open(csv_file1, 'rb').read() 252 # csv_file1 is the same as before 253 self.assertEqual(result, 'name,age\nManfred,28\n') 254 255 def test_append_csv_file_same_src_and_dest(self): 256 # if both csv files are the same file, nothing will be changed 257 datacenter = DataCenter() 258 csv_file1 = os.path.join(self.workdir, 'foo.csv') 259 csv_file2 = csv_file1 260 open(csv_file1, 'wb').write('name,age\nManfred,28\n') 261 # csv_file2 does not exist 262 datacenter._appendCSVFile(csv_file2, csv_file1) 263 result = open(csv_file1, 'rb').read() 264 # csv_file1 is the same as before 265 self.assertEqual(result, 'name,age\nManfred,28\n') 266 self.assertEqual(os.path.exists(csv_file2), True)
Note: See TracChangeset for help on using the changeset viewer.