Changeset 10676 for main/waeup.kofa/trunk/src/waeup/kofa/utils

Timestamp:

31 Oct 2013, 17:51:25 (11 years ago)

Author:

uli

Message:

Check uploaded CSV files more thoroughly before processing.

Location:

main/waeup.kofa/trunk/src/waeup/kofa/utils

Files:

• helpers.py (modified) (1 diff)
• tests/test_helpers.py (modified) (2 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/utils/helpers.py

@@ -713 +713 @@
     def emit(self, record):
         pass
+
+
+def check_csv_charset(iterable):
+    """Check contents of `iterable` regarding valid CSV encoding.
+
+    `iterable` is expected to be an iterable on _rows_ (not
+    chars). This is true for instance for
+    filehandlers. `zope.publisher.browser.FileUpload` instances are
+    _not_ iterable, unfortunately.
+
+    Returns line num of first illegal char or ``None``. Line nums
+    start counting with 1 (not zero).
+    """
+    linenum = 1
+    reader = csv.DictReader(iterable)
+    try:
+        for row in reader:
+            linenum += 1
+    except UnicodeDecodeError:
+        return linenum
+    except:
+        return linenum + 1
+    return None

main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_helpers.py

@@ -492 +492 @@
 
 
+class CheckCSVCharsetTestCase(unittest.TestCase):
+
+    def test_valid_data1(self):
+        csv = (
+            'col1,col2,col3\n'
+            'val1,val2,val3\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), None)
+
+    def test_valid_data2(self):
+        csv = (
+            "code,title,title_prefix\n"
+            "FAC1,Faculty 1,faculty\n"
+            "FAC2,Faculty 2,institute\n"
+            "FAC3,Fäcülty 3,school\n"
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), None)
+
+    def test_invalid_data1(self):
+        csv = (
+            'col1,col2,col3\n' +
+            chr(0x92) + 'val1,val2,val3\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), 1)
+
+    def test_invalid_data2(self):
+        csv = (
+            'some text that \n'
+            '\n'      # this empty line will break
+            'is not a csv file \n' + chr(0x92) + '\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), 2)
 
 def test_suite():
@@ -509 +541 @@
         MergeCSVFileTestCase,
         SimpleHelpersTestCase,
+        CheckCSVCharsetTestCase,
         ]:
         suite.addTests(