Changeset 10676

Timestamp:

31 Oct 2013, 17:51:25 (11 years ago)

Author:

uli

Message:

Check uploaded CSV files more thoroughly before processing.

Location:

main/waeup.kofa/trunk/src/waeup/kofa

Files:

• browser/pages.py (modified) (2 diffs)
• browser/tests/test_browser.py (modified) (1 diff)
• utils/helpers.py (modified) (1 diff)
• utils/tests/test_helpers.py (modified) (2 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/browser/pages.py

@@ -69 +69 @@
 from waeup.kofa.authentication import LocalRoleSetEvent
 from waeup.kofa.widgets.htmlwidget import HTMLDisplayWidget
-from waeup.kofa.utils.helpers import get_user_account
+from waeup.kofa.utils.helpers import get_user_account, check_csv_charset
 from waeup.kofa.mandates.mandate import PasswordMandate
 from waeup.kofa.datacenter import DataCenterFile
@@ -1065 +1065 @@
 
             # Forbid certain characters in import files.
-            for element in filecontent:
-                try:
-                    if ord(element) in FORBIDDEN_CHARACTERS:
-                      self.flash(_(
-                          "Your file contains forbidden characters. "
-                          "Please replace."))
-                      logger.info('%s - invalid file uploaded: %s' %
-                          (ob_class, target))
-                      return
-                except TypeError:
-                    self.flash(_(
-                        "Your file contains forbidden characters. "
-                        "Please replace."))
-                    logger.info('%s - invalid file uploaded: %s' %
-                        (ob_class, target))
-                    return
+            invalid_line = check_csv_charset(filecontent.splitlines())
+            if invalid_line:
+                self.flash(_(
+                    "Your file contains forbidden characters or "
+                    "has invalid CSV format. "
+                    "First problematic line detected: line %s. "
+                    "Please replace." % invalid_line))
+                logger.info('%s - invalid file uploaded: %s' %
+                            (ob_class, target))
+                return
 
             open(target, 'wb').write(filecontent)

main/waeup.kofa/trunk/src/waeup/kofa/browser/tests/test_browser.py

@@ -157 +157 @@
         self.browser.getControl('Upload').click()
         self.assertTrue(
-            'Your file contains forbidden characters. Please replace.'
+            'Your file contains forbidden characters or'
             in self.browser.contents)
         logfile = os.path.join(

main/waeup.kofa/trunk/src/waeup/kofa/utils/helpers.py

@@ -713 +713 @@
     def emit(self, record):
         pass
+
+
+def check_csv_charset(iterable):
+    """Check contents of `iterable` regarding valid CSV encoding.
+
+    `iterable` is expected to be an iterable on _rows_ (not
+    chars). This is true for instance for
+    filehandlers. `zope.publisher.browser.FileUpload` instances are
+    _not_ iterable, unfortunately.
+
+    Returns line num of first illegal char or ``None``. Line nums
+    start counting with 1 (not zero).
+    """
+    linenum = 1
+    reader = csv.DictReader(iterable)
+    try:
+        for row in reader:
+            linenum += 1
+    except UnicodeDecodeError:
+        return linenum
+    except:
+        return linenum + 1
+    return None

main/waeup.kofa/trunk/src/waeup/kofa/utils/tests/test_helpers.py

@@ -492 +492 @@
 
 
+class CheckCSVCharsetTestCase(unittest.TestCase):
+
+    def test_valid_data1(self):
+        csv = (
+            'col1,col2,col3\n'
+            'val1,val2,val3\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), None)
+
+    def test_valid_data2(self):
+        csv = (
+            "code,title,title_prefix\n"
+            "FAC1,Faculty 1,faculty\n"
+            "FAC2,Faculty 2,institute\n"
+            "FAC3,Fäcülty 3,school\n"
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), None)
+
+    def test_invalid_data1(self):
+        csv = (
+            'col1,col2,col3\n' +
+            chr(0x92) + 'val1,val2,val3\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), 1)
+
+    def test_invalid_data2(self):
+        csv = (
+            'some text that \n'
+            '\n'      # this empty line will break
+            'is not a csv file \n' + chr(0x92) + '\n'
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), 2)
 
 def test_suite():
@@ -509 +541 @@
         MergeCSVFileTestCase,
         SimpleHelpersTestCase,
+        CheckCSVCharsetTestCase,
         ]:
         suite.addTests(