Changeset 15258 for main/waeup.ikoba/trunk/src

Timestamp:

30 Nov 2018, 07:34:44 (6 years ago)

Author:

Henrik Bettermann

Message:

Do not allow uploading data with trailing whitespaces.

Location:

main/waeup.ikoba/trunk/src/waeup/ikoba

Files:

• browser/pages.py (modified) (1 diff)
• browser/tests/forbidden_file_2.csv (added)
• browser/tests/test_browser.py (modified) (2 diffs)
• utils/helpers.py (modified) (2 diffs)
• utils/tests/test_helpers.py (modified) (1 diff)

main/waeup.ikoba/trunk/src/waeup/ikoba/browser/pages.py

@@ -1142 +1142 @@
             invalid_line = check_csv_charset(filecontent.splitlines())
             if invalid_line:
-                self.flash(_(
-                    "Your file contains forbidden characters or "
-                    "has invalid CSV format. "
-                    "First problematic line detected: line %s. "
-                    "Please replace." % invalid_line), type='danger')
+                if invalid_line == -1:
+                    self.flash(_(
+                        "The data in your file contain trailing whitespaces."
+                        "Please replace."), type='danger')
+                else:
+                    self.flash(_(
+                        "Your file contains forbidden characters or "
+                        "has invalid CSV format. "
+                        "First problematic line detected: line %s. "
+                        "Please replace." % invalid_line), type='danger')
                 logger.info('%s - invalid file uploaded: %s' %
                             (ob_class, target))

main/waeup.ikoba/trunk/src/waeup/ikoba/browser/tests/test_browser.py

@@ -36 +36 @@
 from waeup.ikoba.tests.test_async import FunctionalAsyncTestCase
 
+SAMPLE_FILE = os.path.join(os.path.dirname(__file__), 'test_file.csv')
+FORBIDDEN_FILE = os.path.join(os.path.dirname(__file__), 'forbidden_file.csv')
+FORBIDDEN_FILE_2 = os.path.join(
+    os.path.dirname(__file__), 'forbidden_file_2.csv')
 
 class CompanySetup(FunctionalTestCase):
@@ -133 +137 @@
             self.datacenter_path + "/@@show/?logname=myspecial.log&query=.*")
         assert "SOME <TAG> AND ÜMLÄUTS IN LOG" in self.browser.contents
+
+    def test_file_download_delete(self):
+        self.browser.addHeader('Authorization', 'Basic mgr:mgrpw')
+        self.browser.open(self.datacenter_path)
+        self.assertEqual(self.browser.headers['Status'], '200 Ok')
+        self.assertEqual(self.browser.url, self.datacenter_path)
+        self.browser.getLink("Upload data").click()
+        file = open(SAMPLE_FILE)
+        ctrl = self.browser.getControl(name='uploadfile:file')
+        file_ctrl = ctrl.mech_control
+        file_ctrl.add_file(file, filename='my_test_data.csv')
+        self.browser.getControl('Upload').click()
+        self.browser.open(self.datacenter_path)
+        self.browser.getLink('my_test_data_zope.mgr.csv').click()
+        self.assertEqual(self.browser.headers['Status'], '200 Ok')
+        self.assertEqual(self.browser.headers['Content-Type'],
+                         'text/csv; charset=UTF-8')
+        logfile = os.path.join(
+            self.app['datacenter'].storage, 'logs', 'datacenter.log')
+        logcontent = open(logfile).read()
+        self.assertTrue('zope.mgr - browser.pages.FileDownloadView - '
+                        'downloaded: my_test_data_zope.mgr.csv' in logcontent)
+        self.browser.open(self.datacenter_path)
+        ctrl = self.browser.getControl(name='val_id')
+        ctrl.getControl(value='my_test_data_zope.mgr.csv').selected = True
+        self.browser.getControl("Remove selected", index=0).click()
+        self.assertTrue('Successfully deleted: my_test_data_zope.mgr.csv'
+            in self.browser.contents)
+        logcontent = open(logfile).read()
+        self.assertTrue('zope.mgr - browser.pages.DatacenterPage - '
+                        'deleted: my_test_data_zope.mgr.csv' in logcontent)
+        return
+
+    def test_forbidden_file_upload(self):
+        self.browser.addHeader('Authorization', 'Basic mgr:mgrpw')
+        self.browser.open(self.datacenter_path)
+        self.assertEqual(self.browser.headers['Status'], '200 Ok')
+        self.assertEqual(self.browser.url, self.datacenter_path)
+        self.browser.getLink("Upload data").click()
+        file = open(FORBIDDEN_FILE)
+        ctrl = self.browser.getControl(name='uploadfile:file')
+        file_ctrl = ctrl.mech_control
+        file_ctrl.add_file(file, filename='my_corrupted_file.csv')
+        self.browser.getControl('Upload').click()
+        self.assertTrue(
+            'Your file contains forbidden characters or'
+            in self.browser.contents)
+        logfile = os.path.join(
+            self.app['datacenter'].storage, 'logs', 'datacenter.log')
+        logcontent = open(logfile).read()
+        self.assertTrue('zope.mgr - browser.pages.DatacenterUploadPage - '
+            'invalid file uploaded:' in logcontent)
+        return
+
+    def test_forbidden_file_upload_2(self):
+        self.browser.addHeader('Authorization', 'Basic mgr:mgrpw')
+        self.browser.open(self.datacenter_path)
+        self.assertEqual(self.browser.headers['Status'], '200 Ok')
+        self.assertEqual(self.browser.url, self.datacenter_path)
+        self.browser.getLink("Upload data").click()
+        file = open(FORBIDDEN_FILE_2)
+        ctrl = self.browser.getControl(name='uploadfile:file')
+        file_ctrl = ctrl.mech_control
+        file_ctrl.add_file(file, filename='my_corrupted_file.csv')
+        self.browser.getControl('Upload').click()
+        self.assertTrue(
+            'The data in your file contain trailing whitespaces.'
+            in self.browser.contents)
+        logfile = os.path.join(
+            self.app['datacenter'].storage, 'logs', 'datacenter.log')
+        logcontent = open(logfile).read()
+        self.assertTrue('zope.mgr - browser.pages.DatacenterUploadPage - '
+            'invalid file uploaded:' in logcontent)
+        return
 
     def test_user_data_import_permission(self):

main/waeup.ikoba/trunk/src/waeup/ikoba/utils/helpers.py

@@ -756 +756 @@
 
 def check_csv_charset(iterable):
-    """Check contents of `iterable` regarding valid CSV encoding.
+    """Check contents of `iterable` regarding valid CSV encoding and
+    trailing whitespaces in data.
 
     `iterable` is expected to be an iterable on _rows_ (not
@@ -764 +765 @@
 
     Returns line num of first illegal char or ``None``. Line nums
-    start counting with 1 (not zero).
+    start counting with 1 (not zero). Returns -1 if data contain
+    trailing whitespaces.
     """
     linenum = 1
-    reader = csv.DictReader(iterable)
     try:
+        reader = csv.DictReader(iterable)
         for row in reader:
             linenum += 1
+            for value in row.values():
+                if value.endswith(' '):
+                    return -1
     except UnicodeDecodeError:
         return linenum

main/waeup.ikoba/trunk/src/waeup/ikoba/utils/tests/test_helpers.py

@@ -533 +533 @@
             'some text that \n'
             '\n'      # this empty line will break
-            'is not a csv file \n' + chr(0x92) + '\n'
+            'is not a csv file\n' + chr(0x92) + '\n'
             ).splitlines()
         self.assertEqual(helpers.check_csv_charset(csv), 2)
+
+    def test_invalid_data3(self):
+        csv = (
+            "code,title,title_prefix\n"
+            "FAC1,Faculty 1,faculty\n"
+            "FAC2,Faculty 2 ,institute\n"
+            "FAC3,Fäcülty 3,school\n"
+            ).splitlines()
+        self.assertEqual(helpers.check_csv_charset(csv), -1)