Changeset 10045 for main/waeup.kofa/trunk

Timestamp:

23 Mar 2013, 10:48:31 (12 years ago)

Author:

uli

Message:

Make logfile parsing unicode-aware.

Location:

main/waeup.kofa/trunk/src/waeup/kofa

Files:

• datacenter.py (modified) (4 diffs)
• tests/test_datacenter.py (modified) (10 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/datacenter.py

@@ -21 +21 @@
 tools for importing/exporting CSV data.
 """
+import codecs
+import fnmatch
+import grok
 import os
-import fnmatch
 import re
 import shutil
-import grok
 from datetime import datetime
 from zope.component import getUtility
@@ -291 +292 @@
         Matches are found per line only (no multiline matches).
 
+        Result lines are returned as unicode instances decoded from
+        UTF-8 encoding. This means that logfiles must provide UTF-8
+        encoding for umlauts etc. if these should be rendered
+        properly. The returned unicode lines can be fed to page
+        templates even if they contain non-ASCII characters.
+
         This method raises ValueError if some basic condition is not
         met, for instance if the given query string is not a valid
@@ -312 +319 @@
             if not os.path.isfile(path):
                 continue
-            for line in open(path, 'rb'):
+            for line in codecs.open(path, 'rb', 'utf-8'):
                 if not re_query.search(line):
                     continue
@@ -386 +393 @@
         if self._parsed:
             return
-        for line in open(self.context, 'rb'):
+        for line in codecs.open(self.context, 'rb', 'utf-8'):
             line = line.strip()
             if not ':' in line:

main/waeup.kofa/trunk/src/waeup/kofa/tests/test_datacenter.py

@@ +1 @@
+# -*- coding: utf-8 -*-
 ## Tests for datacenter
 ##
@@ -42 +43 @@
                 fp.write('Msg %d\n' % (n + 1))
                 fp.write('Other Line %d\n' % (n + 1))
+            fp.write('A Message with Ümläüt')
             fp.close()
             path = self.logpath + '.%d' % m
@@ -57 +59 @@
         open(self.logpath, 'wb').write('Message 1\n')
         result = list(datacenter.queryLogfiles('myapp.log', 'Message'))
-        self.assertEqual(result, ['Message 1\n'])
+        self.assertEqual(result, [u'Message 1\n'])
         return
 
@@ -68 +70 @@
         result = list(datacenter.queryLogfiles('myapp.log', 'Msg'))
         # entry of logfile .10 comes after entry of logfile .2
-        self.assertEqual(result, ['Msg 1\n', 'Msg 2\n', 'Msg 3\n'])
+        self.assertEqual(result, [u'Msg 1\n', u'Msg 2\n', u'Msg 3\n'])
         return
 
@@ -79 +81 @@
         result = list(datacenter.queryLogfiles('myapp.log', 'Msg'))
         # Msg 2 and 3 won't show up in results.
-        self.assertEqual(result, ['Msg 1\n'])
+        self.assertEqual(result, [u'Msg 1\n'])
         return
 
@@ -116 +118 @@
             'myapp.log', 'Msg', start=25))
         self.assertEqual(len(result), 25)
-        self.assertEqual(result[0], 'Msg 26\n')
+        self.assertEqual(result[0], u'Msg 26\n')
         return
 
@@ -126 +128 @@
             'myapp.log', 'Msg', start=25, limit=10))
         self.assertEqual(len(result), 10)
-        self.assertEqual(result[0], 'Msg 26\n')
-        self.assertEqual(result[-1], 'Msg 35\n')
+        self.assertEqual(result[0], u'Msg 26\n')
+        self.assertEqual(result[-1], u'Msg 35\n')
         return
 
@@ -137 +139 @@
             'myapp.log', 'Msg', start=45, limit=10))
         self.assertEqual(len(result), 5)
-        self.assertEqual(result[0], 'Msg 46\n')
-        self.assertEqual(result[-1], 'Msg 50\n')
+        self.assertEqual(result[0], u'Msg 46\n')
+        self.assertEqual(result[-1], u'Msg 50\n')
         return
 
@@ -148 +150 @@
             'myapp.log', 'Msg', start=45, limit=10))
         self.assertEqual(len(result), 10)
-        self.assertEqual(result[0], 'Msg 46\n')
-        self.assertEqual(result[-1], 'Msg 55\n')
+        self.assertEqual(result[0], u'Msg 46\n')
+        self.assertEqual(result[-1], u'Msg 55\n')
         return
 
@@ -158 +160 @@
         result = list(datacenter.queryLogfiles('myapp.log', 'sg 1\n'))
         self.assertEqual(len(result), 1)
-        self.assertEqual(result[0], 'Msg 1\n')
+        self.assertEqual(result[0], u'Msg 1\n')
+        return
+
+    def test_query_logfiles_umlauts(self):
+        # we return results as unicode decoded from utf-8
+        datacenter = DataCenter()
+        self.fill_logfile()
+        result = list(datacenter.queryLogfiles(
+            'myapp.log', u'Ümläüt'))
+        self.assertTrue(isinstance(result[0], unicode))
+        self.assertEqual(result, [u'A Message with Ümläüt'])
         return