Changeset 8512 for main/waeup.kofa/trunk/src

Timestamp:

25 May 2012, 00:30:22 (13 years ago)

Author:

uli

Message:

Add support for datacenter to search log files.

Location:

main/waeup.kofa/trunk/src/waeup/kofa

Files:

• datacenter.py (modified) (3 diffs)
• tests/test_datacenter.py (added)

main/waeup.kofa/trunk/src/waeup/kofa/datacenter.py

@@ -22 +22 @@
 """
 import os
+import re
 import shutil
 import grok
@@ -32 +33 @@
 from waeup.kofa.utils.helpers import copy_filesystem_tree
 from waeup.kofa.utils.logger import Logger
+
+#: Regular expression describing a logfile name with backup extension
+RE_LOGFILE_NAME = re.compile('^.+\.\d+$')
 
 class DataCenter(grok.Container, Logger):
@@ -186 +190 @@
         return
 
+    def _logfiles(self, basename):
+        """Get sorted logfiles starting with `basename`.
+        """
+        def numerical_suffix(name):
+            # return numerical suffix in `name` as number or 0.
+            suffix = name.rsplit('.', 1)[-1]
+            try:
+                return int(suffix)
+            except ValueError:
+                return 0
+            pass
+        files = [basename,]
+        for name in os.listdir(os.path.join(self.storage, 'logs')):
+            if RE_LOGFILE_NAME.match(name):
+                files.append(name)
+        return sorted(files, key=numerical_suffix)
+
+    def queryLogfiles(self, basename, query=None, limit=0, start=0):
+        """Search `query` in all logfiles starting with `basename`.
+
+        Returns an iterator of those lines in logfiles starting with
+        `basename` that match `query`. If you want the result as a
+        list, simply list() the iterator.
+
+        All logfiles with name `basename` and maybe some numerical
+        extension ('.1', '.2', ...) are searched for the `query` term
+        in correct order. So, if you ask for a basename 'app.log',
+        then any file named 'app.log', 'app.log.1', 'app.log.2',
+        etc. will be searched in that order.
+
+        The `query` is expected to be a string containing a regular
+        expression.
+
+        If `limit` is set to some numerical value, at most this number
+        of lines is returned.
+
+        With `start` you can give the number of first matched line to
+        return. `start` is zero-based, i.e. the first match has number
+        0, the scond one 1, etc.
+
+        Together with `limit` this allows some basic
+        batching. Please keep in mind that batching might give
+        unpredictable results, when logfiles change between two
+        requests. This is not a problem when only one file is searched
+        and changes include only appending new log messages.
+
+        Matches are found per line only (no multiline matches).
+
+        This method raises ValueError if some basic condition is not
+        met, for instance if the given query string is not a valid
+        regular expression.
+
+        Please note, that this exception will happen not before you
+        really fetch a result line.
+        """
+        try:
+            re_query = re.compile(query)
+        except:
+            raise ValueError('Invalid query string: %s' % query)
+
+        basename = basename.replace('/../', '')
+        files = self._logfiles(basename)
+
+        # Search the log files
+        num = 0
+        for name in files:
+            path = os.path.join(self.storage, 'logs', name)
+            if not os.path.isfile(path):
+                continue
+            for line in open(path, 'rb'):
+                if not re_query.match(line):
+                    continue
+                num += 1
+                if (num - 1) < start:
+                    continue
+                yield line
+
+                if limit and (num - limit >= start):
+                    raise StopIteration
+        pass
 
 class DataCenterFile(object):