source: WAeUP_SysConf/logging/trunk/usr/local/lib/zevents.py @ 3597

##
## zevents.py
## Read Zope event log from stdin and output SQL data.
##
## Login : <uli@pu.smp.net>
## Started on  Wed Feb  7 03:02:47 2007 Uli Fouquet
## $Id$
## 
## Copyright (C) 2007 Uli Fouquet
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
## 
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
## 
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##

##
## Usage: python zevents.py [-INSERT]
##
## Reads Zope event logs from stdin and turn it into PostGres compatible
## output on stdout.
##
## If no options are given, output is formatted as a COPY command, which is
## *much* (50 times or more) faster in PostGres.
##
## If -INSERT is given, output is formatted as a list of INSERT statements,
## which will need much more time and resources.
##

import sys
import re

##
## Set mode to...
##   "COPY":  give PostGres-compatible COPY-syntax output.
##   other : give usual (slow) SQL-output
##
mode = "COPY"

if '-INSERT' in sys.argv:
    mode = "INSERT"

faillog = "/var/log/updatedblog/zevents-failed.log"

##
## The format of a zope event.log line is like this...
## The expression looks for the following fields:
##
## Fieldcontent        | Fieldnum
## --------------------+---------
## DATE (dd-mm-yyyy)   | 0
## TIME                | 1
## LOGLEVEL            | 2
## MSG_SRC             | 3
## MSG_TEXT            | 4
## ERR_TYPE            | --
## ERR_MSG             | --
## TRACEBACK           | 5
##
## It was hard work. Please do not touch!
##
expr = re.compile("^([0-9]+\-[0-9]+\-[0-9]+)T([0-9]+\:[0-9]+\:[0-9]+) ([^ ]+) ([^ ]+) ?(.*) ?(Traceback \(innermost last\)\: .*)?")


## Number of line
lnum = 1

if mode == "COPY":
    print 'COPY "ZEVENTS" FROM stdin;'

def process_dataset(data):
    data = data.strip()
    data = data.replace("\n", " ")
    data = data.replace("\t", "    ")
    return data

def q(text):
    return text.replace("'","\\\'")

def print_dataset(data, lnum, mode):
    m = expr.match(data)
    if not m:
        # Line did not match...
        #print data
        fd = open( faillog, "a")
        fd.write( str( data ) + '\n' )
        fd.close()
        return

    res = m.groups()
    val = {
        'id' : lnum,
        'date' : res[0],
        'time' : res[1],
        'loglevel' : res[2],
        'msg_src' : res[3] or '',
        'msg_text' : res[4] or '',
        'err_type' : '',
        'err_msg' : '',
        'traceback' : res[5] or '',
        }

    # If msg_text is too long, append it to traceback...
    if len(val['msg_text']) > 1024:
        val['traceback'] = val['msg_text'] + " " + val['traceback']
        val['msg_text'] = val['msg_text'][:1024]

    # Shorten overlong fields...
    for name,maxlen in [('loglevel',32), ('msg_src',1024), ('msg_text', 1024),
                 ('err_type', 32), ('err_msg', 1024)]:
        val[name]=val[name][:maxlen]

    if mode == "COPY":
        print "%07d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (
            val['id'], val['date'], val['time'], val['msg_src'],
            val['msg_text'], val['err_type'], val['err_msg'], val['traceback'],
            val['loglevel']
        )
    else:
        print "INSERT INTO \"ZEVENTS\" (\"ID\",\"DATESTR\",\"TIMESTR\",\"MSG_SRC\",\"MSG\",\"ERR_TYPE\",\"ERR_MSG\",\"TRACEBACK\",\"LOGLEVEL\") VALUES ('%07d','%s','%s','%s','%s','%s','%s','%s','%s');" % (
            val['id'], q(val['date']), q(val['time']), q(val['msg_src']),
            q(val['msg_text']), q(val['err_type']), q(val['err_msg']),
            q(val['traceback']), q(val['loglevel'])
        )
    return

## Read line for line...
data=sys.stdin.readline().strip()
dataset = ''
while data:

    if data.strip() == '------':
        lnum += 1
        dataset = process_dataset(dataset)
        print_dataset(dataset, lnum, mode)
        dataset = ""
    else:
        dataset += data

    data=sys.stdin.readline()
    continue

dataset = process_dataset(dataset)
print_dataset(dataset, lnum, mode)