source: main/waeup.sirp/branches/ulif-stress-multimech/multi-mechanize/lib/results.py @ 7543

Last change on this file since 7543 was 7478, checked in by uli, 13 years ago

Sample usage of multi-mechanize (not finished).

File size: 15.0 KB
Line 
1#!/usr/bin/env python
2#
3#  Copyright (c) 2010-2011 Corey Goldberg (corey@goldb.org)
4#  License: GNU LGPLv3
5
6#  This file is part of Multi-Mechanize
7
8
9import time
10from collections import defaultdict
11import graph
12import reportwriter
13import reportwriterxml
14
15
16
17def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=None, xml_reports=False):
18    results = Results(results_dir + results_file, run_time)
19   
20    report = reportwriter.Report(results_dir)
21   
22    print 'transactions: %i' % results.total_transactions
23    print 'errors: %i' % results.total_errors
24    print ''
25    print 'test start: %s' % results.start_datetime
26    print 'test finish: %s' % results.finish_datetime
27    print ''
28   
29    # write the results in XML
30    if xml_reports:
31        reportwriterxml.write_jmeter_output(results.resp_stats_list, results_dir)
32
33    report.write_line('<h1>Performance Results Report</h1>')
34   
35    report.write_line('<h2>Summary</h2>')
36   
37    report.write_line('<div class="summary">')
38    report.write_line('<b>transactions:</b> %d<br />' % results.total_transactions)
39    report.write_line('<b>errors:</b> %d<br />' % results.total_errors)
40    report.write_line('<b>run time:</b> %d secs<br />' % run_time)
41    report.write_line('<b>rampup:</b> %d secs<br /><br />' % rampup)
42    report.write_line('<b>test start:</b> %s<br />' % results.start_datetime)
43    report.write_line('<b>test finish:</b> %s<br /><br />' % results.finish_datetime)
44    report.write_line('<b>time-series interval:</b> %s secs<br /><br /><br />' % ts_interval)
45    if user_group_configs:
46        report.write_line('<b>workload configuration:</b><br /><br />')
47        report.write_line('<table>')
48        report.write_line('<tr><th>group name</th><th>threads</th><th>script name</th></tr>')
49        for user_group_config in user_group_configs:
50            report.write_line('<tr><td>%s</td><td>%d</td><td>%s</td></tr>' %
51                (user_group_config.name, user_group_config.num_threads, user_group_config.script_file))
52        report.write_line('</table>')
53    report.write_line('</div>')
54   
55    report.write_line('<h2>All Transactions</h2>')
56   
57    # all transactions - response times
58    trans_timer_points = []  # [elapsed, timervalue]
59    trans_timer_vals = []
60    for resp_stats in results.resp_stats_list:
61        t = (resp_stats.elapsed_time, resp_stats.trans_time)
62        trans_timer_points.append(t)
63        trans_timer_vals.append(resp_stats.trans_time)
64    graph.resp_graph_raw(trans_timer_points, 'All_Transactions_response_times.png', results_dir)
65   
66    report.write_line('<h3>Transaction Response Summary (secs)</h3>')
67    report.write_line('<table>')
68    report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
69    report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'  % (
70        results.total_transactions,
71        min(trans_timer_vals),
72        average(trans_timer_vals),
73        percentile(trans_timer_vals, 80),
74        percentile(trans_timer_vals, 90),
75        percentile(trans_timer_vals, 95),
76        max(trans_timer_vals),
77        standard_dev(trans_timer_vals),
78    ))
79    report.write_line('</table>')
80   
81   
82    # all transactions - interval details
83    avg_resptime_points = {}  # {intervalnumber: avg_resptime}
84    percentile_80_resptime_points = {}  # {intervalnumber: 80pct_resptime}
85    percentile_90_resptime_points = {}  # {intervalnumber: 90pct_resptime}
86    interval_secs = ts_interval
87    splat_series = split_series(trans_timer_points, interval_secs)
88    report.write_line('<h3>Interval Details (secs)</h3>')
89    report.write_line('<table>')
90    report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
91    for i, bucket in enumerate(splat_series):
92        interval_start = int((i + 1) * interval_secs)
93        cnt = len(bucket)
94       
95        if cnt == 0:
96            report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1)) 
97        else:
98            rate = cnt / float(interval_secs)
99            mn = min(bucket)
100            avg = average(bucket)
101            pct_80 = percentile(bucket, 80)
102            pct_90 = percentile(bucket, 90)
103            pct_95 = percentile(bucket, 95)
104            mx = max(bucket)
105            stdev = standard_dev(bucket)
106            report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
107
108            avg_resptime_points[interval_start] = avg
109            percentile_80_resptime_points[interval_start] = pct_80
110            percentile_90_resptime_points[interval_start] = pct_90
111
112    report.write_line('</table>')
113    graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, 'All_Transactions_response_times_intervals.png', results_dir)
114   
115   
116    report.write_line('<h3>Graphs</h3>')
117    report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
118    report.write_line('<img src="All_Transactions_response_times_intervals.png"></img>')     
119    report.write_line('<h4>Response Time: raw data (all points)</h4>')
120    report.write_line('<img src="All_Transactions_response_times.png"></img>')
121    report.write_line('<h4>Throughput: 5 sec time-series</h4>')
122    report.write_line('<img src="All_Transactions_throughput.png"></img>') 
123   
124
125
126    # all transactions - throughput
127    throughput_points = {}  # {intervalnumber: numberofrequests}
128    interval_secs = ts_interval
129    splat_series = split_series(trans_timer_points, interval_secs)
130    for i, bucket in enumerate(splat_series):
131        throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
132    graph.tp_graph(throughput_points, 'All_Transactions_throughput.png', results_dir)
133   
134       
135       
136    # custom timers
137    for timer_name in sorted(results.uniq_timer_names):
138        custom_timer_vals = []
139        custom_timer_points = []
140        for resp_stats in results.resp_stats_list:
141            try:
142                val = resp_stats.custom_timers[timer_name]
143                custom_timer_points.append((resp_stats.elapsed_time, val))
144                custom_timer_vals.append(val)
145            except KeyError:
146                pass
147        graph.resp_graph_raw(custom_timer_points, timer_name + '_response_times.png', results_dir)
148       
149        throughput_points = {}  # {intervalnumber: numberofrequests}
150        interval_secs = ts_interval
151        splat_series = split_series(custom_timer_points, interval_secs)
152        for i, bucket in enumerate(splat_series):
153            throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
154        graph.tp_graph(throughput_points, timer_name + '_throughput.png', results_dir)
155       
156        report.write_line('<hr />')
157        report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)
158       
159        report.write_line('<h3>Timer Summary (secs)</h3>')
160       
161        report.write_line('<table>')
162        report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
163        report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'  % (
164            len(custom_timer_vals),
165            min(custom_timer_vals),
166            average(custom_timer_vals),
167            percentile(custom_timer_vals, 80),
168            percentile(custom_timer_vals, 90),
169            percentile(custom_timer_vals, 95),
170            max(custom_timer_vals),
171            standard_dev(custom_timer_vals)
172        ))
173        report.write_line('</table>')
174   
175       
176        # custom timers - interval details
177        avg_resptime_points = {}  # {intervalnumber: avg_resptime}
178        percentile_80_resptime_points = {}  # {intervalnumber: 80pct_resptime}
179        percentile_90_resptime_points = {}  # {intervalnumber: 90pct_resptime}
180        interval_secs = ts_interval
181        splat_series = split_series(custom_timer_points, interval_secs)
182        report.write_line('<h3>Interval Details (secs)</h3>')
183        report.write_line('<table>')
184        report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
185        for i, bucket in enumerate(splat_series):
186            interval_start = int((i + 1) * interval_secs)
187            cnt = len(bucket)
188           
189            if cnt == 0:
190                report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1)) 
191            else:
192                rate = cnt / float(interval_secs)
193                mn = min(bucket)
194                avg = average(bucket)
195                pct_80 = percentile(bucket, 80)
196                pct_90 = percentile(bucket, 90)
197                pct_95 = percentile(bucket, 95)
198                mx = max(bucket)
199                stdev = standard_dev(bucket)
200
201                report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
202                 
203                avg_resptime_points[interval_start] = avg
204                percentile_80_resptime_points[interval_start] = pct_80
205                percentile_90_resptime_points[interval_start] = pct_90
206        report.write_line('</table>')
207        graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, timer_name + '_response_times_intervals.png', results_dir)
208   
209       
210        report.write_line('<h3>Graphs</h3>')
211        report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
212        report.write_line('<img src="%s_response_times_intervals.png"></img>' % timer_name)
213        report.write_line('<h4>Response Time: raw data (all points)</h4>')       
214        report.write_line('<img src="%s_response_times.png"></img>' % timer_name)
215        report.write_line('<h4>Throughput: %s sec time-series</h4>' % ts_interval)
216        report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
217       
218   
219       
220    ## user group times
221    #for user_group_name in sorted(results.uniq_user_group_names):
222    #    ug_timer_vals = []
223    #    for resp_stats in results.resp_stats_list:
224    #        if resp_stats.user_group_name == user_group_name:
225    #            ug_timer_vals.append(resp_stats.trans_time)
226    #    print user_group_name
227    #    print 'min: %.3f' % min(ug_timer_vals)
228    #    print 'avg: %.3f' % average(ug_timer_vals)
229    #    print '80pct: %.3f' % percentile(ug_timer_vals, 80)
230    #    print '90pct: %.3f' % percentile(ug_timer_vals, 90)
231    #    print '95pct: %.3f' % percentile(ug_timer_vals, 95)
232    #    print 'max: %.3f' % max(ug_timer_vals)
233    #    print ''       
234   
235    report.write_line('<hr />')
236    report.write_closing_html()
237
238
239   
240   
241class Results(object):
242    def __init__(self, results_file_name, run_time):
243        self.results_file_name = results_file_name
244        self.run_time = run_time
245        self.total_transactions = 0
246        self.total_errors = 0
247        self.uniq_timer_names = set()
248        self.uniq_user_group_names = set()
249       
250        self.resp_stats_list = self.__parse_file()
251       
252        self.epoch_start = self.resp_stats_list[0].epoch_secs
253        self.epoch_finish = self.resp_stats_list[-1].epoch_secs
254        self.start_datetime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.epoch_start))
255        self.finish_datetime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.epoch_finish))
256       
257       
258       
259    def __parse_file(self):
260        f = open(self.results_file_name, 'rb')
261        resp_stats_list = []
262        for line in f:
263            fields = line.strip().split(',')
264           
265            request_num = int(fields[0])
266            elapsed_time = float(fields[1])
267            epoch_secs = int(fields[2])
268            user_group_name = fields[3]
269            trans_time = float(fields[4])
270            error = fields[5]
271           
272            self.uniq_user_group_names.add(user_group_name)
273           
274            custom_timers = {}
275            timers_string = ''.join(fields[6:]).replace('{', '').replace('}', '')
276            splat = timers_string.split("'")[1:]
277            timers = []
278            vals = []
279            for x in splat:
280                if ':' in x:
281                    x = float(x.replace(': ', ''))
282                    vals.append(x)
283                else:
284                    timers.append(x)
285                    self.uniq_timer_names.add(x)
286            for timer, val in zip(timers, vals):
287                custom_timers[timer] = val
288           
289            r = ResponseStats(request_num, elapsed_time, epoch_secs, user_group_name, trans_time, error, custom_timers)
290           
291            if elapsed_time < self.run_time:  # drop all times that appear after the last request was sent (incomplete interval)
292                resp_stats_list.append(r)
293           
294            if error != '':
295                self.total_errors += 1
296               
297            self.total_transactions += 1
298           
299        return resp_stats_list   
300   
301
302
303class ResponseStats(object):
304    def __init__(self, request_num, elapsed_time, epoch_secs, user_group_name, trans_time, error, custom_timers):
305        self.request_num = request_num
306        self.elapsed_time = elapsed_time
307        self.epoch_secs = epoch_secs
308        self.user_group_name = user_group_name
309        self.trans_time = trans_time
310        self.error = error
311        self.custom_timers = custom_timers
312       
313
314
315def split_series(points, interval):
316    offset = points[0][0]
317    maxval = int((points[-1][0] - offset) // interval)
318    vals = defaultdict(list)
319    for key, value in points:
320        vals[(key - offset) // interval].append(value)
321    series = [vals[i] for i in xrange(maxval + 1)]
322    return series
323
324
325
326def average(seq):
327    avg = (float(sum(seq)) / len(seq))
328    return avg
329
330
331
332def standard_dev(seq):
333    avg = average(seq)
334    sdsq = sum([(i - avg) ** 2 for i in seq])
335    try:
336        stdev = (sdsq / (len(seq) - 1)) ** .5
337    except ZeroDivisionError:
338        stdev = 0
339    return stdev
340
341   
342   
343def percentile(seq, percentile):
344    i = int(len(seq) * (percentile / 100.0))
345    seq.sort()
346    return seq[i]
347
348
349
350
351if __name__ == '__main__':
352    output_results('./', 'results.csv', 60, 30, 10)
Note: See TracBrowser for help on using the repository browser.