Ignore:
Timestamp:
27 Jan 2012, 12:51:05 (13 years ago)
Author:
uli
Message:

Clean up code.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • main/waeup.sirp/branches/ulif-stress-multimech/multi-mechanize/multi-mechanize.py

    r7478 r7519  
    22#
    33#  Copyright (c) 2010-2011 Corey Goldberg (corey@goldb.org)
    4 #  License: GNU LGPLv3 - distributed under the terms of the GNU Lesser General Public License version 3
    5 # 
     4#  License: GNU LGPLv3 - distributed under the terms of the GNU Lesser
     5#  General Public License version 3
     6#
    67#  This file is part of Multi-Mechanize:
    7 #       Multi-Process, Multi-Threaded, Web Load Generator, with python-mechanize agents
     8#       Multi-Process, Multi-Threaded, Web Load Generator, with
     9#       python-mechanize agents
    810#
    911#  requires Python 2.6+
    10 
    11 
    12 
    1312import ConfigParser
    1413import glob
    … …  
    2322import time
    2423import lib.results as results
    25 import lib.progressbar as progressbar       
     24import lib.progressbar as progressbar
    2625
    2726usage = 'Usage: %prog <project name> [options]'
    2827parser = optparse.OptionParser(usage=usage)
    29 parser.add_option('-p', '--port', dest='port', type='int', help='rpc listener port')
    30 parser.add_option('-r', '--results', dest='results_dir', help='results directory to reprocess')
     28parser.add_option(
     29    '-p', '--port', dest='port', type='int', help='rpc listener port')
     30parser.add_option(
     31    '-r', '--results', dest='results_dir',
     32    help='results directory to reprocess')
    3133cmd_opts, args = parser.parse_args()
    3234
    … …  
    3739    sys.stderr.write('usage: python multi-mechanize.py <project_name>\n')
    3840    sys.stderr.write('example: python multi-mechanize.py default_project\n\n')
    39     sys.exit(1) 
     41    sys.exit(1)
    4042
    4143scripts_path = 'projects/%s/test_scripts' % project_name
    4244if not os.path.exists(scripts_path):
    4345    sys.stderr.write('\nERROR: can not find project: %s\n\n' % project_name)
    44     sys.exit(1)
    45 sys.path.append(scripts_path)         
    46 for f in glob.glob( '%s/*.py' % scripts_path):  # import all test scripts as modules
     46    sys.exit(1)
     47sys.path.append(scripts_path)
     48for f in glob.glob( '%s/*.py' % scripts_path):
     49    # import all test scripts as modules
    4750    f = f.replace(scripts_path, '').replace(os.sep, '').replace('.py', '')
    4851    exec('import %s' % f)
    … …  
    5659        import lib.rpcserver
    5760        lib.rpcserver.launch_rpc_server(cmd_opts.port, project_name, run_test)
    58     else: 
     61    else:
    5962        run_test()
    6063    return
    61        
    62        
    63    
     64
     65
    6466def run_test(remote_starter=None):
    6567    if remote_starter is not None:
    6668        remote_starter.test_running = True
    6769        remote_starter.output_dir = None
    68        
    69     run_time, rampup, results_ts_interval, console_logging, progress_bar, results_database, post_run_script, xml_report, user_group_configs = configure(project_name)
    70    
    71     run_localtime = time.localtime()
    72     output_dir = time.strftime('projects/' + project_name + '/results/results_%Y.%m.%d_%H.%M.%S/', run_localtime)
    73        
     70    (run_time, rampup, results_ts_interval, console_logging, progress_bar,
     71     results_database, post_run_script, xml_report,
     72     user_group_configs) = configure(project_name)
     73
     74    run_localtime = time.localtime()
     75    output_dir = time.strftime(
     76        'projects/' + project_name
     77        + '/results/results_%Y.%m.%d_%H.%M.%S/', run_localtime)
     78
    7479    # this queue is shared between all processes/threads
    7580    queue = multiprocessing.Queue()
    … …  
    7782    rw.daemon = True
    7883    rw.start()
    79    
    80     user_groups = [] 
     84
     85    user_groups = []
    8186    for i, ug_config in enumerate(user_group_configs):
    82         ug = UserGroup(queue, i, ug_config.name, ug_config.num_threads, ug_config.script_file, run_time, rampup)
    83         user_groups.append(ug)   
     87        ug = UserGroup(queue, i, ug_config.name, ug_config.num_threads,
     88                       ug_config.script_file, run_time, rampup)
     89        user_groups.append(ug)
    8490    for user_group in user_groups:
    8591        user_group.start()
    86        
    87     start_time = time.time() 
    88    
     92
     93    start_time = time.time()
     94
    8995    if console_logging:
    9096        for user_group in user_groups:
    … …  
    9399        print '\n  user_groups:  %i' % len(user_groups)
    94100        print '  threads: %i\n' % (ug_config.num_threads * len(user_groups))
    95        
     101
    96102        if progress_bar:
    97103            p = progressbar.ProgressBar(run_time)
    … …  
    100106                p.update_time(elapsed)
    101107                if sys.platform.startswith('win'):
    102                     print '%s   transactions: %i  timers: %i  errors: %i\r' % (p, rw.trans_count, rw.timer_count, rw.error_count),
     108                    print '%s   transactions: %i  timers: %i  errors: %i\r' % (
     109                        p, rw.trans_count, rw.timer_count, rw.error_count),
    103110                else:
    104                     print '%s   transactions: %i  timers: %i  errors: %i' % (p, rw.trans_count, rw.timer_count, rw.error_count)
     111                    print '%s   transactions: %i  timers: %i  errors: %i' % (
     112                        p, rw.trans_count, rw.timer_count, rw.error_count)
    105113                    sys.stdout.write(chr(27) + '[A' )
    106114                time.sleep(1)
    107115                elapsed = time.time() - start_time
    108            
    109116            print p
    110            
    111         while [user_group for user_group in user_groups if user_group.is_alive()] != []:
     117
     118        while [user_group for user_group in user_groups
     119               if user_group.is_alive()] != []:
    112120            if progress_bar:
    113121                if sys.platform.startswith('win'):
    … …  
    117125                    sys.stdout.write(chr(27) + '[A' )
    118126            time.sleep(.5)
    119            
     127
    120128        if not sys.platform.startswith('win'):
    121129            print
    … …  
    124132    time.sleep(.2) # make sure the writer queue is flushed
    125133    print '\n\nanalyzing results...\n'
    126     results.output_results(output_dir, 'results.csv', run_time, rampup, results_ts_interval, user_group_configs, xml_report)
     134    results.output_results(
     135        output_dir, 'results.csv', run_time, rampup, results_ts_interval,
     136        user_group_configs, xml_report)
    127137    print 'created: %sresults.html\n' % output_dir
    128138    if xml_report:
    129139        print 'created: %sresults.jtl' % output_dir
    130140        print 'created: last_results.jtl\n'
    131    
     141
    132142    # copy config file to results directory
    133143    project_config = os.sep.join(['projects', project_name, 'config.cfg'])
    134144    saved_config = os.sep.join([output_dir, 'config.cfg'])
    135145    shutil.copy(project_config, saved_config)
    136    
     146
    137147    if results_database is not None:
    138148        print 'loading results into database: %s\n' % results_database
    139149        import lib.resultsloader
    140         lib.resultsloader.load_results_database(project_name, run_localtime, output_dir, results_database,
    141                 run_time, rampup, results_ts_interval, user_group_configs)
    142    
     150        lib.resultsloader.load_results_database(
     151            project_name, run_localtime, output_dir, results_database,
     152            run_time, rampup, results_ts_interval, user_group_configs)
     153
    143154    if post_run_script is not None:
    144155        print 'running post_run_script: %s\n' % post_run_script
    145156        subprocess.call(post_run_script)
    146        
     157
    147158    print 'done.\n'
    148    
     159
    149160    if remote_starter is not None:
    150161        remote_starter.test_running = False
    151162        remote_starter.output_dir = output_dir
    152    
    153163    return
    154    
    155    
    156    
     164
     165
     166
    157167def rerun_results(results_dir):
    158168    output_dir = 'projects/%s/results/%s/' % (project_name, results_dir)
    159169    saved_config = '%s/config.cfg' % output_dir
    160     run_time, rampup, results_ts_interval, console_logging, progress_bar, results_database, post_run_script, xml_report, user_group_configs = configure(project_name, config_file=saved_config)
     170    (run_time, rampup, results_ts_interval, console_logging, progress_bar,
     171     results_database, post_run_script, xml_report,
     172     user_group_configs) = configure(project_name, config_file=saved_config)
    161173    print '\n\nanalyzing results...\n'
    162     results.output_results(output_dir, 'results.csv', run_time, rampup, results_ts_interval, user_group_configs, xml_report)
     174    results.output_results(
     175        output_dir, 'results.csv', run_time, rampup, results_ts_interval,
     176        user_group_configs, xml_report)
    163177    print 'created: %sresults.html\n' % output_dir
    164178    if xml_report:
    165179        print 'created: %sresults.jtl' % output_dir
    166180        print 'created: last_results.jtl\n'
    167    
     181
    168182
    169183
    … …  
    179193            rampup = config.getint(section, 'rampup')
    180194            results_ts_interval = config.getint(section, 'results_ts_interval')
    181             try: 
     195            try:
    182196                console_logging = config.getboolean(section, 'console_logging')
    183197            except ConfigParser.NoOptionError:
    184198                console_logging = False
    185             try: 
     199            try:
    186200                progress_bar = config.getboolean(section, 'progress_bar')
    187201            except ConfigParser.NoOptionError:
    … …  
    208222            user_group_configs.append(ug_config)
    209223
    210     return (run_time, rampup, results_ts_interval, console_logging, progress_bar, results_database, post_run_script, xml_report, user_group_configs)
    211    
     224    return (run_time, rampup, results_ts_interval, console_logging,
     225            progress_bar, results_database, post_run_script, xml_report,
     226            user_group_configs)
     227
    212228
    213229
    … …  
    217233        self.name = name
    218234        self.script_file = script_file
    219    
    220    
    221    
     235
     236
     237
    222238class UserGroup(multiprocessing.Process):
    223     def __init__(self, queue, process_num, user_group_name, num_threads, script_file, run_time, rampup):
     239    def __init__(self, queue, process_num, user_group_name, num_threads,
     240                 script_file, run_time, rampup):
    224241        multiprocessing.Process.__init__(self)
    225242        self.queue = queue
    … …  
    231248        self.rampup = rampup
    232249        self.start_time = time.time()
    233        
     250
    234251    def run(self):
    235252        threads = []
    … …  
    238255            if i > 0:
    239256                time.sleep(spacing)
    240             agent_thread = Agent(self.queue, self.process_num, i, self.start_time, self.run_time, self.user_group_name, self.script_file)
     257            agent_thread = Agent(self.queue, self.process_num, i,
     258                                 self.start_time, self.run_time,
     259                                 self.user_group_name, self.script_file)
    241260            agent_thread.daemon = True
    242261            threads.append(agent_thread)
    243             agent_thread.start()           
     262            agent_thread.start()
    244263        for agent_thread in threads:
    245264            agent_thread.join()
    246        
     265
    247266
    248267
    249268class Agent(threading.Thread):
    250     def __init__(self, queue, process_num, thread_num, start_time, run_time, user_group_name, script_file):
     269    def __init__(self, queue, process_num, thread_num, start_time,
     270                 run_time, user_group_name, script_file):
    251271        threading.Thread.__init__(self)
    252272        self.queue = queue
    … …  
    257277        self.user_group_name = user_group_name
    258278        self.script_file = script_file
    259        
    260         # choose most accurate timer to use (time.clock has finer granularity than time.time on windows, but shouldn't be used on other systems)
     279
     280        # choose most accurate timer to use (time.clock has finer
     281        # granularity than time.time on windows, but shouldn't be used
     282        # on other systems)
    261283        if sys.platform.startswith('win'):
    262284            self.default_timer = time.clock
    263285        else:
    264286            self.default_timer = time.time
    265    
    266    
     287
     288
    267289    def run(self):
    268290        elapsed = 0
    269        
     291
    270292        if self.script_file.lower().endswith('.py'):
    271293            module_name = self.script_file.replace('.py', '')
    272294        else:
    273             sys.stderr.write('ERROR: scripts must have .py extension. can not run test script: %s.  aborting user group: %s\n' % (self.script_file, self.user_group_name))
     295            sys.stderr.write(
     296                'ERROR: scripts must have .py extension. can not run '
     297                'test script: %s.  aborting user group: %s\n' % (
     298                    self.script_file, self.user_group_name))
    274299            return
    275300        try:
    276301            trans = eval(module_name + '.Transaction()')
    277302        except NameError, e:
    278             sys.stderr.write('ERROR: can not find test script: %s.  aborting user group: %s\n' % (self.script_file, self.user_group_name))
     303            sys.stderr.write(
     304                'ERROR: can not find test script: %s.  aborting user '
     305                'group: %s\n' % (self.script_file, self.user_group_name))
    279306            return
    280307        except Exception, e:
    281             sys.stderr.write('ERROR: failed initializing Transaction: %s.  aborting user group: %s\n' % (self.script_file, self.user_group_name))
     308            sys.stderr.write(
     309                'ERROR: failed initializing Transaction: %s.  aborting '
     310                'user group: %s\n' % (self.script_file, self.user_group_name))
    282311            return
    283        
     312
    284313        trans.custom_timers = {}
    285        
    286         # scripts have access to these vars, which can be useful for loading unique data
     314
     315        # scripts have access to these vars, which can be useful for
     316        # loading unique data
    287317        trans.thread_num = self.thread_num
    288318        trans.process_num = self.process_num
    289            
     319
    290320        while elapsed < self.run_time:
    291321            error = ''
    292             start = self.default_timer() 
    293            
     322            start = self.default_timer()
     323
    294324            try:
    295325                trans.run()
    296             except Exception, e:  # test runner catches all script exceptions here
     326            except Exception, e:  # test runner catches all script
     327                                  # exceptions here
    297328                error = str(e).replace(',', '')
    298329
    299330            finish = self.default_timer()
    300            
     331
    301332            scriptrun_time = finish - start
    302             elapsed = time.time() - self.start_time 
     333            elapsed = time.time() - self.start_time
    303334
    304335            epoch = time.mktime(time.localtime())
    305            
    306             fields = (elapsed, epoch, self.user_group_name, scriptrun_time, error, trans.custom_timers)
     336
     337            fields = (elapsed, epoch, self.user_group_name,
     338                      scriptrun_time, error, trans.custom_timers)
    307339            self.queue.put(fields)
    308            
     340
    309341
    310342
    … …  
    318350        self.timer_count = 0
    319351        self.error_count = 0
    320        
     352
    321353        try:
    322354            os.makedirs(self.output_dir, 0755)
    323355        except OSError:
    324356            sys.stderr.write('ERROR: Can not create output directory\n')
    325             sys.exit(1)   
    326    
     357            sys.exit(1)
     358
    327359    def run(self):
    328         with open(self.output_dir + 'results.csv', 'w') as f:     
     360        with open(self.output_dir + 'results.csv', 'w') as f:
    329361            while True:
    330362                try:
    331                     elapsed, epoch, self.user_group_name, scriptrun_time, error, custom_timers = self.queue.get(False)
     363                    (elapsed, epoch, self.user_group_name, scriptrun_time,
     364                    error, custom_timers) = self.queue.get(False)
    332365                    self.trans_count += 1
    333366                    self.timer_count += len(custom_timers)
    334367                    if error != '':
    335368                        self.error_count += 1
    336                     f.write('%i,%.3f,%i,%s,%f,%s,%s\n' % (self.trans_count, elapsed, epoch, self.user_group_name, scriptrun_time, error, repr(custom_timers)))
     369                    f.write('%i,%.3f,%i,%s,%f,%s,%s\n' % (
     370                            self.trans_count, elapsed, epoch,
     371                            self.user_group_name, scriptrun_time, error,
     372                            repr(custom_timers)))
    337373                    f.flush()
    338374                    if self.console_logging:
    339                         print '%i, %.3f, %i, %s, %.3f, %s, %s' % (self.trans_count, elapsed, epoch, self.user_group_name, scriptrun_time, error, repr(custom_timers))
     375                        print '%i, %.3f, %i, %s, %.3f, %s, %s' % (
     376                            self.trans_count, elapsed, epoch,
     377                            self.user_group_name, scriptrun_time, error,
     378                            repr(custom_timers))
    340379                except Queue.Empty:
    341380                    time.sleep(.05)
    342381
    343 
    344 
    345382if __name__ == '__main__':
    346383    main()
Note: See TracChangeset for help on using the changeset viewer.