def export_playback(self, host, export_payload, files, session,
                      playback_session):
      playback_payload = export_payload['playback']
      scenario_name = playback_payload['scenario']
      scenario = u'{0}:{1}'.format(host, scenario_name)
      runnable_info = dict()   
      tracker = Tracker()    
      last_used = tracker.session_last_used(scenario, playback_session, 
                                            'playback')
      if not last_used:
          raise exception_response(400, 
                      title="Unable to find playback session")  
      runnable_info['last_used'] = dict(remote_ip=last_used['remote_ip'],
                                        start_time=str(last_used['start_time']))      
      playback = tracker.get_last_playback(scenario_name, playback_session,
                                           last_used['start_time']) 
      playback = list(playback)
      if not playback:
          raise exception_response(400, 
            title="Unable to find a playback for scenario='{0}', playback_session='{1}'".format(scenario_name, playback_session))
 
      number_of_requests = len(playback)
      runnable_info['number_of_playback_requests'] = number_of_requests
      for nrequest in range(number_of_requests):
          track = playback[nrequest]
          request_text = track.get('request_text')
          if not request_text:
              raise exception_response(400, title='Unable to obtain playback details, was full tracking enabled?')
          stubo_request = StuboRequest(DummyModel(headers=track.get('request_headers'),
                                                  body=request_text))
          vars = track.get('request_params')
          vars.pop('session', None)
          vars.pop('scenario', None)
          request_payload = dict(body=stubo_request.body,
                                 method=stubo_request.method,
                                 host=stubo_request.host,
                                 uri=stubo_request.uri,
                                 path=stubo_request.path,
                                 query=stubo_request.query,
                                 headers=stubo_request.headers)
              
          request_file_name = '{0}_{1}.request'.format(session, nrequest)
          files.append((request_file_name, json.dumps(request_payload, indent=3)))
          # export a response for comparison
          stubo_response_text = track['stubo_response']
          if not isinstance(stubo_response_text, basestring):
              stubo_response_text = unicode(stubo_response_text)
          response_payload = dict(status=track.get('return_code'),
                                  body=stubo_response_text,
                                  headers=track.get('response_headers'))
         
          stubo_response_file_name = '{0}_{1}.stubo_response'.format(session, 
                                                                     nrequest)
          playback_payload['requests'].append(dict(
                              file=request_file_name,
                              vars=vars,
                              response=stubo_response_file_name))
          files.append((stubo_response_file_name, json.dumps(response_payload, 
                                                             indent=3)))
      return runnable_info        

def store_source_recording(scenario_name_key, record_session):
    host, scenario_name = scenario_name_key.split(':')
    # use original put/stub payload logged in tracker
    tracker = Tracker()    
    last_used = tracker.session_last_used(scenario_name_key, 
                                          record_session, 'record')
    if not last_used:
        # empty recordings are currently supported!
        log.debug('Unable to find a recording for session={0}, scenario={1}'.format(record_session, scenario_name_key))
        return 
    
    recording = tracker.get_last_recording(scenario_name, record_session,
                                           last_used['start_time']) 
    recording = list(recording)
    if not recording:
        raise exception_response(400, 
          title="Unable to find a recording for scenario='{0}', record_session='{1}'".format(scenario_name, record_session))
    
    number_of_requests = len(recording)
    scenario_db = Scenario()
    for nrequest in range(number_of_requests):
        track = recording[nrequest]
        request_text = track.get('request_text')
        if not request_text:
            raise exception_response(400, title='Unable to obtain recording details, was full tracking enabled?')
        
        priority = int(track['request_params'].get('priority', nrequest+1))
        stub = parse_stub(request_text, scenario_name_key, 
                          track['request_params'])
        stub.set_priority(priority)
        scenario_db.insert_pre_stub(scenario_name_key, stub)   

def put_bookmark(handler, session_name, name):
    cache = Cache(get_hostname(handler.request)) 
    response = dict(version=version, data = {}) 
    if not session_name:
        raise exception_response(400, title="No session provided")
            
    scenario_key = cache.find_scenario_key(session_name)
    scenario_name = scenario_key.partition(':')[-1]         
               
    # retrieve the request index state for selected session
    index_state = {}
    request_index_data = cache.get_request_index_data(scenario_name)
    if request_index_data:
        for k, v in  request_index_data.iteritems():
            indexed_session_name, _, stub_key = k.partition(':')
            if indexed_session_name == session_name:
                index_state[stub_key] = v          
        
    if not index_state:
        raise exception_response(400,
            title="No indexes found for session '{0}'. Is the session in "
            'playback mode and have state?'.format(session_name)) 
    log.debug("save request index state '{0}' = {1}".format(name, index_state))
    cache.set_saved_request_index_data(scenario_name, name, index_state)
    response['data'][name] = index_state
    return response   

    def assert_valid_session(self, scenario_name, session_name):
        scenario_key = self.scenario_key_name(scenario_name)
        # if session exists it can only be dormant
        if self.exists(scenario_key, session_name):
            session = self.get_session(scenario_name, session_name, local=False)
            session_status = session['status']
            if session_status != 'dormant':
                raise exception_response(400, title='Session already exists '
                                                    '- {0}:{1} in {2} mode.'.format(scenario_key, session_name,
                                                                                    session_status))

                # Limitation: as the session name is not passed in as an arg to get/response
        # we need a mapping to find the session for a scenario. 
        # host:sessions -> session_name->scenario_name
        # we can therefore only have one session name per host/scenario
        sessions_key = '{0}:sessions'.format(self.host)
        scenario_found = self.get_cache_backend()(get_redis_master()).get_raw(sessions_key,
                                                                     session_name)
        if scenario_found and scenario_found != scenario_name:
            raise exception_response(400, title='Session {0} can not be '
                                                'used for scenario: {1}. This session is already being used '
                                                'with another scenario: {2} on host: {3}.'.format(session_name,
                                                                                                  scenario_name,
                                                                                                  scenario_found,
                                                                                                  self.host))

def match(request, session, trace, system_date, url_args, hooks,
          module_system_date=None):
    """Returns the stats of a request match process
    :param request: source stubo request
    :param session: cached session payload associated with this request
    :param module_system_date: optional system date of an external module
    """
    request_text = request.request_body()
    scenario_key = session['scenario']
    session_name = session['session']
    log.debug(u'match: request_text={0}'.format(request_text))
    trace.info('system_date={0}, module_system_date={1}'.format(
        system_date, module_system_date))     
    stats = []
    
    if 'stubs' not in session or not len(session['stubs']):
        if session.get('status') != 'playback':
            raise exception_response(400,
                title="session {0} not in playback mode for scenario "
                "{1}".format(session_name, scenario_key))   
        raise exception_response(500,
          title="no stubs found in session {0} for {1}, status={2}".format(
                session_name, scenario_key, session.get('status')))
 
    stub_count = len(session['stubs'])
    trace.info(u'matching against {0} stubs'.format(stub_count))
    for stub_number in range(stub_count):
        trace.info('stub ({0})'.format(stub_number))
        stub = StubCache(session['stubs'][stub_number], scenario_key, 
                                session_name) 
        source_stub = copy.deepcopy(stub)
        request_copy = copy.deepcopy(request)
        stub, request_copy = transform(stub, 
                                  request_copy, 
                                  module_system_date=module_system_date, 
                                  system_date=system_date,
                                  function='get/response',
                                  cache=session.get('ext_cache'),
                                  hooks=hooks,
                                  stage='matcher',
                                  trace=trace,
                                  url_args=url_args)
        trace.info('finished transformation')
        if source_stub != stub:
            trace.diff('stub ({0}) was transformed'.format(stub_number), 
                       source_stub.payload, stub.payload)
            trace.info('stub ({0}) was transformed into'.format(stub_number),
                       stub.payload)
        if request_copy != request:
            trace.diff('request was transformed', request_copy.request_body(), 
                       request.request_body())
            trace.info('request was transformed into', request_copy.request_body())
                                                                            
        matcher = StubMatcher(trace)
        if matcher.match(request_copy, stub):
            return (True, stub_number, stub)
     
    return (False,)     

def put_module(handler, names):
    module = Module(handler.track.host)
    added = []
    result = dict(version=version)
    for name in names:
        uri, module_name = UriLocation(handler.request)(name)
        log.info('uri={0}, module_name={1}'.format(uri, module_name))
        response, _, code = UrlFetch().get(uri)
        module_name = module_name[:-3]
        last_version = module.latest_version(module_name)
        module_version_name = module.sys_module_name(module_name, 
                                                     last_version+1) 
        if last_version and response == module.get_source(module_name,
                                                          last_version):
            msg = 'Module source has not changed for {0}'.format(
                                                        module_version_name)
            result['data'] = dict(message=msg)
        try:  
            code, mod = module.add_sys_module(module_version_name, response)
            log.debug('{0}, {1}'.format(mod, code))
        except Exception, e:
            msg = 'error={0}'.format(e)
            raise exception_response(400,
                title='Unable to compile {0}:{1}, {2}'.format(module.host(), 
                module_version_name, msg))
        module.add(module_name, response)
        added.append(module_version_name)

 def run_recording(self, recording):
     scenario_args = dict(scenario=recording['scenario'],
                          session=recording['session'])
     recordings = []
     recordings.append(('delete/stubs', self.run_command('delete/stubs',
                                                         **scenario_args)))
     recordings.append(('begin/session', self.run_command('begin/session',
                                                          mode='record', **scenario_args)))
     for stub in recording['stubs']:
         try:
             if 'file' in stub:
                 stub_data_url = urljoin(self.parent_path, stub['file'])
                 stub_payload, _, _ = UrlFetch().get(stub_data_url)
             elif 'json' in stub:
                 stub_payload = stub['json']
             else:
                 raise exception_response(400, title="A stub definition must "
                                                     "contain either a 'file' location key or a 'json' key that "
                                                     "defines an inplace payload.")
             vars = stub.get('vars', {})
             vars.update(scenario_args)
             url = self.get_url('put/stub', **vars)
             log.debug(u'run_command: {0}'.format(url))
             if not isinstance(stub_payload, dict):
                 stub_payload = json.loads(stub_payload)
             response = UrlFetch().post(url, data=None, json=stub_payload)
             recordings.append(('put/stub', response.status_code))
         except Exception, e:
             recordings.append(('put/stub', str(e)))

 def run(self):
     if not self.cmd_file_url:
         raise exception_response(500,
                                  title='run requires a cmd_file_url input to the ctor.')
     cmds, _, _ = UrlFetch().get(self.location(self.cmd_file_url)[0])
     cmds_expanded = run_template(cmds,
                                  # utility functions   
                                  roll_date=roll_date,
                                  today=today_str,
                                  as_date=as_date,
                                  parse_xml=parse_xml,
                                  **self.location.request.arguments)
     try:
         payload = self.parse(cmds_expanded)
     except Exception, e:
         raise exception_response(400, title="Unable to parse '{0}', error={1}".format(self.cmd_file_url, e))

    def _begin_session(self):
        """

        Begins session
        :raise exception_response:

        Example output:
        Record new session
        {
         "version": "0.6.3",
         "data":
            {"status": "record",
            "scenario": "localhost:scenario_rest_api",
            "scenarioRef": "/stubo/api/v2/scenarios/objects/localhost:scenario_rest_api",
            "scenario_id": "55acba53fc456205eaf7e258",
            "session": "new_session_rest2",
            "message": "Record mode initiated...."}
        }
        """
        warm_cache = asbool(self.get_argument('warm_cache', False))
        if not self.mode:
            raise exception_response(400,
                                     title="'mode' of playback or record required")
        # passing parameters to api v2 handler, it avoids creating scenario if there is an existing one,
        # since all scenarios should be existing!
        response = api_v2_begin_session(self, self.scenario_name,
                                        self.session_name,
                                        self.mode,
                                        self.get_argument('system_date', None),
                                        warm_cache)
        # adding scenarioRef key for easier resource access.
        response['data']['scenarioRef'] = '/stubo/api/v2/scenarios/objects/%s' % response['data']['scenario']
        self.write(response)

def run_commands(handler, cmds_text):
    response = {
        'version': version
    }
    host = get_hostname(handler.request)

    cmd_processor = TextCommandsImporter(UriLocation(handler.request))
    cmds = cmd_processor.parse(cmds_text)
    if any(x for x in cmds if urlparse(x).path not in form_input_cmds):
        raise exception_response(400, title='command/s not supported, must be '
                                            'one of these: {0}'.format(form_input_cmds))

    responses = cmd_processor.run_cmds(cmds)
    log.debug('responses: {0}'.format(responses))

    response['data'] = {
        'executed_commands': responses,
        'number_of_requests': len(responses['commands']),
        'number_of_errors': len([x for x in responses['commands'] if x[1] > 399])
    }

    def get_links(cmd):
        cmd_uri = urlparse(cmd)
        scenario_name = cmd_uri.query.partition('=')[-1]
        scenario_name_key = '{0}:{1}'.format(host, scenario_name)
        files = [(scenario_name + '.zip',), (scenario_name + '.tar.gz',),
                 (scenario_name + '.jar',)]
        links = get_export_links(handler, scenario_name_key, files)
        return links

    export_links = [(x, get_links(x)) for x in cmds if 'get/export' in x]
    if export_links:
        response['data']['export_links'] = export_links

    return response

 def find_scenario_key(self, session_name):
     scenario_key = self.get_scenario_key(session_name) 
     if not scenario_key: 
         raise exception_response(400,
             title='session not found - {0}:{1}'.format(self.host, 
                                                        session_name))
     return scenario_key 

 def run_recording(self, recording):
     scenario_args = dict(scenario=recording["scenario"], session=recording["session"])
     recordings = []
     recordings.append(("delete/stubs", self.run_command("delete/stubs", **scenario_args)))
     recordings.append(("begin/session", self.run_command("begin/session", mode="record", **scenario_args)))
     for stub in recording["stubs"]:
         try:
             if "file" in stub:
                 stub_data_url = urljoin(self.parent_path, stub["file"])
                 stub_payload, _, _ = UrlFetch().get(stub_data_url)
             elif "json" in stub:
                 stub_payload = stub["json"]
             else:
                 raise exception_response(
                     400,
                     title="A stub definition must "
                     "contain either a 'file' location key or a 'json' key that "
                     "defines an inplace payload.",
                 )
             vars = stub.get("vars", {})
             vars.update(scenario_args)
             url = self.get_url("put/stub", **vars)
             log.debug(u"run_command: {0}".format(url))
             if not isinstance(stub_payload, dict):
                 stub_payload = json.loads(stub_payload)
             response = UrlFetch().post(url, data=None, json=stub_payload)
             recordings.append(("put/stub", response.status_code))
         except Exception, e:
             recordings.append(("put/stub", str(e)))

 def run_config(files):
     # find the config file in the extract and run it
     config_file =  [x for x in files if x.endswith(config_types)]
     if not config_file:
         raise exception_response(400, title='Config file not'
                 ' found in archive: {0}'.format(cmd_file_url))
     return run(os.path.join('static', 'imports', temp_dir_name, 
                             config_file[0]))    

def begin_session_request(handler):
    scenario = handler.track.scenario = get_scenario_arg(handler)
    session = get_session_arg(handler)
    mode = handler.get_argument('mode', None)
    warm_cache = asbool(handler.get_argument('warm_cache', False)) 
    if not mode:
        raise exception_response(400, 
                                 title="'mode' of playback or record required") 
    return begin_session(handler, scenario, session, mode,
                         handler.get_argument('system_date', None), warm_cache)                             

def run_command_file(cmd_file_url, request, static_path):

    def run(cmd_file_path):
        response = {
            'version' : version
        }
        is_legacy_text_format = cmd_file_path.endswith('.commands')
        location = UriLocation(request) 
        cmd_processor = TextCommandsImporter(location, cmd_file_path) if is_legacy_text_format else YAMLImporter(location, cmd_file_path)  
        responses = cmd_processor.run()
        log.debug('responses: {0}'.format(responses))
        response['data'] = responses
        return response
    
    file_type = os.path.basename(urlparse(cmd_file_url).path).rpartition(
                                '.')[-1]  
    supported_types = ('zip', 'gz', 'tar', 'jar')  
    if file_type in supported_types:
        # import compressed contents and run contained config file
        import_dir = os.path.join(static_path, 'imports')
        with make_temp_dir(dirname=import_dir) as temp_dir: 
            temp_dir_name = os.path.basename(temp_dir)
            response, headers, status_code = UrlFetch().get(
                                        UriLocation(request)(cmd_file_url)[0])
            content_type = headers["Content-Type"]
            log.debug('received {0} file.'.format(content_type))
            config_types = ('.yaml', '.commands')
            
            def run_config(files):
                # find the config file in the extract and run it
                config_file =  [x for x in files if x.endswith(config_types)]
                if not config_file:
                    raise exception_response(400, title='Config file not'
                            ' found in archive: {0}'.format(cmd_file_url))
                return run(os.path.join('static', 'imports', temp_dir_name, 
                                        config_file[0]))    
            
            if content_type == 'application/x-tar' or file_type == 'tar':
                with closing(tarfile.open(fileobj=StringIO(response))) as tar:
                    tar.extractall(path=temp_dir)
                    response = run_config(tar.getnames())         
               
            elif content_type in ('application/zip',
                                  'application/java-archive') or file_type in \
                                  ('zip', 'jar'):
                with zipfile.ZipFile(StringIO(response)) as zipf:
                    zipf.extractall(path=temp_dir)
                    response = run_config(zipf.namelist()) 
            else:
                raise exception_response(400, title='Expected Content-Type has'
                    ' to be one of these: {0} not {1}'.format(supported_types,
                                                              content_type))
    else:
        response = run(cmd_file_url)
    return response

def get_stats(handler):
    # cluster.host
    # "cluster1.ahost or cluster1.*",
    response = {
        'version': version
    }
    cluster = handler.get_argument('cluster', handler.settings['cluster_name'])
    host = handler.get_argument('host', '*')
    cluster_host_server = '{0}.{1}'.format(cluster, host)
    metric = handler.get_argument('metric', 'latency')
    if metric == 'latency':
        target = 'averageSeries(stats.timers.stubo.{0}.stuboapi.get_response.latency.mean_90)'.format(
            cluster_host_server)
        percent_above_value = int(handler.get_argument('percent_above_value', 50))
    else:
        raise exception_response(400,
                                 title="metric '{0}' parameter not supported.".format(metric))

    server = handler.settings.get('graphite.host')
    auth = (handler.settings.get('graphite.user'),
            handler.settings.get('graphite.passwd'))
    from_str = handler.get_argument('from', '-1hours')
    to_str = handler.get_argument('to', 'now')
    json_response, hdrs, status_code = get_graphite_stats(server, auth,
                                                          target=target, from_str=from_str, to_str=to_str)
    if status_code != 200 or (hdrs['content-type'] != 'application/json'):
        raise exception_response(500,
                                 title='unexpected response from graphite => {0}: {1}'.format(
                                     hdrs, json_response))

    ts = get_graphite_datapoints(json_response, target)
    slow = [x for x in ts if x[0] > percent_above_value]
    pcent = len(slow) / float(len(ts)) * 100
    response['data'] = {
        'target': target,
        'metric': metric,
        'pcent': pcent,
        'percent_above_value': percent_above_value,
        'from': from_str,
        'to': to_str
    }
    return response

def get_tracks(handler, scenario_filter, session_filter, show_only_errors, skip,
               limit, start_time, latency, all_hosts, function):
    tracker = Tracker()
    tracker_filter = {}
    if start_time:
        try:
            start = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
            tracker_filter['start_time'] = {"$lt": start}
        except ValueError, e:
            raise exception_response(400,
                title='start_time format error: {0}'.format(e)) 

 def raise_on_error(self, response, url):
     status = response.status_code
     if status != 200:
         if status == 404:
             msg = "File not found using url: {0}".format(url)
             raise exception_response(response.status_code, title=msg)
         else:
             json_response = None
             try:
                 json_response = response.json()
             except Exception:
                 pass
             if json_response and "error" in json_response:
                 # its one of ours, reconstruct the error
                 raise exception_response(response.status_code, title=json_response["error"].get("message"))
             if response.status_code > 399:
                 raise exception_response(
                     response.status_code,
                     title="Error executing request '{0}', reason={1}".format(url, response.reason),
                 )

 def get_session_with_delay(self, scenario_name, session_name, retry_count=5,
                            retry_interval=1):
     for i in range(retry_count):
         scenario_key = self.scenario_key_name(scenario_name)
         session = self.get_session(scenario_name, session_name)
         if not session:
             raise exception_response(500, 
                 title="session {0} not found!".format(session_name))
         if session['status'] == 'playback': 
             # indicates session has been replicated
             break
         elif session['status'] == 'record':
             raise exception_response(409, title="session '{0}' for scenario" \
                 "'{1}' in record mode, playback expected ...".format(
                 session_name, scenario_key))
         else:    
             log.warn("slave session data not available! try again in {0} "
                      'secs'.format(retry_interval))
             time.sleep(retry_interval)       
     return session, i     

 def get_sys_module(self, name, version):
     # lazy load if not in already in sys.modules
     module_name = self.sys_module_name(name, version)
     if module_name not in sys.modules:
         # load from source
         code = self.get_source(name, version)
         if not code:
             raise exception_response(500, title="module '{0}' source not "
                                      "found.".format(module_name))
         self.add_sys_module(module_name, code)
     return sys.modules[module_name]