def initializeProfile(profile_dir, browser_config):
  if browser_config["profile_path"] != {}:
      if not (ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir, browser_config['init_url'])):
         raise talosError("failed to initialize browser")
      time.sleep(10)
      if checkAllProcesses():
         raise talosError("browser failed to close after being initialized") 

    def __init__(self, filename=None, results_raw=None, counter_results=None, global_counters=None):
        """
        - shutdown : whether to record shutdown results or not
        """

        self.counter_results = counter_results
        self.global_counters = global_counters

        if not (results_raw or filename):
            raise utils.talosError("Must specify filename or results_raw")

        self.filename = filename
        if results_raw is None:
            # read the file

            if not os.path.isfile(filename):
                raise utils.talosError("File '%s' does not exist" % filename)
            f = file(filename)
            exception = None
            try:
                results_raw = f.read()
            except Exception, exception:
                pass
            try:
                f.close()
            except:
                pass
            if exception:
                raise exception

def buildCommandLine(test):
  """build firefox command line options for tp tests"""

  # sanity check pageloader values
  # mandatory options: tpmanifest, tpcycles
  if test['tpcycles'] not in range(1, 1000):
    raise talosError('pageloader cycles must be int 1 to 1,000')
  if test.get('tpdelay') and test['tpdelay'] not in range(1, 10000):
    raise talosError('pageloader delay must be int 1 to 10,000')
  if 'tpmanifest' not in test:
    raise talosError("tpmanifest not found in test: %s" % test)
    # TODO: should probably check if the tpmanifest exists

  # build pageloader command from options
  url = ['-tp', test['tpmanifest']]
  CLI_bool_options = ['tpchrome', 'tpmozafterpaint', 'tpnoisy', 'rss', 'tprender']
  CLI_options = ['tpcycles', 'tppagecycles', 'tpdelay']
  for key in CLI_bool_options:
      if test.get(key):
          url.append('-%s' % key)
  for key in CLI_options:
      value = test.get(key)
      if value:
          url.extend(['-%s' % key, str(value)])

  # XXX we should actually return the list but since we abuse
  # the url as a command line flag to pass to firefox all over the place
  # will just make a string for now
  return ' '.join(url)

def InitializeNewProfile(browser_path, process, browser_wait, extra_args, profile_dir, init_url, log):
  """Runs browser with the new profile directory, to negate any performance
     hit that could occur as a result of starting up with a new profile.  
     Also kills the "extra" browser that gets spawned the first time browser
     is run with a new profile.

  Args:
    browser_path: String containing the path to the browser exe
    profile_dir: The full path to the profile directory to load
  """
  PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE)
  command_line = ffprocess.GenerateBrowserCommandLine(browser_path, extra_args, profile_dir, init_url)
  bcontroller = path('talos/bcontroller.py')
  process = subprocess.Popen('python %s --command "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, process, browser_wait, log), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
  res = 0
  total_time = 0
  while total_time < 600: #10 minutes
    time.sleep(1)
    if process.poll() != None: #browser_controller completed, file now full
      if not os.path.isfile(log):
        raise talosError("no output from browser")
      results_file = open(log, "r")
      results_raw = results_file.read()
      results_file.close()
      match = PROFILE_REGEX.search(results_raw)
      if match:
        res = 1
        print match.group(1)
        break
    total_time += 1

  return res

    def results(self):
        """return results instance appropriate to the format detected"""

        if self.format not in self.classes:
            raise utils.talosError("Unable to find a results class for format: %s" % repr(self.format))

        return self.classes[self.format](self.browser_results)

    def copyDirToDevice(self, localDir):
        head, tail = os.path.split(localDir)

        remoteDir = self.rootdir + self.dirSlash + tail
        if (self.testAgent.pushDir(localDir, remoteDir) is None):
            raise talosError("Unable to copy '%s' to remote device '%s'" % (localDir, remoteDir))
        return remoteDir

  def _updateCounterPathsForChildProcesses(self, counter):
    # Create a counter path for each instance of the child process that
    # is running.  If any of these paths are not in our counter list,
    # add them to our counter query and append them to the counter list,
    # so that we'll begin tracking their statistics.  We don't need to
    # worry about removing invalid paths from the list, as getCounterValue()
    # will generate a value of 0 for those.
    hq = self.registeredCounters[counter][0]
    oldCounterListLength = len(self.registeredCounters[counter][1])

    pdh.PdhEnumObjectsA(None, None, 0, 1, 0, True)

    expandedPaths = _getExpandedCounterPaths(self.childProcess, counter)
    if not expandedPaths:
      return
    for expandedPath in expandedPaths:
      alreadyInCounterList = False
      for singleCounter in self.registeredCounters[counter][1]:
        if expandedPath == singleCounter[1]:
          alreadyInCounterList = True
      if not alreadyInCounterList:
        try:
          newhc = HANDLE()
          if pdh.PdhAddCounterA(hq, expandedPath, 0, byref(newhc)) != 0:
            raise talosError("Could not add expanded win32 counter %s" %
                             expandedPath)
          self.registeredCounters[counter][1].append((newhc, expandedPath))
        except:
          continue

    if oldCounterListLength != len(self.registeredCounters[counter][1]):
      pdh.PdhCollectQueryData(hq)

  def run(self):
    self.bwaiter = BrowserWaiter(self.remoteProcess, **self.options)
    noise = 0
    prev_size = 0
    while not self.bwaiter.hasTime():
      if noise > self.test_timeout: # check for frozen browser
        try:
          if os.path.isfile(self.browser_log):
            os.chmod(self.browser_log, 0777)
          results_file = open(self.browser_log, "a")
          results_file.write("\n__FAILbrowser frozen__FAIL\n")
          results_file.close()
        except IOError, e:
          raise talosError(str(e))
        return
      time.sleep(1)
      try:
        open(self.browser_log, "r").close() #HACK FOR WINDOWS: refresh the file information
        size = os.path.getsize(self.browser_log)
      except:
        size = 0

      if size > prev_size:
        prev_size = size
        noise = 0
      else:
        noise += 1

    def add(self, results, counter_results=None):
        """
        accumulate one cycle of results
        - results : TalosResults instance or path to browser log
        - counter_results : counters accumulated for this cycle
        """

        if isinstance(results, basestring):
            # ensure the browser log exists
            if not os.path.isfile(results):
                raise talosError("no output from browser [%s]" % results)

            # convert to a results class via parsing the browser log
            browserLog = BrowserLogResults(filename=results, counter_results=counter_results, global_counters=self.global_counters)
            results = browserLog.results()

        self.using_xperf = browserLog.using_xperf
        # ensure the results format matches previous results
        if self.results:
            if not results.format == self.results[0].format:
                raise utils.talosError("Conflicting formats for results")
        else:
            self.format = results.format

        self.results.append(results)

        if counter_results:
            self.all_counter_results.append(counter_results)

    def runProgram(self, browser_config, command_args, timeout=1200):
        remoteLog = os.path.join(self.getDeviceRoot() + '/' + browser_config['browser_log'])
        self.removeFile(remoteLog)
        # bug 816719, remove sessionstore.js so we don't interfere with talos
        self.removeFile(os.path.join(self.getDeviceRoot(), "profile/sessionstore.js"))

        env = ""
        for key, value in browser_config['env'].items():
            env = "%s %s=%s" % (env, key, value)
        command_line = "%s %s" % (env, ' '.join(command_args))

        self.recordLogcat()
        firstTime = time.time()
        retVal = self.launchProcess(' '.join(command_args), outputFile=remoteLog, timeout=timeout)
        logcat = self.getLogcat()
        if logcat:
            with open('logcat.log', 'w') as f:
                f.write(''.join(logcat[-500:-1]))

        data = self.getFile(remoteLog, browser_config['browser_log'])
        with open(browser_config['browser_log'], 'a') as logfile:
            logfile.write("__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp\n" % (firstTime * 1000))
            logfile.write("__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp\n" % int(time.time() * 1000))
        if not retVal and data == '':
            raise talosError("missing data from remote log file")

        # Wait out the browser closing
        time.sleep(browser_config['browser_wait'])

    def cleanupProcesses(self, process_name, child_process, browser_wait):
        #kill any remaining browser processes
        #returns string of which process_names were terminated and with what signal

        # if we are running this against the metro browser, we currently use
        # metrotestharness.exe as a way to launch the metro browser process.
        # Talos thinks this harness is the browser, see:
        # http://hg.mozilla.org/build/talos/file/8c5f2725fbdd/talos/run_tests.py#l249
        # We must inform talos about the sub process, the metro browser itself,
        # that is spawned from metrotestharness. The metro browser proc is
        # given the same name as the non metro equivalent: 'firefox.exe'
        if process_name == "metrotestharness" and \
                "firefox" not in self.extra_prog:
            self.extra_prog.append("firefox")

        processes_to_kill = filter(lambda n: n, ([process_name, child_process] +
                                                 self.extra_prog))
        utils.debug("Terminating: %s", ", ".join(str(p) for p in processes_to_kill))
        terminate_result = self.TerminateAllProcesses(browser_wait, *processes_to_kill)
        #check if anything is left behind
        if self.checkAllProcesses(process_name, child_process):
            #this is for windows machines.  when attempting to send kill messages to win processes the OS
            # always gives the process a chance to close cleanly before terminating it, this takes longer
            # and we need to give it a little extra time to complete
            time.sleep(browser_wait)
            processes = self.checkAllProcesses(process_name, child_process)
            if processes:
                raise talosError("failed to cleanup processes: %s" % processes)

        return terminate_result

    def getPlatformType(self, remote):

        _ffprocess = None
        if remote == True:
            platform_type = 'remote_'
            import cmanager
            CounterManager = cmanager.CounterManager
        elif platform.system() == "Linux":
            import cmanager_linux
            CounterManager = cmanager_linux.LinuxCounterManager
            platform_type = 'linux_'
            _ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version(): #winxp
                platform_type = 'win_'
            elif '6.1' in platform.version(): #w7
                platform_type = 'w7_'
            elif '6.2' in platform.version(): #w8
                platform_type = 'w8_'
            else:
                raise talosError('unsupported windows version')
            import cmanager_win32
            CounterManager = cmanager_win32.WinCounterManager
            _ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            import cmanager_mac
            CounterManager = cmanager_mac.MacCounterManager
            platform_type = 'mac_'
            _ffprocess = MacProcess()
        return CounterManager, platform_type, _ffprocess

    def addRemoteServerPref(self, profile_dir, server):
        """
          edit the user.js in the profile (on the host machine) and
          add the xpconnect priviledges for the remote server
        """
        import urlparse
        user_js_filename = os.path.join(profile_dir, 'user.js')
        user_js_file = open(user_js_filename, 'a+')

        #NOTE: this should be sufficient for defining a docroot
        scheme = "http://"
        if (server.startswith('http://') or
            server.startswith('chrome://') or
            server.startswith('file:///')):
          scheme = ""
        elif (server.find('://') >= 0):
          raise talosError("Unable to parse user defined webserver: '%s'" % (server))

        url = urlparse.urlparse('%s%s' % (scheme, server))

        port = url.port
        if not url.port or port < 0:
          port = 80

        #TODO: p2 is hardcoded, how do we determine what prefs.js has hardcoded?
        remoteCode = """
user_pref("capability.principal.codebase.p2.granted", "UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead");
user_pref("capability.principal.codebase.p2.id", "http://%(server)s");
user_pref("capability.principal.codebase.p2.subjectName", "");
""" % { "server": server, "host": url.hostname, "port": int(port) }
        user_js_file.write(remoteCode)
        user_js_file.close()

    def InitializeNewProfile(self, profile_dir, browser_config):
        """Runs browser with the new profile directory, to negate any performance
            hit that could occur as a result of starting up with a new profile.  
            Also kills the "extra" browser that gets spawned the first time browser
            is run with a new profile.

        Args:
            browser_config: object containing all the browser_config options
            profile_dir: The full path to the profile directory to load
        """
        PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE)
        command_line = self.ffprocess.GenerateBrowserCommandLine(browser_config["browser_path"], 
                                                                 browser_config["extra_args"], 
                                                                 profile_dir, 
                                                                 browser_config["init_url"])

        log = browser_config['browser_log']
        if (browser_config['webserver'] != 'localhost'):
            b_log = browser_config['deviceroot'] + '/' + browser_config['browser_log']

        b_cmd = self.ffprocess.GenerateBControllerCommandLine(command_line, browser_config, {})
        process = subprocess.Popen(b_cmd, universal_newlines=True, shell=True, bufsize=0, env=os.environ)

        timeout = True
        total_time = 0
        while total_time < 1200: #20 minutes
            time.sleep(1)
            if process.poll() != None: #browser_controller completed, file now full
                timeout = False
                break
            total_time += 1

        res = 0
        if (timeout == False):
            if not os.path.isfile(log):
                raise talosError("initalization has no output from browser")
            results_file = open(log, "r")
            results_raw = results_file.read()
            results_file.close()
            match = PROFILE_REGEX.search(results_raw)
            if match:
                res = 1
                print match.group(1)
        else:
            raise talosError("initialization timed out")

        return res

 def onTimeout(self):
     """
     When we timeout, dictate this in the log file.
     """
     if os.path.isfile(self.logfile):
         os.chmod(self.logfile, 0777)
     self.logToFile("\n__FAILbrowser frozen__FAIL\n")
     self.closeLogFile()
     raise talosError("timeout")

    def InitializeNewProfile(self, profile_dir, browser_config):
        """Runs browser with the new profile directory, to negate any performance
            hit that could occur as a result of starting up with a new profile.  
            Also kills the "extra" browser that gets spawned the first time browser
            is run with a new profile.
            Returns 1 (success) if PROFILE_REGEX is found,
            and 0 (failure) otherwise

        Args:
            browser_config: object containing all the browser_config options
            profile_dir: The full path to the profile directory to load
        """
        INFO_REGEX = re.compile('__browserInfo(.*)__browserInfo', re.DOTALL|re.MULTILINE)
        PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE)

        command_args = utils.GenerateBrowserCommandLine(browser_config["browser_path"], 
                                                        browser_config["extra_args"], 
                                                        browser_config["deviceroot"],
                                                        profile_dir, 
                                                        browser_config["init_url"])

        if not browser_config['remote']:
            browser = talosProcess.talosProcess(command_args, env=os.environ.copy(), logfile=browser_config['browser_log'])
            browser.run()
            browser.wait()
            browser = None
            time.sleep(5)
        else:
            self.ffprocess.runProgram(browser_config, command_args, timeout=1200)

        res = 0
        if not os.path.isfile(browser_config['browser_log']):
            raise talosError("initalization has no output from browser")
        results_file = open(browser_config['browser_log'], "r")
        results_raw = results_file.read()
        results_file.close()
        match = PROFILE_REGEX.search(results_raw)
        if match:
            res = 1
        else:
            utils.info("Could not find %s in browser_log: %s", PROFILE_REGEX.pattern, browser_config['browser_log'])
            utils.info("Raw results:%s", results_raw)
            utils.info("Initialization of new profile failed")
        match = INFO_REGEX.search(results_raw)
        if match:
            binfo = match.group(1)
            print binfo
            for line in binfo.split('\n'):
                if line.strip().startswith('browser_name'):
                    browser_config['browser_name'] = line.split(':')[1]
                if line.strip().startswith('browser_version'):
                    browser_config['browser_version'] = line.split(':')[1]
                if line.strip().startswith('buildID'):
                    browser_config['buildid'] = line.split(':')[1]

        return res

def process_Request(post):
  links = ""
  lines = post.split('\n')
  for line in lines:
    if line.find("RETURN\t") > -1:
        links += line.replace("RETURN\t", "") + '\n'
    utils.debug("process_Request line: " + line.replace("RETURN\t", ""))
  if not links:
    raise talosError("send failed, graph server says:\n" + post)
  return links

    def cleanupAndCheckForCrashes(self, browser_config, profile_dir):
        cleanup_result = self._ffprocess.cleanupProcesses(browser_config['process'], 
                                                          browser_config['child_process'], 
                                                          browser_config['browser_wait']) 
        if platform.system() in ('Windows', 'Microsoft'):
            stackwalkpaths = ['win32', 'minidump_stackwalk.exe']
        elif platform.system() == 'Linux':
            if platform.machine() == 'armv6l':
                stackwalkpaths = ['maemo', 'minidump_stackwalk']
            elif '64' in platform.architecture()[0]: #are we 64 bit?
                stackwalkpaths = ['linux64', 'minidump_stackwalk']
            else:
                stackwalkpaths = ['linux', 'minidump_stackwalk']
        elif platform.system() == 'Darwin':
            stackwalkpaths = ['osx', 'minidump_stackwalk']
        else:
            return
        stackwalkbin = os.path.join(os.path.dirname(__file__), 'breakpad', *stackwalkpaths)

        found = False
        minidumpdir = os.path.join(profile_dir, 'minidumps')
        if browser_config['remote'] == True:
            minidumpdir = tempfile.mkdtemp()
            self._ffprocess.testAgent.getDirectory(profile_dir + '/minidumps/', minidumpdir)
        
        for dump in glob.glob(os.path.join(minidumpdir, '*.dmp')):
            utils.noisy("Found crashdump: " + dump)
            if browser_config['symbols_path']:
                nullfd = open(os.devnull, 'w')
                subprocess.call([stackwalkbin, dump, browser_config['symbols_path']], stderr=nullfd)
                nullfd.close()
            os.remove(dump)
            found = True

        if browser_config['remote'] == True:
            self._hostproc.removeDirectory(minidumpdir)
   
        if found:
            if cleanup_result:
                raise talosError("stack found after process termination (" + cleanup_result+ ")")
            else:
                raise talosError("crash during run (stack found)")

def cleanupProcesses():
  #kill any remaining firefox processes
  ffprocess.TerminateAllProcesses("firefox", "crashreporter", "dwwin", "talkback")
  #check if anything is left behind
  if checkAllProcesses(): 
    #this is for windows machines.  when attempting to send kill messages to win processes the OS
    # always gives the process a chance to close cleanly before terminating it, this takes longer
    # and we need to give it a little extra time to complete
    time.sleep(10)
    if checkAllProcesses():
      raise talosError("failed to cleanup")

 def process_Request(self, post):
     """get links from the graphserver response"""
     links = ""
     for line in post.splitlines():
         if line.find("RETURN\t") > -1:
             line = line.replace("RETURN\t", "")
             links += line + '\n'
         utils.debug("process_Request line: %s", line)
     if not links:
         raise utils.talosError("send failed, graph server says:\n%s" % post)
     return links