def get_bayesdb(venv_dir, versions, options):
    for package in ("crosscat", "bayeslite", "bdbcontrib"):
        pdir = os.path.join(venv_dir, package)
        need_repo = (options.run_tests or not options.from_pypi or
                     not re.search(r'^tags', versions[package]))
        if need_repo:
            check_git()
            if os.path.exists(pdir):
                run("cd -- %s && git checkout master && git pull" % (pdir,),
                    stdout=options.stdout)
            else:
                run("git clone https://github.com/probcomp/%s %s" %
                    (package, shellquote(pdir)),
                    stdout=options.stdout)
            versions['have_repo_for_'+package] = True
        if need_repo and versions[package] != "HEAD":
            venv_run(venv_dir,
                     "cd -- %s && git checkout %s" % (
                         pdir, shellquote(versions[package])),
                     stdout=options.stdout)
        if options.from_pypi and re.search(r'^tags', versions[package]):
            pypi_version = re.sub(r'.*v', '', versions[package])
            install_package(venv_dir, package+"=="+pypi_version, options)
        elif need_repo and options.install_bayesdb:
            venv_run(venv_dir,
                     "cd -- %s && pip install ." % (pdir,),
                     stdout=options.stdout)
        else:
            pass # Not requesting installation is fine.
    return versions

def make_starter_script(macos_path):
  starter_script = '''#!/bin/bash

set -e
wd=`dirname -- "$0"`
cd -- "$wd"
wd=`pwd -P`
NAME=`basename -- "$(dirname -- "$(dirname -- "$wd")")" .app`

activate="$wd/venv/bin/activate"
ldpath="$wd/lib"

# Clear any user's PYTHONPATH setting, which may interfere with what
# we need.
unset PYTHONPATH
export PYTHONPATH="$wd/venv/lib/python2.7/site-packages"

source "$activate"
export DYLD_LIBRARY_PATH="$ldpath"
export MPLBACKEND=pdf

# Download and run the examples in someplace writeable:
"$wd/venv/bin/bayesdb-demo" --destination "$HOME/Documents"
'''
  startsh_path = os.path.join(macos_path, "start.sh")
  with open(startsh_path, "w") as startsh:
    startsh.write(starter_script)
  run("chmod +x %s" % (shellquote(startsh_path),))

def build_dmg():
  run("scp build_dmg.py shell_utils.py [email protected]%s:" % (HOST,))
  build_run('PATH="%s:$PATH" python build_dmg.py' % (HPATH,))
  run("scp [email protected]%s:Desktop/Bayeslite*.dmg %s" % (HOST, SCRATCH))
  name = build_outputof("cd Desktop && ls -t Bayeslite*.dmg | tail -1").strip()
  echo("NAME:", name)
  build_run("/bin/rm -f Desktop/Bayeslite*.dmg")
  return name

def test_dmg(name):
  needed = ['osx/bayeslite/*.scpt',
            'src/shell_utils.py',
            'osx/bayeslite/test_dmg.py',
            os.path.join(SCRATCH, name)]
  run("scp %s [email protected]%s:Desktop/" %
      (" ".join(needed), HOST))
  test_run("python Desktop/test_dmg.py %s" % name)

def Sync(revisions=None, force=False, delete_unversioned_trees=False,
         verbose=False, jobs=None, no_hooks=False, extra_args=None):
  """ Update the local checkout using gclient.

  Args:
      revisions: optional list of (branch, revision) tuples indicating which
          projects to sync to which revisions.
      force: whether to run with --force.
      delete_unversioned_trees: whether to run with --delete-unversioned-trees.
      verbose: whether to run with --verbose.
      jobs: optional argument for the --jobs flag.
      no_hooks: whether to run with --nohooks.
      extra_args: optional list; any additional arguments.
  """
  for branch, _ in (revisions or []):
    # Do whatever it takes to get up-to-date with origin/master.
    if os.path.exists(branch):
      with misc.ChDir(branch):
        # First, fix the git identity if needed.
        maybe_fix_identity()

        # If there are local changes, "git checkout" will fail.
        shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
        # In case HEAD is detached...
        shell_utils.run([GIT, 'checkout', 'master'])
        # Always fetch, in case we're unmanaged.
        shell_utils.run_retry([GIT, 'fetch'], attempts=5)
        # This updates us to origin/master even if master has diverged.
        shell_utils.run([GIT, 'reset', '--hard', 'origin/master'])

  cmd = ['sync', '--no-nag-max']
  if verbose:
    cmd.append('--verbose')
  if force:
    cmd.append('--force')
  if delete_unversioned_trees:
    cmd.append('--delete_unversioned_trees')
  if jobs:
    cmd.append('-j%d' % jobs)
  if no_hooks:
    cmd.append('--nohooks')
  for branch, revision in (revisions or []):
    if revision:
      cmd.extend(['--revision', '%[email protected]%s' % (branch, revision)])
  if extra_args:
    cmd.extend(extra_args)
  output = _RunCmd(cmd)

  # "gclient sync" just downloads all of the commits. In order to actually sync
  # to the desired commit, we have to "git reset" to that commit.
  for branch, revision in (revisions or []):
    with misc.ChDir(branch):
      if revision:
        shell_utils.run([GIT, 'reset', '--hard', revision])
      else:
        shell_utils.run([GIT, 'reset', '--hard', 'origin/master'])
  return output

def wrap_as_macos_dir(build_dir, name):
  """Return the dmg root dir inside build_dir, and within that the MacOs dir."""
  dist_dir = os.path.join(build_dir, "dmgroot")
  macos_path = os.path.join(dist_dir, name + ".app", "Contents", "MacOS")
  os.makedirs(macos_path)
  run("/bin/ln -s /Applications %s" % (shellquote(dist_dir),))
  make_starter_script(macos_path)
  make_launcher_script(macos_path, name)
  return dist_dir, macos_path

def make_dmg_on_desktop(dist_dir, name):
  dmg_path = os.path.join(os.environ['HOME'], 'Desktop', '%s.dmg' % (name,))
  naming_attempt = 0
  while os.path.exists(dmg_path):
    naming_attempt += 1
    dmg_path = os.path.join(os.environ['HOME'], 'Desktop',
                            "%s (%d).dmg" % (name, naming_attempt))
  run("hdiutil create -volname Bayeslite -format UDBZ -size 1g -srcfolder %s %s"
      % (shellquote(dist_dir), shellquote(dmg_path)))

def IsSKPValid(path_to_skp, path_to_skpinfo):
  """Calls the skpinfo binary to see if the specified SKP is valid."""
  skp_info_cmd = [path_to_skpinfo, '-i', path_to_skp]
  try:
    shell_utils.run(skp_info_cmd)
    return True
  except shell_utils.CommandFailedException:
    # Mark SKP as invalid if the skpinfo command gives a non 0 ret code.
    return False

def GotADB(adb):
  """ Returns True iff ADB exists at the given location.

  adb: string; possible path to the ADB executable.
  """
  try:
    shell_utils.run([adb, 'version'], echo=False)
    return True
  except Exception:
    return False

 def __exit__(self, exc_type, _value, _traceback):
   if self._upload:
     # Only upload if no error occurred.
     try:
       if exc_type is None:
         self.commit_and_upload(use_commit_queue=self._commit_queue)
     finally:
       shell_utils.run([GIT, 'checkout', 'master'])
       if self._delete_when_finished:
         shell_utils.run([GIT, 'branch', '-D', self._branch_name])

def make_venv_dir(venv_dir, options):
    parent = os.path.dirname(venv_dir)
    if parent and not os.path.exists(parent):
        os.mkdir(parent, 0755)
    cmd = "virtualenv"
    if options.python:
        assert os.path.exists(options.python)
        cmd += " --python=%s" % (shellquote(options.python))
    cmd += " " + shellquote(venv_dir)
    run(cmd, stdout=options.stdout)

def build_dmg():
  needed = ['osx/bayeslite/build_dmg.py',
            'src/shell_utils.py',
            'src/build_venv.py']
  run("scp %s [email protected]%s:" % (" ".join(needed), HOST))
  build_run('PATH="%s:$PATH" python build_dmg.py -v HEAD' % (HPATH,))
  run("scp [email protected]%s:Desktop/Bayeslite*.dmg %s" % (HOST, SCRATCH))
  name = build_outputof("cd Desktop && ls -t Bayeslite*.dmg | tail -1").strip()
  echo("NAME:", name)
  build_run("/bin/rm -f Desktop/Bayeslite*.dmg")
  return name

 def __enter__(self):
   shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
   shell_utils.run([GIT, 'checkout', 'master'])
   if self._branch_name in shell_utils.run([GIT, 'branch']):
     shell_utils.run([GIT, 'branch', '-D', self._branch_name])
   shell_utils.run([GIT, 'checkout', '-b', self._branch_name,
                    '-t', 'origin/master'])
   return self

def basic_sanity_check(venv_dir):
  test_dir = tempfile.mkdtemp('bayeslite-test')
  try:
    getoptfile = 'from bdbcontrib.population import OPTFILE; print OPTFILE;'
    optfilename = venv_outputof(venv_dir, "python -c '%s'" % (getoptfile,))
    with open(os.path.join(test_dir, optfilename), "w") as optfile:
      optfile.write("False\n")
    envs = "MPLBACKEND=pdf"
    venv_run(venv_dir,
             "cd -- %s && %s bayesdb-demo --runipy" %
             (shellquote(test_dir), envs))
  finally:
    run("rm -rf -- %s" % (shellquote(test_dir),))

def make_venv_truly_relocatable(venv_dir):
  relocable = '''VIRTUAL_ENV=$(dirname -- "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" && pwd )")\n'''
  new_activate = tempfile.NamedTemporaryFile(delete=False)
  old_activate_path = os.path.join(venv_dir, "bin", "activate")
  with open(old_activate_path, "r") as old_activate:
    for line in old_activate:
      if line[:len("VIRTUAL_ENV=")] == "VIRTUAL_ENV=":
        new_activate.write(relocable)
      else:
        new_activate.write(line)
  new_activate.close()
  run("mv %s %s" %
      (shellquote(new_activate.name), shellquote(old_activate_path)))

  def test_git_executable(git):
    """Test the git executable.

    Args:
        git: git executable path.
    Returns:
        True if test is successful.
    """
    try:
      shell_utils.run([git, '--version'], echo=False)
      return True
    except (OSError,):
      return False

def basic_sanity_check(venv_dir):
  test_dir = tempfile.mkdtemp('bayeslite-test')
  try:
    venv_run(venv_dir,
             "cd -- %s && bayesdb-demo fetch" % (shellquote(test_dir),))
    venv_run(venv_dir,
             "cd -- %s && "
             "MPLBACKEND=pdf PYTHONPATH=%s runipy %s" %
             (shellquote(test_dir),
              shellquote(os.path.join(venv_dir,
                                      "lib/python2.7/site-packages")),
              "Bayeslite-v*/satellites/Satellites.ipynb"))
  finally:
    run("rm -rf -- %s" % (shellquote(test_dir),))

def IsMerge(commit):
  """Return True if the commit is a merge, False otherwise."""
  rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1',
                               '--no-merges'])
  last_non_merge = rev_parse.split('\n')[0]
  # Get full hash since that is what was returned by rev-parse.
  return FullHash(commit) != last_non_merge

def GetGitRepoPOSIXTimestamp():
  """Returns the POSIX timestamp for the current Skia commit as in int."""
  git_show_command = [GIT, 'show', '--format=%at', '-s']
  raw_timestamp = shell_utils.run(
      git_show_command, log_in_real_time=False, echo=False,
      print_timestamps=False)
  return int(raw_timestamp)

def GetCheckedOutHash():
  """ Determine what commit we actually got. If there are local modifications,
  raise an exception. """
  checkout_root, config_dict = _GetLocalConfig()

  # Get the checked-out commit hash for the first gclient solution.
  with misc.ChDir(os.path.join(checkout_root, config_dict[0]['name'])):
    # First, print out the remote from which we synced, just for debugging.
    cmd = [GIT, 'remote', '-v']
    try:
      shell_utils.run(cmd)
    except shell_utils.CommandFailedException as e:
      print e

    # "git rev-parse HEAD" returns the commit hash for HEAD.
    return shell_utils.run([GIT, 'rev-parse', 'HEAD'],
                           log_in_real_time=False).rstrip('\n')