def configure_python(self, source_roots, test_roots, lib_roots):
   self.py_sources.extend(SourceSet(get_buildroot(), root, None, False) for root in source_roots)
   self.py_sources.extend(SourceSet(get_buildroot(), root, None, True) for root in test_roots)
   for root in lib_roots:
     for path in os.listdir(os.path.join(get_buildroot(), root)):
       if os.path.isdir(os.path.join(get_buildroot(), root, path)) or path.endswith('.egg'):
         self.py_libs.append(SourceSet(get_buildroot(), root, path, False))

    def _resolve_paths(self, rel_base, paths):
        """Resolves paths relative to the given rel_base from the build root.

    For example:
      target: ~/workspace/src/java/com/twitter/common/base/BUILD
      rel_base: src/resources

    Resolves paths from:
      ~/workspace/src/resources/com/twitter/common/base
    """

        if not paths:
            return []

        def flatten_paths(*items):
            """Flattens one or more items into a list.

      If the item is iterable each of its items is flattened.  If an item is callable, it is called
      and the result is flattened.  Otherwise the atom is appended to the flattened list.  These
      rules are applied recursively such that the returned list will only contain non-iterable,
      non-callable atoms.
      """

            flat = []

            def flatmap(item):
                if isinstance(item, Compatibility.string):
                    flat.append(item)
                else:
                    try:
                        for i in iter(item):
                            flatmap(i)
                    except TypeError:
                        if callable(item):
                            flatmap(item())
                        else:
                            flat.append(item)

            for item in items:
                flatmap(item)

            return flat

        src_relpath = os.path.relpath(
            self.address.buildfile.parent_path, os.path.join(get_buildroot(), self.target_base)
        )

        resolve_basepath = os.path.join(get_buildroot(), rel_base, src_relpath)
        with pushd(resolve_basepath):
            return [os.path.normpath(os.path.join(src_relpath, path)) for path in flatten_paths(paths)]

    def __init__(self, basedir, *types):
        """
      :basedir The base directory to resolve sources relative to
      :types The target types to register :basedir: as a source root for
    """
        basepath = os.path.abspath(basedir or os.path.curdir)
        if get_buildroot() != os.path.commonprefix((basepath, get_buildroot())):
            raise ValueError(
                "The supplied basedir %s is not a sub-path of the project root %s" % (basepath, get_buildroot())
            )

        self.basedir = os.path.relpath(basepath, get_buildroot())
        self.types = types
        SourceRoot._register(self)

 def load(configpath=os.path.join(get_buildroot(), 'pants.ini'), defaults=None):
   """
     Loads a Config from the given path, by default the path to the pants.ini file in the current
     build root directory.  Any defaults supplied will act as if specified in the loaded config
     file's DEFAULT section.  The 'buildroot', invoking 'user' and invoking user's 'homedir' are
     automatically defaulted.
   """
   standard_defaults = dict(
     buildroot=get_buildroot(),
     homedir=os.path.expanduser('~'),
     user=getpass.getuser()
   )
   if defaults:
     standard_defaults.update(defaults)
   return Config(configpath, defaults=standard_defaults)

 def _owning_targets(self, file):
   for build_file in self._candidate_owners(file):
     is_build_file = (build_file.full_path == os.path.join(get_buildroot(), file))
     for address in Target.get_all_addresses(build_file):
       target = Target.get(address)
       if target and (is_build_file or (has_sources(target) and self._owns(target, file))):
         yield target

  def execute(self, targets):
    if not self.skip:
      args = []
      if self.tests:
        args.append('--classes')
        args.append(','.join(self.tests))
      else:
        tests = self.calculate_tests(targets)
        if tests:
          args.append('--specs-files=%s' % ','.join(tests))

      if args:
        if self.color:
          args.append('--color')

        classpath = profile_classpath(self.profile)
        classpath.extend(os.path.join(get_buildroot(), path)
                         for path in ('src/resources', 'tests/resources'))
        with self.context.state('classpath', []) as cp:
          classpath.extend(jar for conf, jar in cp if conf in self.confs)

        result = runjava(
          jvmargs=self.java_args,
          classpath=classpath,
          main='run' if self.tests else 'com.twitter.common.testing.ExplicitSpecsRunnerMain',
          args=args
        )
        if result != 0:
          raise TaskError()

  def create_target(category, target_name, target_index, targets):
    def name(name):
      return "%s-%s-%d" % (target_name, name, target_index)

    # TODO(John Sirois): JavaLibrary and ScalaLibrary can float here between src/ and tests/ - add
    # ant build support to allow the same treatment for JavaThriftLibrary and JavaProtobufLibrary
    # so that tests can house test IDL in tests/
    target_type, base = category
    def create():
      if target_type == JavaProtobufLibrary:
        return _aggregate(JavaProtobufLibrary, name('protobuf'), targets, buildflags=buildflags)
      elif target_type == JavaThriftLibrary:
        return _aggregate(JavaThriftLibrary, name('thrift'), targets, buildflags=buildflags)
      elif target_type == AnnotationProcessor:
        return _aggregate(AnnotationProcessor, name('apt'), targets)
      elif target_type == JavaLibrary:
        return _aggregate(JavaLibrary, name('java'), targets, deployjar, buildflags)
      elif target_type == ScalaLibrary:
        return _aggregate(ScalaLibrary, name('scala'), targets, deployjar, buildflags)
      elif target_type == JavaTests:
        return _aggregate(JavaTests, name('java-tests'), targets, buildflags=buildflags)
      elif target_type == ScalaTests:
        return _aggregate(ScalaTests, name('scala-tests'), targets, buildflags=buildflags)
      else:
        raise Exception("Cannot aggregate targets of type: %s" % target_type)
    return ParseContext(BuildFile(get_buildroot(), base, must_exist=False)).do_in_context(create)

  def _runjava_common(self, runjava, main, classpath=None, opts=None, args=None, jvmargs=None):
    cp = (self._classpath or []) + (classpath or [])
    if self._daemon:
      nailgun = self._get_nailgun_client()

      def call_nailgun(main_class, *args):
        if self.dry_run:
          print('********** NailgunClient dry run: %s %s' % (main_class, ' '.join(args)))
          return 0
        else:
          return nailgun(main_class, *args)

      try:
        if cp:
          call_nailgun('ng-cp', *[os.path.relpath(jar, get_buildroot()) for jar in cp])
        opts_args = []
        if opts:
          opts_args.extend(opts)
        if args:
          opts_args.extend(args)
        return call_nailgun(main, *opts_args)
      except NailgunError as e:
        self._ng_shutdown()
        raise e
    else:
      ret = runjava(main=main, classpath=cp, opts=opts, args=args, jvmargs=jvmargs,
                    dryrun=self.dry_run)
      if self.dry_run:
        print('********** Direct Java dry run: %s' % ret)
        return 0
      else:
        return ret

  def split(self, splits, catchall=False):
    buildroot = get_buildroot()
    src_to_split_idx = {}
    for i, split in enumerate(splits):
      for s in split:
        src_to_split_idx[s if os.path.isabs(s) else os.path.join(buildroot, s)] = i
    num_outputs = len(splits) + 1 if catchall else len(splits)
    catchall_idx = len(splits) if catchall else -1

    split_pcd_entries = []
    split_src_to_deps = []
    for _ in xrange(0, num_outputs):
      split_pcd_entries.append([])
      split_src_to_deps.append({})

    for pcd_entry in self.pcd_entries:
      split_idx = src_to_split_idx.get(pcd_entry[1], catchall_idx)
      if split_idx != -1:
        split_pcd_entries[split_idx].append(pcd_entry)
    for src, deps in self.src_to_deps.items():
      split_idx = src_to_split_idx.get(src, catchall_idx)
      if split_idx != -1:
        split_src_to_deps[split_idx][src] = deps

    return [JMakeAnalysis(x, y) for x, y in zip(split_pcd_entries, split_src_to_deps)]

def extract_target(java_targets, is_classpath):
  primary_target = InternalTarget.sort_targets(java_targets)[0]

  def create_target():
    internal_deps, jar_deps = _extract_target(java_targets, is_classpath)

    # TODO(John Sirois): make an empty source set work in ant/compile.xml
    sources = [ '__no_source__' ]

    all_deps = OrderedSet()
    all_deps.update(internal_deps)
    all_deps.update(jar_deps)

    if is_java(primary_target):
      return JavaLibrary('ide',
                         sources,
                         dependencies = all_deps,
                         is_meta = True)
    elif is_scala(primary_target):
      return ScalaLibrary('ide',
                          sources,
                          dependencies = all_deps,
                          is_meta = True)
    else:
      raise TypeError("Cannot generate IDE configuration for targets: %s" % java_targets)

  buildfile = BuildFile(get_buildroot(), primary_target.target_base, must_exist=False)
  return ParseContext(buildfile).do_in_context(create_target)

  def findclasses(self, targets):
    """
      Returns a mapping from a target to its source to classes mapping.
      For example:

      dependencies = Dependencies(outdir, depfile)
      mapping = dependencies.findclasses(targets)
      for target, src_to_classes in mapping.items():
        for source, classes in src_to_classes.items():
          print('source: %s produces classes: %s' % (
            os.path.join(target.target_base, source),
            [os.path.join(outdir, cls) for cls in classes]
          ))
    """

    sources = set()
    target_by_source = dict()
    for target in targets:
      for source in target.sources:
        src = os.path.normpath(os.path.join(target.target_base, source))
        target_by_source[src] = target
        sources.add(src)

    classes_by_target_by_source = defaultdict(lambda: defaultdict(set))
    if os.path.exists(self.depfile):
      with open(self.depfile, 'r') as deps:
        for dep in deps.readlines():
          src, cls = dep.strip().split('->')
          sourcefile = os.path.relpath(os.path.join(self.outputdir, src.strip()), get_buildroot())
          if sourcefile in sources:
            classfile = os.path.relpath(os.path.join(self.outputdir, cls.strip()), self.outputdir)
            target = target_by_source[sourcefile]
            relsrc = os.path.relpath(sourcefile, target.target_base)
            classes_by_target_by_source[target][relsrc].add(classfile)
    return classes_by_target_by_source

  def __init__(self, context, nailgun_task, color, bootstrap_utils):
    self.context = context
    self._nailgun_task = nailgun_task  # We run zinc on this task's behalf.
    self._color = color
    self._bootstrap_utils = bootstrap_utils

    self._pants_home = get_buildroot()

    # The target scala version.
    self._compile_bootstrap_key = 'scalac'
    compile_bootstrap_tools = context.config.getlist('scala-compile', 'compile-bootstrap-tools',
                                                     default=[':scala-compile-2.9.2'])
    self._bootstrap_utils.register_jvm_build_tools(self._compile_bootstrap_key, compile_bootstrap_tools)

    # The zinc version (and the scala version it needs, which may differ from the target version).
    self._zinc_bootstrap_key = 'zinc'
    zinc_bootstrap_tools = context.config.getlist('scala-compile', 'zinc-bootstrap-tools', default=[':zinc'])
    self._bootstrap_utils.register_jvm_build_tools(self._zinc_bootstrap_key, zinc_bootstrap_tools)

    # Compiler plugins.
    plugins_bootstrap_tools = context.config.getlist('scala-compile', 'scalac-plugin-bootstrap-tools',
                                                     default=[])
    if plugins_bootstrap_tools:
      self._plugins_bootstrap_key = 'plugins'
      self._bootstrap_utils.register_jvm_build_tools(self._plugins_bootstrap_key, plugins_bootstrap_tools)
    else:
      self._plugins_bootstrap_key = None

    self._main = context.config.get('scala-compile', 'main')
    self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')

    # For localizing/relativizing analysis files.
    self._java_home = context.java_home
    self._ivy_home = context.ivy_home

 def test_depeendees_type(self):
   self._set_up_mocks(PythonTests, ["%s/tests" % get_buildroot()])
   self.assert_console_output(
     'tests/d/BUILD:d',
     args=['--test-type=python_tests'],
     targets=[self.target('common/d')]
   )

    def find(target):
        """
      Finds the source root for the given target target.  If none is registered, the parent
      directory of the target's BUILD file is returned.
    """
        target_path = os.path.relpath(target.address.buildfile.parent_path, get_buildroot())

        def _find():
            for typ in target.__class__.mro():
                for root in SourceRoot._ROOTS.get(typ, ()):
                    if target_path.startswith(root):
                        return root

        # Try already registered roots
        root = _find()
        if root:
            return root

        # Fall back to searching the ancestor path for a root
        for buildfile in reversed(target.address.buildfile.ancestors()):
            if buildfile not in SourceRoot._SEARCHED:
                SourceRoot._SEARCHED.add(buildfile)
                ParseContext(buildfile).parse()
                root = _find()
                if root:
                    return root

        # Finally, resolve files relative to the BUILD file parent dir as the target base
        return target_path

  def make_build_properties(cls):
    pi = PythonInterpreter()
    base_info = {
      'class': pi.identity().interpreter,
      'version': pi.identity().version,
      'platform': get_platform(),
    }
    try:
      from twitter.pants import get_buildroot, get_scm
      buildroot = get_buildroot()
      scm = get_scm()

      now = localtime()
      revision = scm.commit_id
      tag = scm.tag_name or 'none'
      branchname = scm.branch_name or revision
      base_info.update({
        'date': strftime('%A %b %d, %Y', now),
        'time': strftime('%H:%M:%S', now),
        'timestamp': strftime('%m.%d.%Y %H:%M', now),
        'branch': branchname,
        'tag': tag,
        'sha': revision,
        'user': getpass.getuser(),
        'machine': socket.gethostname(),
        'path': buildroot
      })
    except ImportError:
      pass
    return base_info

def _run():
  root_dir = get_buildroot()
  version = get_version()

  if not os.path.exists(root_dir):
    _exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)

  if len(sys.argv) < 2 or (len(sys.argv) == 2 and sys.argv[1] in _HELP_ALIASES):
    _help(version, root_dir)

  command_class, command_args = _parse_command(root_dir, sys.argv[1:])

  parser = optparse.OptionParser(version = '%%prog %s' % version)
  RcFile.install_disable_rc_option(parser)
  parser.add_option(_LOG_EXIT_OPTION, action = 'store_true', dest = 'log_exit',
                    default = False, help = 'Log an exit message on success or failure')
  command = command_class(root_dir, parser, command_args)

  if command.serialized():
    def onwait(pid):
      print('Waiting on pants process %s to complete' % _process_info(pid), file=sys.stderr)
      return True
    runfile = os.path.join(root_dir, '.pants.run')
    lock = Lock.acquire(runfile, onwait=onwait)
  else:
    lock = Lock.unlocked()
  try:
    result = command.run(lock)
    _do_exit(result)
  finally:
    lock.release()

  def __init__(self, configparser, configpath):
    # Base Config
    self.configparser = configparser
    with open(configpath) as ini:
      self.configparser.readfp(ini, filename=configpath)
    self.file = configpath

    # Overrides
    # 
    # This feature allows a second configuration file which will override 
    # pants.ini to be specified.  The file is currently specified via an env
    # variable because the cmd line flags are parsed after config is loaded.
    # 
    # The main use of the extra file is to have different settings based on
    # the environment.  For example, the setting used to compile or locations
    # of caches might be different between a developer's local environment
    # and the environment used to build and publish artifacts (e.g. Jenkins)
    # 
    # The files cannot reference each other's values, so make sure each one is 
    # internally consistent
    self.overrides_path = os.environ.get('PANTS_CONFIG_OVERRIDE')
    self.overrides_parser = None
    if self.overrides_path is not None:
      self.overrides_path = os.path.join(get_buildroot(), self.overrides_path)
      self.overrides_parser = Config.create_parser()
      with open(self.overrides_path) as o_ini:
        self.overrides_parser.readfp(o_ini, filename=self.overrides_path)

  def configure_project(self, targets, checkstyle_suppression_files, debug_port,
                        scala_compiler_profile):

    jvm_targets = extract_jvm_targets(targets)
    if self.intransitive:
      jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
    project = Project(self.project_name,
                      self.python,
                      self.skip_java,
                      self.skip_scala,
                      get_buildroot(),
                      checkstyle_suppression_files,
                      debug_port,
                      jvm_targets,
                      not self.intransitive,
                      self.context.new_workunit)

    if self.python:
      python_source_paths = self.context.config.getlist('ide', 'python_source_paths', default=[])
      python_test_paths = self.context.config.getlist('ide', 'python_test_paths', default=[])
      python_lib_paths = self.context.config.getlist('ide', 'python_lib_paths', default=[])
      project.configure_python(python_source_paths, python_test_paths, python_lib_paths)

    extra_source_paths = self.context.config.getlist('ide', 'extra_jvm_source_paths', default=[])
    extra_test_paths = self.context.config.getlist('ide', 'extra_jvm_test_paths', default=[])
    all_targets = project.configure_jvm(
      scala_compiler_profile,
      extra_source_paths,
      extra_test_paths
    )
    return all_targets, project

  def __init__(self, context, classpath=None, workdir=None):
    Task.__init__(self, context)

    self._classpath = classpath
    self._nailgun_bootstrap_key = 'nailgun'
    nailgun_bootstrap_tools = context.config.getlist('nailgun', 'bootstrap-tools',
                                                     default=[':nailgun-server'])

    self._bootstrap_utils.register_jvm_build_tools(self._nailgun_bootstrap_key, nailgun_bootstrap_tools)

    self._ng_server_args = context.config.getlist('nailgun', 'args')
    self._daemon = context.options.nailgun_daemon

    workdir = workdir or context.config.get('nailgun', 'workdir')

    # Allows us to identify the nailgun process by its cmd-line.
    self._identifier_arg = '-Dpants.ng.identifier=%s' % os.path.relpath(workdir, get_buildroot())

    self._current_pidport = None

    self._ng_out = os.path.join(workdir, 'stdout')
    self._ng_err = os.path.join(workdir, 'stderr')

    # Prevent concurrency issues when starting up a nailgun.
    self._spawn_lock = threading.Lock()

  def create_binary(self, binary):
    import platform
    safe_mkdir(self.outdir)

    jarmap = self.context.products.get('jars')

    binary_jarname = '%s.jar' % binary.basename
    binaryjarpath = os.path.join(self.outdir, binary_jarname)
    self.context.log.info('creating %s' % os.path.relpath(binaryjarpath, get_buildroot()))

    with open_jar(binaryjarpath, 'w', compression=self.compression, allowZip64=self.zip64) as jar:
      def add_jars(target):
        generated = jarmap.get(target)
        if generated:
          for basedir, jars in generated.items():
            for internaljar in jars:
              self.dump(os.path.join(basedir, internaljar), jar)

      binary.walk(add_jars, is_internal)

      if self.deployjar:
        for basedir, externaljar in self.list_jar_dependencies(binary):
          self.dump(os.path.join(basedir, externaljar), jar)

      manifest = Manifest()
      manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
      manifest.addentry(
        Manifest.CREATED_BY,
        'python %s pants %s (Twitter, Inc.)' % (platform.python_version(), get_version())
      )
      main = binary.main or '*** java -jar not supported, please use -cp and pick a main ***'
      manifest.addentry(Manifest.MAIN_CLASS,  main)
      jar.writestr(Manifest.PATH, manifest.contents())

      jarmap.add(binary, self.outdir, [binary_jarname])