TEST_F(DockerTest, ROOT_DOCKER_CheckPortResource)
{
  const string containerName = NAME_PREFIX + "-port-resource-test";
  Owned<Docker> docker(Docker::create(tests::flags.docker,
                                     tests::flags.docker_socket,
                                     false).get());

  // Make sure the container is removed.
  Future<Nothing> remove = docker->rm(containerName, true);

  ASSERT_TRUE(process::internal::await(remove, Seconds(10)));

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::DOCKER);

  ContainerInfo::DockerInfo dockerInfo;
  dockerInfo.set_image("busybox");
  dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);

  ContainerInfo::DockerInfo::PortMapping portMapping;
  portMapping.set_host_port(10000);
  portMapping.set_container_port(80);

  dockerInfo.add_port_mappings()->CopyFrom(portMapping);
  containerInfo.mutable_docker()->CopyFrom(dockerInfo);

  CommandInfo commandInfo;
  commandInfo.set_shell(false);
  commandInfo.set_value("true");

  Resources resources =
    Resources::parse("ports:[9998-9999];ports:[10001-11000]").get();

  Future<Nothing> run = docker->run(
      containerInfo,
      commandInfo,
      containerName,
      "dir",
      "/mnt/mesos/sandbox",
      resources);

  // Port should be out side of the provided ranges.
  AWAIT_EXPECT_FAILED(run);

  resources = Resources::parse("ports:[9998-9999];ports:[10000-11000]").get();

  Try<string> directory = environment->mkdtemp();
  CHECK_SOME(directory) << "Failed to create temporary directory";

  run = docker->run(
      containerInfo,
      commandInfo,
      containerName,
      directory.get(),
      "/mnt/mesos/sandbox",
      resources);

  AWAIT_READY(run);
}

	TaskInfo buildTask (string hostname, string id, const SlaveID& slave)  {
		hostProfile profile = hostList[hostname];
	// Define the Docker container.
		/*  Since there is no "executor" to manage the tasks, the
			container will be built and attached directly into the task below */
		ContainerInfo container;
		container.set_type(container.DOCKER);
		ContainerInfo::DockerInfo docker;
		docker.set_image(DOCKER_IMAGE);
		container.mutable_docker()->MergeFrom(docker);

		// Mount local volume inside Container
		Volume * volume = container.add_volumes();
		volume->set_container_path("/mnt");
		volume->set_host_path("/local/mesos");
		volume->set_mode(Volume_Mode_RW);

		// Define the task
		TaskInfo task;
		task.set_name("K3-" + k3binary);
		task.mutable_task_id()->set_value(id);
		task.mutable_slave_id()->MergeFrom(slave);
		task.mutable_container()->MergeFrom(container);
		//task.set_data(stringify(localTasks));

		// Define include files for the command
		CommandInfo command;

		CommandInfo_URI * k3_bin = command.add_uris();
		k3_bin->set_value(fileServer + "/" + k3binary);
		k3_bin->set_executable(true);
		k3_bin->set_extract(false);

//		CommandInfo_URI * k3_args = command.add_uris();
//		k3_args->set_value(runpath + "/k3input.yaml");
		
//		command.set_value("$MESOS_SANDBOX/" + k3binary + " -l INFO -p " +
//				"$MESOS_SANDBOX/k3input.yaml");
		task.mutable_command()->MergeFrom(command);

		// Option A for doing resources management (see scheduler for option B)
		Resource* resource;

		resource = task.add_resources();
		resource->set_name("cpus");
		resource->set_type(Value::SCALAR);
		resource->mutable_scalar()->set_value(profile.cpu);

		resource = task.add_resources();
		resource->set_name("mem");
		resource->set_type(Value::SCALAR);
		resource->mutable_scalar()->set_value(profile.mem);
		
		return task;
	}

  virtual void resourceOffers(
      SchedulerDriver* driver,
      const vector<Offer>& offers)
  {
    static const Try<Resources> TASK_RESOURCES = Resources::parse(resources);

    if (TASK_RESOURCES.isError()) {
      cerr << "Failed to parse resources '" << resources
           << "': " << TASK_RESOURCES.error() << endl;
      driver->abort();
      return;
    }

    foreach (const Offer& offer, offers) {
      if (!launched &&
          Resources(offer.resources()).contains(TASK_RESOURCES.get())) {
        TaskInfo task;
        task.set_name(name);
        task.mutable_task_id()->set_value(name);
        task.mutable_slave_id()->MergeFrom(offer.slave_id());
        task.mutable_resources()->CopyFrom(TASK_RESOURCES.get());
        task.mutable_command()->set_value(command);
        if (uri.isSome()) {
          task.mutable_command()->add_uris()->set_value(uri.get());
        }

        if (dockerImage.isSome()) {
          ContainerInfo containerInfo;
          containerInfo.set_type(ContainerInfo::DOCKER);

          ContainerInfo::DockerInfo dockerInfo;
          dockerInfo.set_image(dockerImage.get());

          containerInfo.mutable_docker()->CopyFrom(dockerInfo);
          task.mutable_container()->CopyFrom(containerInfo);
        }

        vector<TaskInfo> tasks;
        tasks.push_back(task);

        driver->launchTasks(offer.id(), tasks);
        cout << "task " << name << " submitted to slave "
             << offer.slave_id() << endl;

        launched = true;
      } else {
        driver->declineOffer(offer.id());
      }
    }
  }

// This test verifies mounting in an absolute path when running a
// docker container works.
TEST_F(DockerTest, ROOT_DOCKER_MountAbsolute)
{
  Owned<Docker> docker = Docker::create(
      tests::flags.docker,
      tests::flags.docker_socket,
      false).get();

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::DOCKER);

  Try<string> directory = environment->mkdtemp();
  CHECK_SOME(directory) << "Failed to create temporary directory";

  const string testFile = path::join(directory.get(), "test_file");
  EXPECT_SOME(os::write(testFile, "data"));

  Volume* volume = containerInfo.add_volumes();
  volume->set_host_path(testFile);
  volume->set_container_path("/tmp/test_file");
  volume->set_mode(Volume::RO);

  ContainerInfo::DockerInfo dockerInfo;
  dockerInfo.set_image("busybox");

  containerInfo.mutable_docker()->CopyFrom(dockerInfo);

  CommandInfo commandInfo;
  commandInfo.set_shell(true);
  commandInfo.set_value("ls /tmp/test_file");

  Future<Nothing> run = docker->run(
      containerInfo,
      commandInfo,
      NAME_PREFIX + "-mount-absolute-test",
      directory.get(),
      directory.get());

  AWAIT_READY(run);
}

bool operator == (
    const ContainerInfo::DockerInfo& left,
    const ContainerInfo::DockerInfo& right)
{
    // Order of port mappings is not important.
    if (left.port_mappings().size() != right.port_mappings().size()) {
        return false;
    }

    for (int i = 0; i < left.port_mappings().size(); i++) {
        bool found = false;
        for (int j = 0; j < right.port_mappings().size(); j++) {
            if (left.port_mappings().Get(i) == right.port_mappings().Get(j)) {
                found = true;
                break;
            }
        }
        if (!found) {
            return false;
        }
    }

    // Order of parameters is not important.
    if (left.parameters().size() != right.parameters().size()) {
        return false;
    }

    for (int i = 0; i < left.parameters().size(); i++) {
        bool found = false;
        for (int j = 0; j < right.parameters().size(); j++) {
            if (left.parameters().Get(i) == right.parameters().Get(j)) {
                found = true;
                break;
            }
        }
        if (!found) {
            return false;
        }
    }

    return left.image() == right.image() &&
           left.network() == right.network() &&
           left.privileged() == right.privileged() &&
           left.force_pull_image() == right.force_pull_image();
}

// Test that the prepare launch docker hook execute before launch
// a docker container. Test hook create a file "foo" in the sandbox
// directory. When the docker container launched, the sandbox directory
// is mounted to the docker container. We validate the hook by verifying
// the "foo" file exists in the docker container or not.
TEST_F(HookTest, ROOT_DOCKER_VerifySlavePreLaunchDockerHook)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  MockDocker* mockDocker =
    new MockDocker(tests::flags.docker, tests::flags.docker_socket);

  Shared<Docker> docker(mockDocker);

  slave::Flags flags = CreateSlaveFlags();

  Fetcher fetcher;

  Try<ContainerLogger*> logger =
    ContainerLogger::create(flags.container_logger);

  ASSERT_SOME(logger);

  MockDockerContainerizer containerizer(
      flags,
      &fetcher,
      Owned<ContainerLogger>(logger.get()),
      docker);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave =
    StartSlave(detector.get(), &containerizer, flags);
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_NE(0u, offers.get().size());

  const Offer& offer = offers.get()[0];

  SlaveID slaveId = offer.slave_id();

  TaskInfo task;
  task.set_name("");
  task.mutable_task_id()->set_value("1");
  task.mutable_slave_id()->CopyFrom(offer.slave_id());
  task.mutable_resources()->CopyFrom(offer.resources());

  CommandInfo command;
  command.set_value("test -f " + path::join(flags.sandbox_directory, "foo"));

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::DOCKER);

  // TODO(tnachen): Use local image to test if possible.
  ContainerInfo::DockerInfo dockerInfo;
  dockerInfo.set_image("alpine");
  containerInfo.mutable_docker()->CopyFrom(dockerInfo);

  task.mutable_command()->CopyFrom(command);
  task.mutable_container()->CopyFrom(containerInfo);

  vector<TaskInfo> tasks;
  tasks.push_back(task);

  Future<ContainerID> containerId;
  EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _))
    .WillOnce(DoAll(FutureArg<0>(&containerId),
                    Invoke(&containerizer,
                           &MockDockerContainerizer::_launch)));

  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished))
    .WillRepeatedly(DoDefault());

  driver.launchTasks(offers.get()[0].id(), tasks);

  AWAIT_READY_FOR(containerId, Seconds(60));
  AWAIT_READY_FOR(statusRunning, Seconds(60));
//.........这里部分代码省略.........

  virtual void resourceOffers(
      SchedulerDriver* driver,
      const vector<Offer>& offers)
  {
    static const Try<Resources> TASK_RESOURCES = Resources::parse(resources);

    if (TASK_RESOURCES.isError()) {
      cerr << "Failed to parse resources '" << resources
           << "': " << TASK_RESOURCES.error() << endl;
      driver->abort();
      return;
    }

    foreach (const Offer& offer, offers) {
      if (!launched &&
          Resources(offer.resources()).contains(TASK_RESOURCES.get())) {
        TaskInfo task;
        task.set_name(name);
        task.mutable_task_id()->set_value(name);
        task.mutable_slave_id()->MergeFrom(offer.slave_id());
        task.mutable_resources()->CopyFrom(TASK_RESOURCES.get());

        CommandInfo* commandInfo = task.mutable_command();
        commandInfo->set_value(command);
        if (environment.isSome()) {
          Environment* environment_ = commandInfo->mutable_environment();
          foreachpair (const std::string& name,
                       const std::string& value,
                       environment.get()) {
            Environment_Variable* environmentVariable =
              environment_->add_variables();
            environmentVariable->set_name(name);
            environmentVariable->set_value(value);
          }
        }

        if (uri.isSome()) {
          task.mutable_command()->add_uris()->set_value(uri.get());
        }

        if (dockerImage.isSome()) {
          ContainerInfo containerInfo;

          if (containerizer == "mesos") {
            containerInfo.set_type(ContainerInfo::MESOS);

            ContainerInfo::MesosInfo mesosInfo;

            Image mesosImage;
            mesosImage.set_type(Image::DOCKER);
            mesosImage.mutable_docker()->set_name(dockerImage.get());
            mesosInfo.mutable_image()->CopyFrom(mesosImage);

            containerInfo.mutable_mesos()->CopyFrom(mesosInfo);
          } else if (containerizer == "docker") {
            containerInfo.set_type(ContainerInfo::DOCKER);

            ContainerInfo::DockerInfo dockerInfo;
            dockerInfo.set_image(dockerImage.get());

            containerInfo.mutable_docker()->CopyFrom(dockerInfo);
          } else {
            cerr << "Unsupported containerizer: " << containerizer << endl;;

            driver->abort();

            return;
          }

          task.mutable_container()->CopyFrom(containerInfo);
        }

        vector<TaskInfo> tasks;
        tasks.push_back(task);

        driver->launchTasks(offer.id(), tasks);
        cout << "task " << name << " submitted to slave "
             << offer.slave_id() << endl;

        launched = true;
      } else {

  virtual void resourceOffers(SchedulerDriver* driver,
                              const vector<Offer>& offers)
  {
    cout << "." << flush;
    for (size_t i = 0; i < offers.size(); i++) {
      const Offer& offer = offers[i];

      // Lookup resources we care about.
      // TODO(benh): It would be nice to ultimately have some helper
      // functions for looking up resources.
      double cpus = 0;
      double mem = 0;

      for (int i = 0; i < offer.resources_size(); i++) {
        const Resource& resource = offer.resources(i);
        if (resource.name() == "cpus" &&
            resource.type() == Value::SCALAR) {
          cpus = resource.scalar().value();
        } else if (resource.name() == "mem" &&
                   resource.type() == Value::SCALAR) {
          mem = resource.scalar().value();
        }
      }

      // Launch tasks.
      vector<TaskInfo> tasks;
      while (tasksLaunched < totalTasks &&
             cpus >= CPUS_PER_TASK &&
             mem >= MEM_PER_TASK) {
        int taskId = tasksLaunched++;

        cout << "Starting task " << taskId << " on "
             << offer.hostname() << endl;

        TaskInfo task;
        task.set_name("Task " + lexical_cast<string>(taskId));
        task.mutable_task_id()->set_value(lexical_cast<string>(taskId));
        task.mutable_slave_id()->MergeFrom(offer.slave_id());
        task.mutable_command()->set_value("echo hello");

        // Use Docker to run the task.
        ContainerInfo containerInfo;
        containerInfo.set_type(ContainerInfo::DOCKER);

        ContainerInfo::DockerInfo dockerInfo;
        dockerInfo.set_image("busybox");

        containerInfo.mutable_docker()->CopyFrom(dockerInfo);
        task.mutable_container()->CopyFrom(containerInfo);

        Resource* resource;

        resource = task.add_resources();
        resource->set_name("cpus");
        resource->set_type(Value::SCALAR);
        resource->mutable_scalar()->set_value(CPUS_PER_TASK);

        resource = task.add_resources();
        resource->set_name("mem");
        resource->set_type(Value::SCALAR);
        resource->mutable_scalar()->set_value(MEM_PER_TASK);

        tasks.push_back(task);

        cpus -= CPUS_PER_TASK;
        mem -= MEM_PER_TASK;
      }

      driver->launchTasks(offer.id(), tasks);
    }
  }

// This test tests the functionality of the  docker's interfaces.
TEST(DockerTest, ROOT_DOCKER_interface)
{
  string containerName = "mesos-docker-test";
  Resources resources = Resources::parse("cpus:1;mem:512").get();
  Docker docker = Docker::create(tests::flags.docker, false).get();

  // Cleaning up the container first if it exists.
  Future<Nothing> status = docker.rm(containerName, true);
  ASSERT_TRUE(status.await(Seconds(10)));

  // Verify that we do not see the container.
  Future<list<Docker::Container> > containers = docker.ps(true, containerName);
  AWAIT_READY(containers);
  foreach (const Docker::Container& container, containers.get()) {
    EXPECT_NE("/" + containerName, container.name);
  }

  Try<string> directory = environment->mkdtemp();
  CHECK_SOME(directory) << "Failed to create temporary directory";

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::DOCKER);

  ContainerInfo::DockerInfo dockerInfo;
  dockerInfo.set_image("busybox");
  containerInfo.mutable_docker()->CopyFrom(dockerInfo);

  CommandInfo commandInfo;
  commandInfo.set_value("sleep 120");

  // Start the container.
  status = docker.run(
      containerInfo,
      commandInfo,
      containerName,
      directory.get(),
      "/mnt/mesos/sandbox",
      resources);

  AWAIT_READY(status);

  // Should be able to see the container now.
  containers = docker.ps();
  AWAIT_READY(containers);
  bool found = false;
  foreach (const Docker::Container& container, containers.get()) {
    if ("/" + containerName == container.name) {
      found = true;
      break;
    }
  }
  EXPECT_TRUE(found);

  Future<Docker::Container> container = docker.inspect(containerName);
  AWAIT_READY(container);

  // Test some fields of the container.
  EXPECT_NE("", container.get().id);
  EXPECT_EQ("/" + containerName, container.get().name);
  EXPECT_SOME(container.get().pid);

  // Kill the container.
  status = docker.kill(containerName);
  AWAIT_READY(status);

  // Now, the container should not appear in the result of ps().
  // But it should appear in the result of ps(true).
  containers = docker.ps();
  AWAIT_READY(containers);
  foreach (const Docker::Container& container, containers.get()) {
    EXPECT_NE("/" + containerName, container.name);
  }

  containers = docker.ps(true, containerName);
  AWAIT_READY(containers);
  found = false;
  foreach (const Docker::Container& container, containers.get()) {
    if ("/" + containerName == container.name) {
      found = true;
      break;
    }
  }
  EXPECT_TRUE(found);

  // Check the container's info, both id and name should remain the
  // same since we haven't removed it, but the pid should be none
  // since it's not running.
  container = docker.inspect(containerName);
  AWAIT_READY(container);

  EXPECT_NE("", container.get().id);
  EXPECT_EQ("/" + containerName, container.get().name);
  EXPECT_NONE(container.get().pid);

  // Remove the container.
  status = docker.rm(containerName);
  AWAIT_READY(status);

  // Should not be able to inspect the container.
//.........这里部分代码省略.........