import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private void waitForContainerToFinishOnNM(ContainerId containerId) {
  Context nmContet = yarnCluster.getNodeManager(0).getNMContext();
  int interval = 4 * 60; // Max time for container token to expire.
  Assert.assertNotNull(nmContet.getContainers().containsKey(containerId));
  while ((interval-- > 0)
      && !nmContet.getContainers().get(containerId)
        .cloneAndGetContainerStatus().getState()
        .equals(ContainerState.COMPLETE)) {
    try {
      LOG.info("Waiting for " + containerId + " to complete.");
      Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
  }
  // Normally, Containers will be removed from NM context after they are
  // explicitly acked by RM. Now, manually remove it for testing.
  yarnCluster.getNodeManager(0).getNodeStatusUpdater()
    .addCompletedContainer(containerId);
  nmContet.getContainers().remove(containerId);
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
/**
 * Finds the local directories that logs for the given container are stored
 * on.
 */
public static List<File> getContainerLogDirs(ContainerId containerId,
    String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);

  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  // It is not required to have null check for container ( container == null )
  // and throw back exception.Because when container is completed, NodeManager
  // remove container information from its NMContext.Configuring log
  // aggregation to false, container log view request is forwarded to NM. NM
  // does not have completed container information,but still NM serve request for
  // reading container logs. 
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  return getContainerLogDirs(containerId, context.getLocalDirsHandler());
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
/**
 * Finds the log file with the given filename for the given container.
 */
public static File getContainerLogFile(ContainerId containerId,
    String fileName, String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);
  
  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  try {
    LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();
    String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(
        application.getAppId().toString(), containerId.toString());
    Path logPath = dirsHandler.getLogPathToRead(
        relativeContainerLogDir + Path.SEPARATOR + fileName);
    URI logPathURI = new File(logPath.toString()).toURI();
    File logFile = new File(logPathURI.getPath());
    return logFile;
  } catch (IOException e) {
    LOG.warn("Failed to find log file", e);
    throw new NotFoundException("Cannot find this log on the local disk.");
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private static void checkAccess(String remoteUser, Application application,
    Context context) throws YarnException {
  UserGroupInformation callerUGI = null;
  if (remoteUser != null) {
    callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
  }
  if (callerUGI != null
      && !context.getApplicationACLsManager().checkAccess(callerUGI,
          ApplicationAccessType.VIEW_APP, application.getUser(),
          application.getAppId())) {
    throw new YarnException(
        "User [" + remoteUser
            + "] is not authorized to view the logs for application "
            + application.getAppId());
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
    Context context) throws IOException {
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationId applicationId = containerId.getApplicationAttemptId()
      .getApplicationId();
  String user = context.getApplications().get(
      applicationId).getUser();
  
  try {
    return SecureIOUtils.openForRead(logFile, user, null);
  } catch (IOException e) {
    if (e.getMessage().contains(
      "did not match expected owner '" + user
          + "'")) {
      LOG.error(
          "Exception reading log file " + logFile.getAbsolutePath(), e);
      throw new IOException("Exception reading log file. Application submitted by '"
          + user
          + "' doesn't own requested log file : "
          + logFile.getName(), e);
    } else {
      throw new IOException("Exception reading log file. It might be because log "
          + "file was aggregated : " + logFile.getName(), e);
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
@Override
protected void serviceStart() throws Exception {
  String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(),
                        YarnConfiguration.NM_BIND_HOST,
                        WebAppUtils.getNMWebAppURLWithoutScheme(getConfig()));
  
  LOG.info("Instantiating NMWebApp at " + bindAddress);
  try {
    this.webApp =
        WebApps
          .$for("node", Context.class, this.nmContext, "ws")
          .at(bindAddress)
          .with(getConfig())
          .withHttpSpnegoPrincipalKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
          .withHttpSpnegoKeytabKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
          .start(this.nmWebApp);
    this.port = this.webApp.httpServer().getConnectorAddress(0).getPort();
  } catch (Exception e) {
    String msg = "NMWebapps failed to start.";
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
  super.serviceStart();
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public ResourceLocalizationService(Dispatcher dispatcher,
    ContainerExecutor exec, DeletionService delService,
    LocalDirsHandlerService dirsHandler, Context context) {

  super(ResourceLocalizationService.class.getName());
  this.exec = exec;
  this.dispatcher = dispatcher;
  this.delService = delService;
  this.dirsHandler = dirsHandler;

  this.cacheCleanup = new ScheduledThreadPoolExecutor(1,
      new ThreadFactoryBuilder()
        .setNameFormat("ResourceLocalizationService Cache Cleanup")
        .build());
  this.stateStore = context.getNMStateStore();
  this.nmContext = context;
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);

  this.olr = new OwnLocalResources();
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private List<String> getContainerLogFiles(ContainerId id, String remoteUser,
    Context nmContext) {
  List<String> logFiles = new ArrayList<>();
  try {
    List<File> logDirs =
        ContainerLogsUtils.getContainerLogDirs(id, remoteUser, nmContext);
    for (File containerLogsDir : logDirs) {
      File[] logs = containerLogsDir.listFiles();
      if (logs != null) {
        for (File log : logs) {
          if (log.isFile()) {
            logFiles.add(log.getName());
          }
        }
      }
    }
  } catch (Exception ye) {
    return logFiles;
  }
  return logFiles;
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public NodeInfo(final Context context, final ResourceView resourceView) {

    this.id = context.getNodeId().toString();
    this.nodeHostName = context.getNodeId().getHost();
    this.totalVmemAllocatedContainersMB = resourceView
        .getVmemAllocatedForContainers() / BYTES_IN_MB;
    this.vmemCheckEnabled = resourceView.isVmemCheckEnabled();
    this.totalPmemAllocatedContainersMB = resourceView
        .getPmemAllocatedForContainers() / BYTES_IN_MB;
    this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
    this.totalVCoresAllocatedContainers = resourceView
        .getVCoresAllocatedForContainers();
    this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
    this.lastNodeUpdateTime = context.getNodeHealthStatus()
        .getLastHealthReportTime();

    this.healthReport = context.getNodeHealthStatus().getHealthReport();

    this.nodeManagerVersion = YarnVersionInfo.getVersion();
    this.nodeManagerBuildVersion = YarnVersionInfo.getBuildVersion();
    this.nodeManagerVersionBuiltOn = YarnVersionInfo.getDate();
    this.hadoopVersion = VersionInfo.getVersion();
    this.hadoopBuildVersion = VersionInfo.getBuildVersion();
    this.hadoopVersionBuiltOn = VersionInfo.getDate();
    this.nmStartupTime = NodeManager.getNMStartupTime();
  }
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
@GET
@Path("/containers")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context
    HttpServletRequest hsr) {
  init();
  ContainersInfo allContainers = new ContainersInfo();
  for (Entry<ContainerId, Container> entry : this.nmContext.getContainers()
      .entrySet()) {
    if (entry.getValue() == null) {
      // just skip it
      continue;
    }
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
        uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
    allContainers.add(info);
  }
  return allContainers;
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context
    HttpServletRequest hsr, @PathParam("containerid") String id) {
  ContainerId containerId = null;
  init();
  try {
    containerId = ConverterUtils.toContainerId(id);
  } catch (Exception e) {
    throw new BadRequestException("invalid container id, " + id);
  }

  Container container = nmContext.getContainers().get(containerId);
  if (container == null) {
    throw new NotFoundException("container with id, " + id + ", not found");
  }
  return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
      .toString(), webapp.name(), hsr.getRemoteUser());

}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private ContainerManagerImpl createContainerManager(Context context,
    DeletionService delSrvc) {
  return new ContainerManagerImpl(context, exec, delSrvc,
      mock(NodeStatusUpdater.class), metrics, dirsHandler) {
    @Override
    public void
    setBlockNewContainerRequests(boolean blockNewContainerRequests) {
      // do nothing
    }
    @Override
    protected void authorizeGetAndStopContainerRequest(
        ContainerId containerId, Container container,
        boolean stopRequest, NMTokenIdentifier identifier)
        throws YarnException {
      if(container == null || container.getUser().equals("Fail")){
        throw new YarnException("Reject this container");
      }
    }
  };
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
private ContainerStatus getContainerStatus(
    Context context, final ContainerManagerImpl cm, ContainerId cid)
    throws  Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  List<ContainerId> containerIds = new ArrayList<>();
  containerIds.add(cid);
  final GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  return user.doAs(
      new PrivilegedExceptionAction<ContainerStatus>() {
        @Override
        public ContainerStatus run() throws Exception {
          return cm.getContainerStatuses(gcsRequest)
              .getContainerStatuses().get(0);
        }
      });
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
@Before
public void setup() {
  executor = new MockExecutor();
  dispatcher = new AsyncDispatcher();
  context = Mockito.mock(Context.class);
  Mockito.doReturn(new ConcurrentSkipListMap<ContainerId, Container>())
      .when(context).getContainers();
  conf = new Configuration();
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
      MockResourceCalculatorPlugin.class.getCanonicalName());
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
      MockResourceCalculatorProcessTree.class.getCanonicalName());
  dispatcher.init(conf);
  dispatcher.start();
  containerEventHandler = new MockContainerEventHandler();
  dispatcher.register(ContainerEventType.class, containerEventHandler);
}
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public NodeInfo(final Context context, final ResourceView resourceView) {

    this.id = context.getNodeId().toString();
    this.nodeHostName = context.getNodeId().getHost();
    this.totalVmemAllocatedContainersMB = resourceView
        .getVmemAllocatedForContainers() / BYTES_IN_MB;
    this.vmemCheckEnabled = resourceView.isVmemCheckEnabled();
    this.totalPmemAllocatedContainersMB = resourceView
        .getPmemAllocatedForContainers() / BYTES_IN_MB;
    this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
    this.totalVCoresAllocatedContainers = resourceView
        .getVCoresAllocatedForContainers();
    this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
    this.lastNodeUpdateTime = context.getNodeHealthStatus()
        .getLastHealthReportTime();

    this.healthReport = context.getNodeHealthStatus().getHealthReport();

    this.nodeManagerVersion = YarnVersionInfo.getVersion();
    this.nodeManagerBuildVersion = YarnVersionInfo.getBuildVersion();
    this.nodeManagerVersionBuiltOn = YarnVersionInfo.getDate();
    this.hadoopVersion = VersionInfo.getVersion();
    this.hadoopBuildVersion = VersionInfo.getBuildVersion();
    this.hadoopVersionBuiltOn = VersionInfo.getDate();
  }
 

import org.apache.hadoop.yarn.server.nodemanager.Context; //导入依赖的package包/类
public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,Set<Integer> cpuCores) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.currentResource = resource;
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();
  this.cpuCores  = cpuCores;
  this.context = context;

  stateMachine = stateMachineFactory.make(this);
}