• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号

Java LocalizerEventType类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Java中org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType的典型用法代码示例。如果您正苦于以下问题:Java LocalizerEventType类的具体用法?Java LocalizerEventType怎么用?Java LocalizerEventType使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



LocalizerEventType类属于org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event包,在下文中一共展示了LocalizerEventType类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。

示例1: LocalizerEventMatcher

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
public LocalizerEventMatcher(ContainerId idRef, Credentials creds,
    LocalResourceVisibility vis, LocalizerEventType type) {
  this.vis = vis;
  this.type = type;
  this.creds = creds;
  this.idRef = idRef;
}
 
开发者ID:naver,项目名称:hadoop,代码行数:8,代码来源:TestLocalizedResource.java


示例2: serviceInit

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.validateConf(conf);
  this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher,
      true, conf, stateStore);
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

  try {
    lfs = getLocalFileContext(conf);
    lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));

    if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
      cleanUpLocalDirs(lfs, delService);
      initializeLocalDirs(lfs);
      initializeLogDirs(lfs);
    }
  } catch (Exception e) {
    throw new YarnRuntimeException(
      "Failed to initialize LocalizationService", e);
  }

  cacheTargetSize =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
  cacheCleanupPeriod =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
  localizationServerAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

  localizerTracker = createLocalizerTracker(conf);
  addService(localizerTracker);
  dispatcher.register(LocalizerEventType.class, localizerTracker);
  super.serviceInit(conf);
}
 
开发者ID:yncxcw,项目名称:big-c,代码行数:37,代码来源:ResourceLocalizationService.java


示例3: serviceInit

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.validateConf(conf);
  this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher,
      true, conf, stateStore, dirsHandler);
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

  try {
    lfs = getLocalFileContext(conf);
    lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));

    if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
      cleanUpLocalDirs(lfs, delService);
      initializeLocalDirs(lfs);
      initializeLogDirs(lfs);
    }
  } catch (Exception e) {
    throw new YarnRuntimeException(
      "Failed to initialize LocalizationService", e);
  }

  cacheTargetSize =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
  cacheCleanupPeriod =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
  localizationServerAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

  localizerTracker = createLocalizerTracker(conf);
  addService(localizerTracker);
  dispatcher.register(LocalizerEventType.class, localizerTracker);
  localDirsChangeListener = new DirsChangeListener() {
    @Override
    public void onDirsChanged() {
      checkAndInitializeLocalDirs();
    }
  };
  logDirsChangeListener = new DirsChangeListener() {
    @Override
    public void onDirsChanged() {
      initializeLogDirs(lfs);
    }
  };
  super.serviceInit(conf);
}
 
开发者ID:naver,项目名称:hadoop,代码行数:49,代码来源:ResourceLocalizationService.java


示例4: testConsistency

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService(), null);

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:71,代码来源:TestLocalResourcesTrackerImpl.java


示例5: testStateStoreSuccessfulLocalization

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  DeletionService mockDelService = mock(DeletionService.class);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    // Simulate lr1 getting localized
    ResourceLocalizedEvent rle1 =
        new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
    tracker.handle(rle1);
    dispatcher.await();

    ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
        ArgumentCaptor.forClass(LocalizedResourceProto.class);
    verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
        localizedProtoCaptor.capture());
    LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
    Assert.assertEquals(lr1, new LocalResourceRequest(
        new LocalResourcePBImpl(localizedProto.getResource())));
    Assert.assertEquals(localizedPath1.toString(),
        localizedProto.getLocalPath());
    LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
    Assert.assertNotNull(localizedRsrc1);

    // simulate release and retention processing
    tracker.handle(new ResourceReleaseEvent(lr1, cId1));
    dispatcher.await();
    boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);

    Assert.assertTrue(removeResult);
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:82,代码来源:TestLocalResourcesTrackerImpl.java


示例6: testStateStoreFailedLocalization

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    ResourceFailedLocalizationEvent rfe1 =
        new ResourceFailedLocalizationEvent(
            lr1, new Exception("Test").toString());
    tracker.handle(rfe1);
    dispatcher.await();
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:62,代码来源:TestLocalResourcesTrackerImpl.java


示例7: testRecoveredResource

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp/localdir");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    Assert.assertNull(tracker.getLocalizedResource(lr1));
    final long localizedId1 = 52;
    Path hierarchicalPath1 = new Path(localDir,
        Long.toString(localizedId1));
    Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr1));

    // verify new paths reflect recovery of previous resources
    LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
    ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2,
        LocalResourceVisibility.APPLICATION, lc2);
    tracker.handle(reqEvent2);
    dispatcher.await();
    Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir,
        null);
    long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
    Assert.assertEquals(localizedId1 + 1, localizedId2);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:53,代码来源:TestLocalResourcesTrackerImpl.java


示例8: testGetPathForLocalization

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testGetPathForLocalization() throws Exception {
  FileContext lfs = FileContext.getLocalFSFileContext();
  Path base_path = new Path("target",
      TestLocalResourcesTrackerImpl.class.getSimpleName());
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);
  DeletionService delService = mock(DeletionService.class);
  try {
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, localrsrc, true, conf, stateStore, null);
    Path conflictPath = new Path(base_path, "10");
    Path qualifiedConflictPath = lfs.makeQualified(conflictPath);
    lfs.mkdir(qualifiedConflictPath, null, true);
    Path rPath = tracker.getPathForLocalization(req1, base_path,
        delService);
    Assert.assertFalse(lfs.util().exists(rPath));
    verify(delService, times(1)).delete(eq(user), eq(conflictPath));
  } finally {
    lfs.delete(base_path, true);
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:43,代码来源:TestLocalResourcesTrackerImpl.java


示例9: testResourcePresentInGoodDir

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
    List<String> goodDirs = new ArrayList<String>();
    // /tmp/somedir2 is bad
    goodDirs.add("/tmp/somedir1/");
    goodDirs.add("/tmp/somedir2");
    Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
    Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
    LocalResourcesTrackerImpl tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            true , conf, new NMNullStateStoreService(), dirsHandler);
    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
    // Localize R1 for C1
    tracker.handle(req11Event);
    // Localize R2 for C1
    tracker.handle(req21Event);
    dispatcher.await();
    // Localize resource1
    Path p1 = tracker.getPathForLocalization(req1,
        new Path("/tmp/somedir1"), null);
    Path p2 = tracker.getPathForLocalization(req2,
        new Path("/tmp/somedir2"), null);
    ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
    tracker.handle(rle1);
    ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
    tracker.handle(rle2);
    dispatcher.await();
    // Remove somedir2 from gooddirs
    Assert.assertTrue(tracker.checkLocalResource(lr2));
    goodDirs.remove(1);
    Assert.assertFalse(tracker.checkLocalResource(lr2));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:67,代码来源:TestLocalResourcesTrackerImpl.java


示例10: testReleaseWhileDownloading

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testReleaseWhileDownloading() throws Exception {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc = new LocalizerContext(user, cId, null);

    LocalResourceRequest req =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr = createLocalizedResource(req, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req, lr);
    LocalResourcesTracker tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            false, conf, new NMNullStateStoreService(), null);

    // request the resource
    ResourceEvent reqEvent =
        new ResourceRequestEvent(req, LocalResourceVisibility.PUBLIC, lc);
    tracker.handle(reqEvent);

    // release the resource
    ResourceEvent relEvent = new ResourceReleaseEvent(req, cId);
    tracker.handle(relEvent);

    // download completing after release
    ResourceLocalizedEvent rle =
        new ResourceLocalizedEvent(req, new Path("file:///tmp/r1"), 1);
    tracker.handle(rle);

    dispatcher.await();
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:aliyun-beta,项目名称:aliyun-oss-hadoop-fs,代码行数:50,代码来源:TestLocalResourcesTrackerImpl.java


示例11: testConsistency

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService());

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:yncxcw,项目名称:big-c,代码行数:71,代码来源:TestLocalResourcesTrackerImpl.java


示例12: testStateStoreSuccessfulLocalization

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  DeletionService mockDelService = mock(DeletionService.class);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    // Simulate lr1 getting localized
    ResourceLocalizedEvent rle1 =
        new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
    tracker.handle(rle1);
    dispatcher.await();

    ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
        ArgumentCaptor.forClass(LocalizedResourceProto.class);
    verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
        localizedProtoCaptor.capture());
    LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
    Assert.assertEquals(lr1, new LocalResourceRequest(
        new LocalResourcePBImpl(localizedProto.getResource())));
    Assert.assertEquals(localizedPath1.toString(),
        localizedProto.getLocalPath());
    LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
    Assert.assertNotNull(localizedRsrc1);

    // simulate release and retention processing
    tracker.handle(new ResourceReleaseEvent(lr1, cId1));
    dispatcher.await();
    boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);

    Assert.assertTrue(removeResult);
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:yncxcw,项目名称:big-c,代码行数:81,代码来源:TestLocalResourcesTrackerImpl.java


示例13: testStateStoreFailedLocalization

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    ResourceFailedLocalizationEvent rfe1 =
        new ResourceFailedLocalizationEvent(
            lr1, new Exception("Test").toString());
    tracker.handle(rfe1);
    dispatcher.await();
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:yncxcw,项目名称:big-c,代码行数:61,代码来源:TestLocalResourcesTrackerImpl.java


示例14: testRecoveredResource

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp/localdir");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    Assert.assertNull(tracker.getLocalizedResource(lr1));
    final long localizedId1 = 52;
    Path hierarchicalPath1 = new Path(localDir,
        Long.toString(localizedId1));
    Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr1));

    // verify new paths reflect recovery of previous resources
    LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
    ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2,
        LocalResourceVisibility.APPLICATION, lc2);
    tracker.handle(reqEvent2);
    dispatcher.await();
    Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
    long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
    Assert.assertEquals(localizedId1 + 1, localizedId2);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:yncxcw,项目名称:big-c,代码行数:52,代码来源:TestLocalResourcesTrackerImpl.java


示例15: serviceInit

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.validateConf(conf);
  this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher,
      true, conf, stateStore);
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

  try {
    lfs = getLocalFileContext(conf);
    lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));

    if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
      cleanUpLocalDirs(lfs, delService);
      initializeLocalDirs(lfs);
      initializeLogDirs(lfs);
    }
  } catch (Exception e) {
    throw new YarnRuntimeException(
      "Failed to initialize LocalizationService", e);
  }

  cacheTargetSize =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
  cacheCleanupPeriod =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
  localizationServerAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

  localizerTracker = createLocalizerTracker(conf);
  addService(localizerTracker);
  dispatcher.register(LocalizerEventType.class, localizerTracker);
  localDirsChangeListener = new DirsChangeListener() {
    @Override
    public void onDirsChanged() {
      checkAndInitializeLocalDirs();
    }
  };
  logDirsChangeListener = new DirsChangeListener() {
    @Override
    public void onDirsChanged() {
      initializeLogDirs(lfs);
    }
  };
  super.serviceInit(conf);
}
 
开发者ID:Nextzero,项目名称:hadoop-2.6.0-cdh5.4.3,代码行数:49,代码来源:ResourceLocalizationService.java


示例16: serviceInit

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.validateConf(conf);
  this.publicRsrc =
      new LocalResourcesTrackerImpl(null, dispatcher, true, conf);
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

  try {
    FileContext lfs = getLocalFileContext(conf);
    lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));

    cleanUpLocalDir(lfs,delService);

    List<String> localDirs = dirsHandler.getLocalDirs();
    for (String localDir : localDirs) {
      // $local/usercache
      Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE);
      lfs.mkdir(userDir, null, true);
      // $local/filecache
      Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE);
      lfs.mkdir(fileDir, null, true);
      // $local/nmPrivate
      Path sysDir = new Path(localDir, NM_PRIVATE_DIR);
      lfs.mkdir(sysDir, NM_PRIVATE_PERM, true);
    }

    List<String> logDirs = dirsHandler.getLogDirs();
    for (String logDir : logDirs) {
      lfs.mkdir(new Path(logDir), null, true);
    }
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to initialize LocalizationService", e);
  }

  cacheTargetSize =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
  cacheCleanupPeriod =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
  localizationServerAddress = conf.getSocketAddr(
      YarnConfiguration.NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

  localizerTracker = createLocalizerTracker(conf);
  addService(localizerTracker);
  dispatcher.register(LocalizerEventType.class, localizerTracker);
  super.serviceInit(conf);
}
 
开发者ID:ict-carch,项目名称:hadoop-plus,代码行数:49,代码来源:ResourceLocalizationService.java


示例17: testConsistency

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; //导入依赖的package包/类
@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        dispatcher, localrsrc, false, conf);

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    d 

鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Java DeploymentBuilder类代码示例发布时间:2022-05-22
下一篇:
Java Logger类代码示例发布时间:2022-05-22
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap