本文整理汇总了Java中org.apache.hadoop.hbase.wal.WAL类的典型用法代码示例。如果您正苦于以下问题:Java WAL类的具体用法?Java WAL怎么用?Java WAL使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
WAL类属于org.apache.hadoop.hbase.wal包,在下文中一共展示了WAL类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: addWALEdits
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
String familyStr = Bytes.toString(family);
long txid = -1;
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
// uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
// about legacy coprocessors
txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
ee.currentTime(), mvcc), edit, true);
}
if (-1 != txid) {
wal.sync(txid);
}
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:20,代码来源:TestWALObserver.java
示例2: addWALEdits
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc)
throws IOException {
String familyStr = Bytes.toString(family);
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes,
ee.currentTime(), columnBytes));
wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,999, mvcc),
edit, true);
}
wal.sync();
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:17,代码来源:TestWALReplay.java
示例3: getWAL
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
@Override public WAL getWAL(HRegionInfo regionInfo) throws IOException {
WAL wal;
LogRoller roller = walRoller;
//_ROOT_ and hbase:meta regions have separate WAL.
if (regionInfo != null && regionInfo.isMetaTable()
&& regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
roller = ensureMetaWALRoller();
wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes());
} else if (regionInfo == null) {
wal = walFactory.getWAL(UNSPECIFIED_REGION);
} else {
wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes());
}
roller.addWAL(wal);
return wal;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:17,代码来源:HRegionServer.java
示例4: writeMarker
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private static long writeMarker(final WAL wal, final HTableDescriptor htd, final HRegionInfo hri,
final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync)
throws IOException {
// TODO: Pass in current time to use?
WALKey key =
new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc);
// Add it to the log but the false specifies that we don't need to add it to the memstore
long trx = MultiVersionConcurrencyControl.NONE;
try {
trx = wal.append(htd, hri, key, edit, false);
if (sync) wal.sync(trx);
} finally {
// If you get hung here, is it a real WAL or a mocked WAL? If the latter, you need to
// trip the latch that is inside in getWriteEntry up in your mock. See down in the append
// called from onEvent in FSHLog.
MultiVersionConcurrencyControl.WriteEntry we = key.getWriteEntry();
if (mvcc != null && we != null) mvcc.complete(we);
}
return trx;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:21,代码来源:WALUtil.java
示例5: createHRegion
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Convenience method creating new HRegions. Used by createTable. The {@link WAL} for the created
* region needs to be closed explicitly, if it is not null. Use {@link HRegion#getWAL()} to get
* access.
*
* @param info Info for region to create.
* @param rootDir Root directory for HBase instance
* @param tableDir table directory
* @param wal shared WAL
* @param initialize - true to initialize the region
* @param ignoreWAL - true to skip generate new wal if it is null, mostly for createTable
* @return new HRegion
* @throws IOException
*/
public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
final Path tableDir, final Configuration conf, final HTableDescriptor hTableDescriptor,
final WAL wal, final boolean initialize, final boolean ignoreWAL) throws IOException {
LOG.info("creating HRegion " + info.getTable().getNameAsString() + " HTD == " + hTableDescriptor
+ " RootDir = " + rootDir + " Table name == " + info.getTable().getNameAsString());
FileSystem fs = FileSystem.get(conf);
HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, info);
WAL effectiveWAL = wal;
if (wal == null && !ignoreWAL) {
// TODO HBASE-11983 There'll be no roller for this wal?
// The WAL subsystem will use the default rootDir rather than the passed
// in rootDir
// unless I pass along via the conf.
Configuration confForWAL = new Configuration(conf);
confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
effectiveWAL = (new WALFactory(confForWAL,
Collections.<WALActionsListener>singletonList(new MetricsWAL()),
"hregion-" + RandomStringUtils.randomNumeric(8))).getWAL(info.getEncodedNameAsBytes());
}
HRegion region =
HRegion.newHRegion(tableDir, effectiveWAL, fs, conf, info, hTableDescriptor, null);
if (initialize) region.initialize(null);
return region;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:39,代码来源:HRegion.java
示例6: warmupHRegion
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
public static void warmupHRegion(final HRegionInfo info, final HTableDescriptor htd,
final WAL wal, final Configuration conf, final RegionServerServices rsServices,
final CancelableProgressable reporter) throws IOException {
if (info == null) throw new NullPointerException("Passed region info is null");
if (LOG.isDebugEnabled()) {
LOG.debug("HRegion.Warming up region: " + info);
}
Path rootDir = FSUtils.getRootDir(conf);
Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
FileSystem fs = null;
if (rsServices != null) {
fs = rsServices.getFileSystem();
}
if (fs == null) {
fs = FileSystem.get(conf);
}
HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, null);
r.initializeWarmup(reporter);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:25,代码来源:HRegion.java
示例7: testReplayingFlushRequestRestoresReadsEnabledState
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Test the case where the secondary region replica is not in reads enabled state because it is
* waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH
* flush marker entry should restore the reads enabled status in the region and allow the reads
* to continue.
*/
@Test
public void testReplayingFlushRequestRestoresReadsEnabledState() throws IOException {
disableReads(secondaryRegion);
// Test case 1: Test that replaying CANNOT_FLUSH request marker assuming this came from
// triggered flush restores readsEnabled
primaryRegion.flushcache(true, true);
reader = createWALReaderForPrimary();
while (true) {
WAL.Entry entry = reader.next();
if (entry == null) {
break;
}
FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
if (flush != null) {
secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum());
}
}
// now reads should be enabled
secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:29,代码来源:TestHRegionReplayEvents.java
示例8: writeFlushRequestMarkerToWAL
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Writes a marker to WAL indicating a flush is requested but cannot be complete due to various
* reasons. Ignores exceptions from WAL. Returns whether the write succeeded.
*
* @param wal
* @return whether WAL write was successful
*/
private boolean writeFlushRequestMarkerToWAL(WAL wal, boolean writeFlushWalMarker) {
if (writeFlushWalMarker && wal != null && !writestate.readOnly) {
FlushDescriptor desc = ProtobufUtil
.toFlushDescriptor(FlushAction.CANNOT_FLUSH, getRegionInfo(), -1,
new TreeMap<byte[], List<Path>>(Bytes.BYTES_COMPARATOR));
try {
WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(), desc, true, mvcc);
return true;
} catch (IOException e) {
LOG.warn(getRegionInfo().getEncodedName() + " : "
+ "Received exception while trying to write the flush request to wal", e);
}
}
return false;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:23,代码来源:HRegion.java
示例9: getNextSequenceId
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Method to safely get the next sequence number.
*
* @return Next sequence number unassociated with any actual edit.
* @throws IOException
*/
@VisibleForTesting protected long getNextSequenceId(final WAL wal) throws IOException {
// TODO: For review. Putting an empty edit in to get a sequenceid out will
// not work if the
// WAL is banjaxed... if it has gotten an exception and the WAL has not yet
// been rolled or
// aborted. In this case, we'll just get stuck here. For now, until
// HBASE-12751, just have
// a timeout. May happen in tests after we tightened the semantic via
// HBASE-14317.
// Also, the getSequenceId blocks on a latch. There is no global list of
// outstanding latches
// so if an abort or stop, there is no way to call them in.
WALKey key = this.appendEmptyEdit(wal);
mvcc.complete(key.getWriteEntry());
return key.getSequenceId(this.maxWaitForSeqId);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:23,代码来源:HRegion.java
示例10: appendEmptyEdit
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Append a faked WALEdit in order to get a long sequence number and wal syncer will just ignore
* the WALEdit append later.
*
* @param wal
* @return Return the key used appending with no sync and no append.
* @throws IOException
*/
private WALKey appendEmptyEdit(final WAL wal) throws IOException {
// we use HLogKey here instead of WALKey directly to support legacy
// coprocessors.
@SuppressWarnings("deprecation") WALKey key =
new HLogKey(getRegionInfo().getEncodedNameAsBytes(), getRegionInfo().getTable(),
WALKey.NO_SEQUENCE_ID, 0, null, HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());
// Call append but with an empty WALEdit. The returned sequence id will not
// be associated
// with any edit and we can be sure it went in after all outstanding
// appends.
try {
wal.append(getTableDesc(), getRegionInfo(), key, WALEdit.EMPTY_WALEDIT, false);
} catch (Throwable t) {
// If exception, our mvcc won't get cleaned up by client, so do it here.
getMVCC().complete(key.getWriteEntry());
}
return key;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:28,代码来源:HRegion.java
示例11: mergeAndVerify
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private HRegion mergeAndVerify(final String msg, final String regionName1,
final String regionName2, final WAL log, final int upperbound)
throws Exception {
Merge merger = new Merge(this.conf);
LOG.info(msg);
LOG.info("fs2=" + this.conf.get("fs.defaultFS"));
int errCode = ToolRunner.run(this.conf, merger,
new String[] {this.desc.getTableName().getNameAsString(), regionName1, regionName2}
);
assertTrue("'" + msg + "' failed with errCode " + errCode, errCode == 0);
HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
// Now verify that we can read all the rows from regions 0, 1
// in the new merged region.
HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf);
verifyMerge(merged, upperbound);
merged.close();
LOG.info("Verified " + msg);
return merged;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:21,代码来源:TestMergeTool.java
示例12: mockWAL
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Utility method to setup a WAL mock.
* Needs to do the bit where we close latch on the WALKey on append else test hangs.
* @return
* @throws IOException
*/
private WAL mockWAL() throws IOException {
WAL wal = mock(WAL.class);
Mockito.when(wal.append((HTableDescriptor)Mockito.any(), (HRegionInfo)Mockito.any(),
(WALKey)Mockito.any(), (WALEdit)Mockito.any(), Mockito.anyBoolean())).
thenAnswer(new Answer<Long>() {
@Override
public Long answer(InvocationOnMock invocation) throws Throwable {
WALKey key = invocation.getArgumentAt(2, WALKey.class);
MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
key.setWriteEntry(we);
return 1L;
}
});
return wal;
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:23,代码来源:TestHRegion.java
示例13: testAtomicBulkLoad
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Atomic bulk load.
*/
@Test
public void testAtomicBulkLoad() throws Exception {
TableName TABLE_NAME = TableName.valueOf("atomicBulkLoad");
int millisToRun = 30000;
int numScanners = 50;
UTIL.startMiniCluster(1);
try {
WAL log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(null);
FindBulkHBaseListener listener = new FindBulkHBaseListener();
log.registerWALActionsListener(listener);
runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners);
assertThat(listener.isFound(), is(true));
} finally {
UTIL.shutdownMiniCluster();
}
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:22,代码来源:TestHRegionServerBulkLoad.java
示例14: addEdits
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
protected void addEdits(WAL log,
HRegionInfo hri,
HTableDescriptor htd,
int times,
MultiVersionConcurrencyControl mvcc)
throws IOException {
final byte[] row = Bytes.toBytes("row");
for (int i = 0; i < times; i++) {
long timestamp = System.currentTimeMillis();
WALEdit cols = new WALEdit();
cols.add(new KeyValue(row, row, row, timestamp, row));
WALKey key = new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(),
WALKey.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
HConstants.NO_NONCE, mvcc);
log.append(htd, hri, key, cols, true);
}
log.sync();
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:19,代码来源:TestFSHLog.java
示例15: testNoEdits
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Tests that the LogRoller perform the roll even if there are no edits
*/
@Test
public void testNoEdits() throws Exception {
TableName tableName = TableName.valueOf("TestLogRollPeriodNoEdits");
TEST_UTIL.createTable(tableName, "cf");
try {
Table table = new HTable(TEST_UTIL.getConfiguration(), tableName);
try {
HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(tableName);
WAL log = server.getWAL(null);
checkMinLogRolls(log, 5);
} finally {
table.close();
}
} finally {
TEST_UTIL.deleteTable(tableName);
}
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:21,代码来源:TestLogRollPeriod.java
示例16: checkMinLogRolls
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private void checkMinLogRolls(final WAL log, final int minRolls)
throws Exception {
final List<Path> paths = new ArrayList<Path>();
log.registerWALActionsListener(new WALActionsListener.Base() {
@Override
public void postLogRoll(Path oldFile, Path newFile) {
LOG.debug("postLogRoll: oldFile="+oldFile+" newFile="+newFile);
paths.add(newFile);
}
});
// Sleep until we should get at least min-LogRoll events
long wtime = System.currentTimeMillis();
Thread.sleep((minRolls + 1) * LOG_ROLL_PERIOD);
// Do some extra sleep in case the machine is slow,
// and the log-roll is not triggered exactly on LOG_ROLL_PERIOD.
final int NUM_RETRIES = 1 + 8 * (minRolls - paths.size());
for (int retry = 0; paths.size() < minRolls && retry < NUM_RETRIES; ++retry) {
Thread.sleep(LOG_ROLL_PERIOD / 4);
}
wtime = System.currentTimeMillis() - wtime;
LOG.info(String.format("got %d rolls after %dms (%dms each) - expected at least %d rolls",
paths.size(), wtime, wtime / paths.size(), minRolls));
assertFalse(paths.size() < minRolls);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:26,代码来源:TestLogRollPeriod.java
示例17: testLogRolling
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
/**
* Tests that logs are deleted
* @throws IOException
* @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
*/
@Test
public void testLogRolling() throws Exception {
this.tableName = getName();
// TODO: Why does this write data take for ever?
startAndWriteData();
final WAL log = server.getWAL(null);
LOG.info("after writing there are " + DefaultWALProvider.getNumRolledLogFiles(log) +
" log files");
// flush all regions
for (Region r: server.getOnlineRegionsLocalContext()) {
r.flush(true);
}
// Now roll the log
log.rollWriter();
int count = DefaultWALProvider.getNumRolledLogFiles(log);
LOG.info("after flushing all regions and rolling logs there are " + count + " log files");
assertTrue(("actual count: " + count), count <= 2);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:27,代码来源:TestLogRolling.java
示例18: testIncrementWithReturnResultsSetToFalse
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
@Test
public void testIncrementWithReturnResultsSetToFalse() throws Exception {
byte[] row1 = Bytes.toBytes("row1");
byte[] col1 = Bytes.toBytes("col1");
// Setting up region
final WALFactory wals = new WALFactory(CONF, null, "testIncrementWithReturnResultsSetToFalse");
byte[] tableName = Bytes.toBytes("testIncrementWithReturnResultsSetToFalse");
final WAL wal = wals.getWAL(tableName);
HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
Increment inc1 = new Increment(row1);
inc1.setReturnResults(false);
inc1.addColumn(FAMILY, col1, 1);
Result res = region.increment(inc1);
assertNull(res);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:18,代码来源:TestDurability.java
示例19: createHRegion
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private HRegion createHRegion (byte [] tableName, String callingMethod,
WAL log, Durability durability)
throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
htd.setDurability(durability);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
Path path = new Path(DIR + callingMethod);
if (FS.exists(path)) {
if (!FS.delete(path, true)) {
throw new IOException("Failed delete of " + path);
}
}
return HRegion.createHRegion(info, path, CONF, htd, log);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:17,代码来源:TestDurability.java
示例20: getRegion
import org.apache.hadoop.hbase.wal.WAL; //导入依赖的package包/类
private HRegion getRegion(final Configuration conf, final String tableName) throws IOException {
WAL wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
TEST_UTIL.getDataTestDir().toString(), conf);
return (HRegion)TEST_UTIL.createLocalHRegion(Bytes.toBytes(tableName),
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, tableName, conf,
false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
}
开发者ID:fengchen8086,项目名称:ditb,代码行数:8,代码来源:TestRegionIncrement.java
注:本文中的org.apache.hadoop.hbase.wal.WAL类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论