本文整理汇总了Java中org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector类的典型用法代码示例。如果您正苦于以下问题：Java BytesColumnVector类的具体用法？Java BytesColumnVector怎么用？Java BytesColumnVector使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
BytesColumnVector类属于org.apache.hadoop.hive.ql.exec.vector包，在下文中一共展示了BytesColumnVector类的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: compareStringcolumn
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
static void compareStringcolumn(ColumnVector oneStringColumn, boolean[] isNull,
long currentBatchRow, Vec h2oFrame, Long startRowIndex, String columnType) {
byte[][] oneColumn = ((BytesColumnVector) oneStringColumn).vector;
int[] stringLength = ((BytesColumnVector) oneStringColumn).length;
int[] stringStart = ((BytesColumnVector) oneStringColumn).start;
long frameRowIndex = startRowIndex;
BufferedString tempH2o = new BufferedString();
BufferedString tempOrc = new BufferedString();
for (int rowIndex = 0; rowIndex < currentBatchRow; rowIndex++) {
if (isNull[rowIndex])
assertEquals("Na is found: ", true, h2oFrame.isNA(frameRowIndex));
else {
if (!oneStringColumn.isRepeating || rowIndex == 0)
tempOrc.set(oneColumn[rowIndex], stringStart[rowIndex], stringLength[rowIndex]);
h2oFrame.atStr(tempH2o, frameRowIndex);
assertEquals("isRepeating = " + oneStringColumn.isRepeating + " String/char elements should equal: ", true, tempOrc.equals(tempH2o));
}
frameRowIndex++;
}
}
开发者ID:h2oai，项目名称:h2o-3，代码行数:23，代码来源:OrcTestUtils.java
示例2: setColumnVector
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
public void setColumnVector( final ColumnVector vector , final IExpressionIndex indexList , final int start , final int length ) throws IOException{
BytesColumnVector columnVector = (BytesColumnVector)vector;
PrimitiveObject[] primitiveObjectArray = column.getPrimitiveObjectArray( indexList , start , length );
for( int i = 0 ; i < length ; i++ ){
if( primitiveObjectArray[i] == null ){
VectorizedBatchUtil.setNullColIsNullValue( columnVector , i );
}
else{
if( primitiveObjectArray[i] instanceof IBytesLink ){
IBytesLink linkObj = (IBytesLink)primitiveObjectArray[i];
columnVector.vector[i] = linkObj.getLinkBytes();
columnVector.start[i] = linkObj.getStart();
columnVector.length[i] = linkObj.getLength();
}
else{
byte[] strBytes = primitiveObjectArray[i].getBytes();
if( strBytes == null ){
VectorizedBatchUtil.setNullColIsNullValue( columnVector , i );
}
else{
columnVector.vector[i] = strBytes;
columnVector.start[i] = 0;
columnVector.length[i] = strBytes.length;
}
}
}
}
}
开发者ID:yahoojapan，项目名称:multiple-dimension-spread，代码行数:31，代码来源:BytesColumnVectorAssignor.java
示例3: writeBatch
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
void writeBatch(ColumnVector vector, int offset, int length) throws IOException {
super.writeBatch(vector, offset, length);
BytesColumnVector vec = (BytesColumnVector) vector;
if (vector.isRepeating) {
if (vector.noNulls || !vector.isNull[0]) {
for (int i = 0; i < length; ++i) {
stream.write(vec.vector[0], vec.start[0], vec.length[0]);
this.length.write(vec.length[0]);
}
indexStatistics.updateBinary(vec.vector[0], vec.start[0], vec.length[0], length);
if (createBloomFilter) {
bloomFilter.addBytes(vec.vector[0], vec.start[0], vec.length[0]);
}
}
} else {
for (int i = 0; i < length; ++i) {
if (vec.noNulls || !vec.isNull[i + offset]) {
stream.write(vec.vector[offset + i], vec.start[offset + i], vec.length[offset + i]);
this.length.write(vec.length[offset + i]);
indexStatistics.updateBinary(vec.vector[offset + i], vec.start[offset + i],
vec.length[offset + i], 1);
if (createBloomFilter) {
bloomFilter.addBytes(vec.vector[offset + i], vec.start[offset + i],
vec.length[offset + i]);
}
}
}
}
}
开发者ID:ampool，项目名称:monarch，代码行数:31，代码来源:AWriterImpl.java
示例4: process
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
public void process(RelationContainer container) {
DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
ListColumnVector members = (ListColumnVector) batch.cols[6];
checkLimit();
addCommonProperties(container);
lat.isNull[row] = true;
lon.isNull[row] = true;
lat.set(row, (HiveDecimal) null);
lon.set(row, (HiveDecimal) null);
Relation relation = container.getEntity();
members.lengths[row] = relation.getMembers().size();
members.childCount += members.lengths[row];
members.child.ensureSize(members.childCount, members.offsets[row] != 0);
for (int j = 0; j < relation.getMembers().size(); j++) {
StructColumnVector membersStruct = (StructColumnVector) members.child;
((BytesColumnVector) membersStruct.fields[0]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberType().toString().toLowerCase().getBytes());
((LongColumnVector) membersStruct.fields[1]).vector[(int) members.offsets[row] + j] = relation.getMembers().get(j).getMemberId();
((BytesColumnVector) membersStruct.fields[2]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberRole().getBytes());
}
}
开发者ID:mojodna，项目名称:osm2orc，代码行数:29，代码来源:OrcWriter.java
示例5: convert
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
public void convert(JsonElement value, ColumnVector vect, int row) {
if (value == null || value.isJsonNull()) {
vect.noNulls = false;
vect.isNull[row] = true;
} else {
BytesColumnVector vector = (BytesColumnVector) vect;
byte[] bytes = value.getAsString().getBytes(
StandardCharsets.UTF_8);
vector.setRef(row, bytes, 0, bytes.length);
}
}
开发者ID:pinterest，项目名称:secor，代码行数:12，代码来源:VectorColumnFiller.java
示例6: setBytesColumnVector
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private void setBytesColumnVector( BytesColumnVector bytesColumnVector, String value ) {
if ( value == null ) {
setBytesColumnVector( bytesColumnVector, new byte[ 0 ] );
} else {
setBytesColumnVector( bytesColumnVector, value.getBytes() );
}
}
开发者ID:pentaho，项目名称:pentaho-hadoop-shims，代码行数:8，代码来源:PentahoOrcRecordWriter.java
示例7: flush
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private boolean flush(BufferSegment segment, String path, TypeDescription schema)
{
Configuration conf = new Configuration();
try {
Writer writer = OrcFile.createWriter(new Path(path),
OrcFile.writerOptions(conf)
.setSchema(schema)
.stripeSize(orcFileStripeSize)
.bufferSize(orcFileBufferSize)
.blockSize(orcFileBlockSize)
.compress(CompressionKind.ZLIB)
.version(OrcFile.Version.V_0_12));
VectorizedRowBatch batch = schema.createRowBatch();
while (segment.hasNext()) {
String[] contents = segment.getNext();
int rowCount = batch.size++;
// System.out.println("contents : message.getValues() : " + Arrays.toString(contents));
System.out.println("contents.length : " + contents.length);
for (int i = 0; i < contents.length; i++) {
((BytesColumnVector) batch.cols[i]).setVal(rowCount, contents[i].getBytes());
//batch full
if (batch.size == batch.getMaxSize()) {
writer.addRowBatch(batch);
batch.reset();
}
}
if (batch.size != 0) {
writer.addRowBatch(batch);
batch.reset();
}
writer.close();
segment.setFilePath(path);
System.out.println("path : " + path);
}
return true;
}
catch (IOException e) {
e.printStackTrace();
return false;
}
}
开发者ID:dbiir，项目名称:paraflow，代码行数:42，代码来源:OrcFlushThread.java
示例8: addCommonProperties
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private void addCommonProperties(EntityContainer container) {
LongColumnVector id = (LongColumnVector) batch.cols[0];
BytesColumnVector type = (BytesColumnVector) batch.cols[1];
MapColumnVector tags = (MapColumnVector) batch.cols[2];
ListColumnVector nds = (ListColumnVector) batch.cols[5];
ListColumnVector members = (ListColumnVector) batch.cols[6];
LongColumnVector changeset = (LongColumnVector) batch.cols[7];
TimestampColumnVector timestamp = (TimestampColumnVector) batch.cols[8];
LongColumnVector uid = (LongColumnVector) batch.cols[9];
BytesColumnVector user = (BytesColumnVector) batch.cols[10];
LongColumnVector version = (LongColumnVector) batch.cols[11];
LongColumnVector visible = (LongColumnVector) batch.cols[12];
Entity entity = container.getEntity();
id.vector[row] = entity.getId();
changeset.vector[row] = entity.getChangesetId();
type.setVal(row, entity.getType().toString().toLowerCase().getBytes());
tags.offsets[row] = tags.childCount;
tags.lengths[row] = entity.getTags().size(); // number of key/value pairings
tags.childCount += tags.lengths[row];
tags.keys.ensureSize(tags.childCount, tags.offsets[row] != 0);
tags.values.ensureSize(tags.childCount, tags.offsets[row] != 0);
int i = 0;
for (Tag tag : entity.getTags()) {
((BytesColumnVector) tags.keys).setVal((int) tags.offsets[row] + i, tag.getKey().getBytes());
((BytesColumnVector) tags.values).setVal((int) tags.offsets[row] + i, tag.getValue().getBytes());
i++;
}
timestamp.time[row] = entity.getTimestamp().getTime();
timestamp.nanos[row] = 0;
uid.vector[row] = entity.getUser().getId();
user.setVal(row, entity.getUser().getName().getBytes());
version.vector[row] = entity.getVersion();
visible.vector[row] = 1;
if (entity.getMetaTags().get("visible") == Boolean.FALSE) {
visible.vector[row] = 0;
}
nds.offsets[row] = nds.childCount;
nds.lengths[row] = 0;
members.offsets[row] = members.childCount;
members.lengths[row] = 0;
}
开发者ID:mojodna，项目名称:osm2orc，代码行数:54，代码来源:OrcWriter.java
示例9: setValue
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
static void setValue(JSONWriter writer, ColumnVector vector,
TypeDescription schema, int row) throws JSONException {
if (vector.isRepeating) {
row = 0;
}
if (vector.noNulls || !vector.isNull[row]) {
switch (schema.getCategory()) {
case BOOLEAN:
writer.value(((LongColumnVector) vector).vector[row] != 0);
break;
case BYTE:
case SHORT:
case INT:
case LONG:
writer.value(((LongColumnVector) vector).vector[row]);
break;
case FLOAT:
case DOUBLE:
writer.value(((DoubleColumnVector) vector).vector[row]);
break;
case STRING:
case CHAR:
case VARCHAR:
writer.value(((BytesColumnVector) vector).toString(row));
break;
case DECIMAL:
writer.value(((DecimalColumnVector) vector).vector[row]
.toString());
break;
case DATE:
writer.value(new DateWritable(
(int) ((LongColumnVector) vector).vector[row])
.toString());
break;
case TIMESTAMP:
writer.value(((TimestampColumnVector) vector)
.asScratchTimestamp(row).toString());
break;
case LIST:
setList(writer, (ListColumnVector) vector, schema, row);
break;
case STRUCT:
setStruct(writer, (StructColumnVector) vector, schema, row);
break;
case UNION:
// printUnion(writer, (UnionColumnVector) vector, schema, row);
break;
case BINARY:
// printBinary(writer, (BytesColumnVector) vector, row);
break;
case MAP:
// printMap(writer, (MapColumnVector) vector, schema, row);
break;
default:
throw new IllegalArgumentException("Unknown type "
+ schema.toString());
}
} else {
writer.value(null);
}
}
开发者ID:pinterest，项目名称:secor，代码行数:62，代码来源:JsonFieldFiller.java
示例10: convertFromSourceToTargetDataType
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
protected static Object convertFromSourceToTargetDataType( ColumnVector columnVector, int currentBatchRow,
int orcValueMetaInterface ) {
if ( columnVector.isNull[currentBatchRow] ) {
return null;
}
switch ( orcValueMetaInterface ) {
case ValueMetaInterface.TYPE_INET:
try {
return InetAddress.getByName( new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
( (BytesColumnVector) columnVector ).length[ currentBatchRow ] ) );
} catch ( UnknownHostException e ) {
e.printStackTrace();
}
case ValueMetaInterface.TYPE_STRING:
return new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );
case ValueMetaInterface.TYPE_INTEGER:
return (long) ( (LongColumnVector) columnVector ).vector[ currentBatchRow ];
case ValueMetaInterface.TYPE_NUMBER:
return ( (DoubleColumnVector) columnVector ).vector[ currentBatchRow ];
case ValueMetaInterface.TYPE_BIGNUMBER:
HiveDecimalWritable obj = ( (DecimalColumnVector) columnVector ).vector[ currentBatchRow ];
return obj.getHiveDecimal().bigDecimalValue();
case ValueMetaInterface.TYPE_TIMESTAMP:
Timestamp timestamp = new Timestamp( ( (TimestampColumnVector) columnVector ).time[ currentBatchRow ] );
timestamp.setNanos( ( (TimestampColumnVector) columnVector ).nanos[ currentBatchRow ] );
return timestamp;
case ValueMetaInterface.TYPE_DATE:
LocalDate localDate = LocalDate.ofEpochDay( 0 ).plusDays( ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] );
Date dateValue = Date.from( localDate.atStartOfDay( ZoneId.systemDefault() ).toInstant() );
return dateValue;
case ValueMetaInterface.TYPE_BOOLEAN:
return ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] == 0 ? false : true;
case ValueMetaInterface.TYPE_BINARY:
byte[] origBytes = ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ];
int startPos = ( (BytesColumnVector) columnVector ).start[ currentBatchRow ];
byte[] newBytes = Arrays.copyOfRange( origBytes, startPos,
startPos + ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );
return newBytes;
}
//if none of the cases match return a null
return null;
}
开发者ID:pentaho，项目名称:pentaho-hadoop-shims，代码行数:56，代码来源:OrcConverter.java
注：本文中的org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector类示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论