import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testOverrideTypeMapping() throws IOException {
  String [] types = { "INT" };
  String [] vals = { "10" };
  createTableWithColTypes(types, vals);

  String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
  runImport(getOutputArgv(true, extraArgs));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "DATA_COL0", Type.STRING);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testNonIdentCharactersInColumnName() throws IOException {
  String [] names = { "test_p-a+r/quet" };
  String [] types = { "INT" };
  String [] vals = { "2015" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "TEST_P_A_R_QUET", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("TEST_P_A_R_QUET", 2015, record1.get("TEST_P_A_R_QUET"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testNullableParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { null };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertNull(record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testQueryImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputQueryArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testIncrementalParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));
  runImport(getOutputArgv(true, new String[]{"--append"}));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
                @DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {

  DatasetReader<StandardEvent> reader = input.newReader();
  DatasetWriter<StandardEvent> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
                @DataOut(name="target_users") View<GenericRecord> output) {

  DatasetReader<GenericRecord> reader = input.newReader();
  DatasetWriter<GenericRecord> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get the epoch timestamps for every snapshot time of an URL in HBase.
 * 
 * @param url
 *          The URL of the page to get snapshot timestamps for
 * @return The list of timestamps
 */
public List<Long> getSnapshotTimestamps(String url) {
  url = normalizeUrl(url);
  List<Long> snapshotTimestamps = new ArrayList<Long>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    while (reader.hasNext()) {
      snapshotTimestamps.add(reader.next().getFetchedAt());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return snapshotTimestamps;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get the most recent WebPageSnapshotModel from HBase
 * 
 * @param url
 *          The URL to get the snapshotted page from HBase
 * @return The WebPageSnapshotModel, or null if there are no fetches for this
 *         URL
 */
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
  url = normalizeUrl(url);
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    // we don't know the exact timestamp in the key, but we know since keys
    // are in timestamp descending order that the first row for an URL will be
    // the most recent.
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    if (reader.hasNext()) {
      return reader.next();
    } else {
      return null;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get WebPageSnapshotModels for an URL from HBase since the since param.
 * 
 * @param url
 *          The URL of the page to fetch
 * @param since
 *          The models to fetch since
 * @return The list of models that have been fetched for an URL since the
 *         since param.
 */
private List<WebPageSnapshotModel> getWebPageSnapshotsSince(String url,
    long since) {
  url = normalizeUrl(url);
  List<WebPageSnapshotModel> models = new ArrayList<WebPageSnapshotModel>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", since).newReader();
    while (reader.hasNext()) {
      models.add(reader.next());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return models;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hdfs:/tmp/data/users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.newReader();
    for (GenericRecord user : reader) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hive?dataset=users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.newReader();
    for (GenericRecord user : users.newReader()) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hdfs:/tmp/data/users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.with("favoriteColor", "green").newReader();
    for (GenericRecord user : reader) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the products dataset
  Dataset<Product> products = Datasets.load(
      "dataset:hdfs:/tmp/data/products", Product.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Product> reader = null;
  try {
    reader = products.newReader();
    for (Product product : reader) {
      System.out.println(product);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {

  // Load the events dataset
  Dataset<GenericRecord> events = Datasets.load("dataset:hive:/tmp/data/default/events");

  // Get a reader for the dataset and read all the events
  DatasetReader<GenericRecord> reader = events.newReader();
  try {
    for (GenericRecord event : reader) {
      System.out.println(event);
    }
  } finally {
    reader.close();
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  Dataset<Record> movies = Datasets.load(
      "dataset:hdfs:/tmp/data/movies", Record.class);

  DatasetReader<Record> reader = null;
  try {
    reader = movies.newReader();
    for (Record rec : reader) {
      System.err.println("Movie: " + rec);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public static <T> HashSet<T> read(View<T> view) {
  DatasetReader<T> reader = null;
  try {
    reader = view.newReader();
    return Sets.newHashSet(reader.iterator());
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
private void verifyHiveDataset(String tableName, Object[][] valsArray) {
  String datasetUri = String.format("dataset:hive:default/%s",
      tableName.toLowerCase());
  assertTrue(Datasets.exists(datasetUri));
  Dataset dataset = Datasets.load(datasetUri);
  assertFalse(dataset.isEmpty());

  DatasetReader<GenericRecord> reader = dataset.newReader();
  try {
    List<String> expectations = new ArrayList<String>();
    if (valsArray != null) {
      for (Object[] vals : valsArray) {
        expectations.add(Arrays.toString(vals));
      }
    }

    while (reader.hasNext() && expectations.size() > 0) {
      String actual = Arrays.toString(
          convertGenericRecordToArray(reader.next()));
      assertTrue("Expect record: " + actual, expectations.remove(actual));
    }
    assertFalse(reader.hasNext());
    assertEquals(0, expectations.size());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testMultiTableImportAsParquetFormat() throws IOException {
  String [] argv = getArgv(new String[]{"--as-parquetfile"}, null);
  runImport(new ImportAllTablesTool(), argv);

  Path warehousePath = new Path(this.getWarehouseDir());
  int i = 0;
  for (String tableName : this.tableNames) {
    Path tablePath = new Path(warehousePath, tableName);
    Dataset dataset = Datasets.load("dataset:file:" + tablePath);

    // dequeue the expected value for this table. This
    // list has the same order as the tableNames list.
    String expectedVal = Integer.toString(i++) + ","
        + this.expectedStrings.get(0);
    this.expectedStrings.remove(0);

    DatasetReader<GenericRecord> reader = dataset.newReader();
    try {
      GenericRecord record = reader.next();
      String line = record.get(0) + "," + record.get(1);
      assertEquals("Table " + tableName + " expected a different string",
          expectedVal, line);
      assertFalse(reader.hasNext());
    } finally {
      reader.close();
    }
  }
}