void DCDataSet::append(size_t count, size_t offset, size_t stride, const void* data)
    throw (DCException)
    {
        log_msg(2, "DCDataSet::append");

        if (!opened)
            throw DCException(getExceptionString("append: Dataset has not been opened/created."));

        log_msg(3, "logical_size = %s", getLogicalSize().toString().c_str());

        Dimensions target_offset(getLogicalSize());
        // extend size (dataspace) of existing dataset with count elements
        getLogicalSize()[0] += count;

        hsize_t * max_dims = new hsize_t[ndims];
        for (size_t i = 0; i < ndims; ++i)
            max_dims[i] = H5F_UNLIMITED;

        if (H5Sset_extent_simple(dataspace, 1, getLogicalSize().getPointer(), max_dims) < 0)
            throw DCException(getExceptionString("append: Failed to set new extent"));

        delete[] max_dims;
        max_dims = NULL;

        log_msg(3, "logical_size = %s", getLogicalSize().toString().c_str());

        if (H5Dset_extent(dataset, getLogicalSize().getPointer()) < 0)
            throw DCException(getExceptionString("append: Failed to extend dataset"));

        // select the region in the target DataSpace to write to
        Dimensions dim_data(count, 1, 1);
        if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, target_offset.getPointer(),
                NULL, dim_data.getPointer(), NULL) < 0 ||
                H5Sselect_valid(dataspace) < 0)
            throw DCException(getExceptionString("append: Invalid target hyperslap selection"));

        // append data to the dataset.
        // select the region in the source DataSpace to read from
        Dimensions dim_src(offset + count * stride, 1, 1);
        hid_t dsp_src = H5Screate_simple(1, dim_src.getPointer(), NULL);
        if (dsp_src < 0)
            throw DCException(getExceptionString("append: Failed to create src dataspace while appending"));

        if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, Dimensions(offset, 0, 0).getPointer(),
                Dimensions(stride, 1, 1).getPointer(), dim_data.getPointer(), NULL) < 0 ||
                H5Sselect_valid(dsp_src) < 0)
            throw DCException(getExceptionString("append: Invalid source hyperslap selection"));

        if (!data || (count == 0))
        {
            H5Sselect_none(dataspace);
            data = NULL;
        }

        if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0)
            throw DCException(getExceptionString("append: Failed to append dataset"));

        H5Sclose(dsp_src);
    }

void
avtGTCFileFormat::ReadVariable( int domain, int varIdx, int varDim, float **ptrVar )
{
    debug5 << "Reading Variable: " << startOffset << " " << nPoints << endl;
    hid_t dataspace = H5Dget_space(particleHandle);

    //Select the Var.
    hsize_t start[2] = { static_cast<hsize_t>(startOffset),
                         static_cast<hsize_t>(varIdx) };

    hsize_t count[2] = { static_cast<hsize_t>(nPoints),
                         static_cast<hsize_t>(varDim) };

    H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, NULL, count, NULL);

    hsize_t dataDim[1] = { static_cast<hsize_t>(nPoints*varDim) };
    hid_t memspace = H5Screate_simple(1, dataDim, NULL);
    H5Sselect_all(memspace);

    //Read the variable from file.
    float *var = new float[nPoints*varDim];
    H5Dread(particleHandle, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, var );
    H5Sclose(memspace);

    //Select ID
    start[0] = startOffset;
    start[1] = VarNameToIndex( "id" );
    
    count[0] = nPoints;
    count[1] = 1;    
    H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, NULL, count, NULL);
    
    // Read in ID.
    dataDim[0] = nPoints;
    memspace = H5Screate_simple(1, dataDim, NULL);
    H5Sselect_all(memspace);
    float *ids = new float[nPoints];
    H5Dread(particleHandle, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, ids );
    
    H5Sclose(memspace);
    H5Sclose(dataspace);

#ifdef PARALLEL
    ParallelReadVariable( domain, varDim, var, ids );
#endif

    //Put the variables into the right order.
    for ( int i = 0; i < nPoints; i++ )
    {
        int id = (int)ids[i] - startOffset - 1;
        memcpy( (void *)&((*ptrVar)[i*varDim]), (void*)&var[id*varDim], varDim*sizeof(float) );
    }

    delete [] ids;
    delete [] var;
}

char* ossimHdf5SubDataset::getTileBuf(const  ossimIrect& rect, ossim_uint32 band)
{
   hsize_t     count[3];
   hsize_t     offset[3];
   hid_t       memspace;
   hsize_t     col_dims[3];

   if (m_rank == 3)
   {
      offset[0] = band;
      offset[1] = rect.ul().y;
      offset[2] = rect.ul().x;

      count[0] = 1;
      count[1] = rect.height();
      count[2] = rect.width();

      col_dims[0] = 1;
      col_dims[1] = rect.height();
      col_dims[2] = rect.width();
   }
   else
   {
      offset[0] = rect.ul().y;
      offset[1] = rect.ul().x;

      count[0] = rect.height();
      count[1] = rect.width();

      col_dims[0] = rect.height();
      col_dims[1] = rect.width();
   }

   // herr_t status =  H5Sselect_hyperslab(m_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL );
   H5Sselect_hyperslab(m_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL );   

   memspace = H5Screate_simple(m_rank, col_dims, NULL);
   hsize_t mem_offset[3];
   mem_offset[0] = 0;
   mem_offset[1] = 0;
   mem_offset[2] = 0;
   // int status =  H5Sselect_hyperslab(memspace, H5S_SELECT_SET, mem_offset, NULL, count, NULL);
   H5Sselect_hyperslab(memspace, H5S_SELECT_SET, mem_offset, NULL, count, NULL);

   ossim_int32 numValues = rect.width() * rect.height();
   char* data = new char[m_dataSize * numValues];
   // status = H5Dread(m_dataset_id, m_dataType, memspace, m_dataspace, H5P_DEFAULT, (void*)data);
   H5Dread(m_dataset_id, m_dataType, memspace, m_dataspace, H5P_DEFAULT, (void*)data);   

   H5Sclose(memspace);
   return data;
}

/* ------- begin --------------------------   writeMPI_all.c --- */
void writeMPI_all(void) {
/* Writes output on indata file, MPI group, all tasks at once */
  const char routineName[] = "writeMPI_all";
  int      task;
  hsize_t  offset[] = {0, 0, 0, 0};
  hsize_t  count[] = {1, 1, 1, 1};
  hsize_t  dims[4];
  hid_t    file_dspace, mem_dspace;

  /* Write single values of Ntasks, one value at a time */
  dims[0] = 1;
  if (( mem_dspace = H5Screate_simple(1, dims, NULL) ) < 0)
    HERR(routineName);

  for (task = 0; task < mpi.Ntasks; task++) {
    offset[0] = mpi.taskmap[task + mpi.my_start][0];
    offset[1] = mpi.taskmap[task + mpi.my_start][1];
    if (( file_dspace = H5Dget_space(io.in_mpi_it) ) < 0) HERR(routineName);
    if (( H5Sselect_hyperslab(file_dspace, H5S_SELECT_SET, offset,
                              NULL, count, NULL) ) < 0) HERR(routineName);
    if (( H5Dwrite(io.in_mpi_it, H5T_NATIVE_INT, mem_dspace, file_dspace,
                   H5P_DEFAULT, &mpi.niter[task]) ) < 0) HERR(routineName);
    if (( H5Dwrite(io.in_mpi_conv, H5T_NATIVE_INT, mem_dspace, file_dspace,
                H5P_DEFAULT, &mpi.convergence[task]) ) < 0) HERR(routineName);
    if (( H5Dwrite(io.in_mpi_zc, H5T_NATIVE_INT, mem_dspace, file_dspace,
                  H5P_DEFAULT, &mpi.zcut_hist[task]) ) < 0) HERR(routineName);
    if (( H5Dwrite(io.in_mpi_dm, H5T_NATIVE_DOUBLE, mem_dspace, file_dspace,
                   H5P_DEFAULT, &mpi.dpopsmax[task]) ) < 0) HERR(routineName);
    if (( H5Sclose(file_dspace) ) < 0) HERR(routineName);
  }
  if (( H5Sclose(mem_dspace) ) < 0) HERR(routineName);

  /* Write array with multiple values */
  for (task = 0; task < mpi.Ntasks; task++) {
    dims[0] = mpi.niter[task];
    if (( mem_dspace = H5Screate_simple(1, dims, NULL) ) < 0)
      HERR(routineName);
    offset[0] = mpi.taskmap[task + mpi.my_start][0];
    offset[1] = mpi.taskmap[task + mpi.my_start][1];
    count[2] = mpi.niter[task];
    if (( file_dspace = H5Dget_space(io.in_mpi_dmh) ) < 0) HERR(routineName);
    if (( H5Sselect_hyperslab(file_dspace, H5S_SELECT_SET, offset,
                              NULL, count, NULL) ) < 0) HERR(routineName);
    if (( H5Dwrite(io.in_mpi_dmh, H5T_NATIVE_DOUBLE, mem_dspace, file_dspace,
               H5P_DEFAULT, mpi.dpopsmax_hist[task]) ) < 0) HERR(routineName);
    if (( H5Sclose(file_dspace) ) < 0) HERR(routineName);
    if (( H5Sclose(mem_dspace) ) < 0) HERR(routineName);
  }
  return;
}

void InfiniteDimensionalMCMCSampler::_append_scalar_dataset(hid_t dset, double data)
{
  // Only subprocess with rank 0 manipulates the output file
  if ((this->m_env).subRank() == 0) {
    int err;
    // Create a memory dataspace for data to append
    const int ndims = 1;
    hsize_t dims[ndims] = { 1 };  // Only writing one double
    hid_t mem_space = H5Screate_simple(ndims, dims, NULL);

    // Extend the dataset
    // Set dims to be the *new* dimension of the extended dataset
    dims[0] = { this->_iteration / this->m_ov->m_save_freq };
    err = H5Dset_extent(dset, dims);

    // Select hyperslab on file dataset
    hid_t file_space = H5Dget_space(dset);
    hsize_t start[1] = {(this->_iteration / this->m_ov->m_save_freq) - 1};
    hsize_t count[1] = {1};

    err = H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start, NULL, count, NULL);

    // hsize_t      size[1];
    // size[0]   = this->_iteration / this->m_ov->m_save_freq;

    // Write the data
    H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, H5P_DEFAULT, &data);

    // Close a bunch of stuff
    H5Sclose(file_space);
    H5Sclose(mem_space);
  }
}

/*writes a sampled chunk into the appropriate hyperslab of hdf5 file*/
herr_t /*hdf5 error type*/ h5write_current_chunk(hdf5block_t *h5block,/*holds hdf5 properties and ids*/ gsl_matrix *log_para_chunk, /*log-parameter chunk*/ gsl_vector *log_post_chunk)/*log-posterior value chunk*/{
  herr_t status;
  assert(log_para_chunk);
  assert(log_post_chunk);
  int D=log_para_chunk->size2;
  
  h5block->block[0]=CHUNK;
  h5block->block[1]=D;
  status = H5Sselect_hyperslab(h5block->para_dataspace_id, H5S_SELECT_SET, h5block->offset, h5block->stride, h5block->count, h5block->block);
  H5Dwrite(h5block->parameter_set_id, H5T_NATIVE_DOUBLE, h5block->para_chunk_id, h5block->para_dataspace_id, H5P_DEFAULT, log_para_chunk->data);

  h5block->block[1]=1;
  status = H5Sselect_hyperslab(h5block->post_dataspace_id, H5S_SELECT_SET, h5block->offset, h5block->stride, h5block->count, h5block->block);
  H5Dwrite(h5block->posterior_set_id, H5T_NATIVE_DOUBLE, h5block->post_chunk_id, h5block->post_dataspace_id, H5P_DEFAULT, log_post_chunk->data);
  return status;
}

void Hdf5DataReader::GetVariableOverNodes(Vec data,
                                          const std::string& rVariableName,
                                          unsigned timestep)
{
    if (!mIsDataComplete)
    {
        EXCEPTION("You can only get a vector for complete data");
    }
    if (!mIsUnlimitedDimensionSet && timestep!=0)
    {
        EXCEPTION("The dataset '" << mDatasetName << "' does not contain time dependent data");
    }

    std::map<std::string, unsigned>::iterator col_iter = mVariableToColumnIndex.find(rVariableName);
    if (col_iter == mVariableToColumnIndex.end())
    {
        EXCEPTION("The dataset '" << mDatasetName << "' does not contain data for variable " << rVariableName);
    }
    unsigned column_index = (*col_iter).second;

    // Check for valid timestep
    if (timestep >= mNumberTimesteps)
    {
        EXCEPTION("The dataset '" << mDatasetName << "' does not contain data for timestep number " << timestep);
    }

    int lo, hi, size;
    VecGetSize(data, &size);
    if ((unsigned)size != mDatasetDims[1])
    {
        EXCEPTION("Could not read data because Vec is the wrong size");
    }
    // Get range owned by each processor
    VecGetOwnershipRange(data, &lo, &hi);

    if (hi > lo) // i.e. we own some...
    {
        // Define a dataset in memory for this process
        hsize_t v_size[1] = {(unsigned)(hi-lo)};
        hid_t memspace = H5Screate_simple(1, v_size, NULL);

        // Select hyperslab in the file.
        hsize_t offset[3] = {timestep, (unsigned)(lo), column_index};
        hsize_t count[3]  = {1, (unsigned)(hi-lo), 1};
        hid_t hyperslab_space = H5Dget_space(mVariablesDatasetId);
        H5Sselect_hyperslab(hyperslab_space, H5S_SELECT_SET, offset, NULL, count, NULL);

        double* p_petsc_vector;
        VecGetArray(data, &p_petsc_vector);

        herr_t err = H5Dread(mVariablesDatasetId, H5T_NATIVE_DOUBLE, memspace, hyperslab_space, H5P_DEFAULT, p_petsc_vector);
        UNUSED_OPT(err);
        assert(err==0);

        VecRestoreArray(data, &p_petsc_vector);

        H5Sclose(hyperslab_space);
        H5Sclose(memspace);
    }
}

bool Hdf5Dataset::getSphereRI(MapVecDouble &mvec)
{
  hsize_t dims_out[2], count[2], offset[2], dimsm[2];
  hid_t dataspace = H5Dget_space(this->sphere_dataset_); // dataspace handle
  int rank = H5Sget_simple_extent_ndims(dataspace);
  herr_t status_n = H5Sget_simple_extent_dims(dataspace, dims_out, NULL);
  herr_t status;
  offset[0] = 0;
  offset[1] = 0;
  count[0] = dims_out[0];
  count[1] = 4;
  double data_out[count[0]][count[1]];
  status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL);
  dimsm[0] = count[0];
  dimsm[1] = count[1];
  hid_t memspace;
  memspace = H5Screate_simple(RANK_OUT, dimsm, NULL);
  status = H5Dread(this->sphere_dataset_, H5T_NATIVE_DOUBLE, memspace, dataspace, H5P_DEFAULT, data_out);
  for (int i = 0; i < count[0]; i++)
  {
    std::vector< double > sphere_center(3);
    double ri;
    for (int j = 0; j < 3; j++)
    {
      sphere_center[j] = data_out[i][j];
    }
    for (int k = 3; k < 4; k++)
    {
      ri = data_out[i][k];
    }
    mvec.insert(std::pair< std::vector< double >, double >(sphere_center, ri));
   }
  return 0;
}

static void
ufo_hdf5_reader_read (UfoReader *reader,
                      UfoBuffer *buffer,
                      UfoRequisition *requisition,
                      guint roi_y,
                      guint roi_height,
                      guint roi_step)
{
    UfoHdf5ReaderPrivate *priv;
    gpointer data;
    hid_t dst_dataspace_id;
    hsize_t dst_dims[2];

    priv = UFO_HDF5_READER_GET_PRIVATE (reader);
    data = ufo_buffer_get_host_array (buffer, NULL);

    hsize_t offset[3] = { priv->current, roi_y, 0 };
    hsize_t count[3] = { 1, roi_height, requisition->dims[0] };

    dst_dims[0] = roi_height;
    dst_dims[1] = requisition->dims[0];
    dst_dataspace_id = H5Screate_simple (2, dst_dims, NULL);

    H5Sselect_hyperslab (priv->src_dataspace_id, H5S_SELECT_SET, offset, NULL, count, NULL);
    H5Dread (priv->dataset_id, H5T_NATIVE_FLOAT, dst_dataspace_id, priv->src_dataspace_id, H5P_DEFAULT, data);
    H5Sclose (dst_dataspace_id);

    priv->current++;
}

/*-------------------------------------------------------------------------
 * Function:	test_diag
 *
 * Purpose:	Reads windows diagonally across the dataset.  Each window is
 *		offset from the previous window by OFFSET in the x and y
 *		directions.  The reading ends after the (k,k) value is read
 *		where k is the maximum index in the dataset.
 *
 * Return:	Efficiency.
 *
 * Programmer:	Robb Matzke
 *              Friday, May 15, 1998
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static double
test_diag (int op, size_t cache_size, size_t io_size, size_t offset)
{
    hid_t	file, dset, mem_space, file_space;
    hsize_t	i, hs_size[2];
    hsize_t	nio = 0;
    hsize_t	hs_offset[2];
    signed char	*buf = calloc (1, (size_t)(SQUARE (io_size)));
#ifdef H5_WANT_H5_V1_4_COMPAT
    int		mdc_nelmts, rdcc_nelmts;
#else /* H5_WANT_H5_V1_4_COMPAT */
    int		mdc_nelmts;
    size_t	rdcc_nelmts;
#endif /* H5_WANT_H5_V1_4_COMPAT */
    double	w0;

    H5Pget_cache (fapl_g, &mdc_nelmts, &rdcc_nelmts, NULL, &w0);
#ifdef DIAG_W0
    w0 = DIAG_W0;
#endif
#ifdef DIAG_NRDCC
    rdcc_nelmts = DIAG_NRDCC;
#endif
    H5Pset_cache (fapl_g, mdc_nelmts, rdcc_nelmts,
		  cache_size*SQUARE (CH_SIZE), w0);
    file = H5Fopen (FILE_NAME, H5F_ACC_RDWR, fapl_g);
    dset = H5Dopen (file, "dset");
    file_space = H5Dget_space (dset);
    nio_g = 0;

    for (i=0, hs_size[0]=io_size; hs_size[0]==io_size; i+=offset) {
	hs_offset[0] = hs_offset[1] = i;
	hs_size[0] = hs_size[1] = MIN (io_size, CH_SIZE*DS_SIZE-i);
	mem_space = H5Screate_simple (2, hs_size, hs_size);
	H5Sselect_hyperslab (file_space, H5S_SELECT_SET, hs_offset, NULL,
			     hs_size, NULL);
	if (READ==op) {
	    H5Dread (dset, H5T_NATIVE_SCHAR, mem_space, file_space,
		     H5P_DEFAULT, buf);
	} else {
	    H5Dwrite (dset, H5T_NATIVE_SCHAR, mem_space, file_space,
		      H5P_DEFAULT, buf);
	}
	H5Sclose (mem_space);
	nio += hs_size[0]*hs_size[1];
	if (i>0) nio -= SQUARE (io_size-offset);
    }

    free (buf);
    H5Sclose (file_space);
    H5Dclose (dset);
    H5Fclose (file);

    /*
     * The extra cast in the following statement is a bug workaround for the
     * Win32 version 5.0 compiler.
     * 1998-11-06 ptl
     */
    return (double)(hssize_t)nio/(hssize_t)nio_g;
}

/**
   Append a vector to a specified dataset and return the error status
   of the write operation. */
herr_t HDF5DataWriter::appendToDataset(hid_t dataset_id, const vector< double >& data)
{
    herr_t status;
    if (dataset_id < 0){
        return -1;
    }
    hid_t filespace = H5Dget_space(dataset_id);
    if (filespace < 0){
        return -1;
    }
    if (data.size() == 0){
        return 0;
    }
    hsize_t size = H5Sget_simple_extent_npoints(filespace) + data.size();
    status = H5Dset_extent(dataset_id, &size);
    if (status < 0){
        return status;
    }
    filespace = H5Dget_space(dataset_id);
    hsize_t size_increment = data.size();
    hid_t memspace = H5Screate_simple(1, &size_increment, NULL);
    hsize_t start = size - data.size();
    H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, NULL, &size_increment, NULL);
    status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, &data[0]);
    return status;
}

void HDF5Output::flush() const {
	hsize_t n = buffer.size();

	if (n == 0)
		return;

	hid_t file_space = H5Dget_space(dset);
	hsize_t count = H5Sget_simple_extent_npoints(file_space);

	// resize dataset
	hsize_t new_size[RANK] = {count + n};
	H5Dset_extent(dset, new_size);

	// get updated filespace
	H5Sclose(file_space);
	file_space = H5Dget_space(dset);

	hsize_t offset[RANK] = {count};
	hsize_t cnt[RANK] = {n};

	H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, cnt, NULL);
	hid_t mspace_id = H5Screate_simple(RANK, cnt, NULL);

	H5Dwrite(dset, sid, mspace_id, file_space, H5P_DEFAULT, buffer.data());

	H5Sclose(mspace_id);
	H5Sclose(file_space);

	buffer.clear();
}

int getLinks(PIODataset dataset, link_t* links)
{
	ERROR_SWITCH_INIT
	herr_t read_err;
    
	hsize_t position[1] = {-1};
	hsize_t number[1] = {-1};
	hid_t dataspaceForLink = -1;
	hid_t bufferDataspaceForLink = -1;
	hid_t link_datatype = -1;
	
	// read link dataset
	position[0] = 0; // from first 'link'
	number[0] = dataset.ntimeranges;   // to last 'link'
	dataspaceForLink = H5Dget_space(dataset.link_identifier);
	H5Sselect_hyperslab(dataspaceForLink, H5S_SELECT_SET, position, NULL, number, NULL);
	bufferDataspaceForLink = H5Screate_simple(1, number, NULL);
	link_datatype = linkDatatype();
	ERROR_SWITCH_OFF
	read_err = H5Dread(dataset.link_identifier, link_datatype, bufferDataspaceForLink, dataspaceForLink, H5P_DEFAULT, links);
	ERROR_SWITCH_ON
	H5Tclose(link_datatype);
	H5Sclose(bufferDataspaceForLink);
	H5Sclose(dataspaceForLink);
	
	if (read_err < 0) return -1;
	return 1;
}

 // dataspace from lengths and strides. Correct for the complex. strides must be >0
 dataspace dataspace_from_LS(int R, bool is_complex, hsize_t const *Ltot, hsize_t const *L, hsize_t const *S,
                             hsize_t const *offset) {
  int rank = R + (is_complex ? 1 : 0);
  hsize_t totdimsf[rank], dimsf[rank], stridesf[rank], offsetf[rank]; // dataset dimensions
  for (size_t u = 0; u < R; ++u) {
   offsetf[u] = (offset ? offset[u] : 0);
   dimsf[u] = L[u];
   totdimsf[u] = Ltot[u];
   stridesf[u] = S[u];
  }
  if (is_complex) {
   offsetf[rank - 1] = 0;
   dimsf[rank - 1] = 2;
   totdimsf[rank - 1] = 2;
   stridesf[rank - 1] = 1;
  }

  dataspace ds = H5Screate_simple(rank, totdimsf, NULL);
  if (!ds.is_valid()) TRIQS_RUNTIME_ERROR << "Cannot create the dataset";

  herr_t err = H5Sselect_hyperslab(ds, H5S_SELECT_SET, offsetf, stridesf, dimsf, NULL);
  if (err < 0) TRIQS_RUNTIME_ERROR << "Cannot set hyperslab";

  return ds;
 }

/**
 * Appends along the last dimensions.
 */
static hid_t make_dataset(ndio_hdf5_t self,nd_type_id_t type_id,unsigned ndim,size_t *shape, hid_t* filespace)
{ hsize_t *sh=0,*ori=0,*ext=0;
  TRY(self->isw);
  STACK_ALLOC(hsize_t,sh ,ndim);
  STACK_ALLOC(hsize_t,ori,ndim);
  STACK_ALLOC(hsize_t,ext,ndim);
  if(self->dataset>=0) // data set already exists...needs extending, append on slowest dim
  { HTRY(H5Sget_simple_extent_dims(space(self),sh,NULL));
    ZERO(hsize_t,ori,ndim);
    ori[0]=sh[0];
    sh[0]+=shape[ndim-1];
    reverse_hsz_sz(ndim,ext,shape);
    HTRY(H5Dextend(self->dataset,sh));
    HTRY(*filespace=H5Dget_space(self->dataset));
    HTRY(H5Sselect_hyperslab(*filespace,H5S_SELECT_SET,ori,NULL,ext,NULL));
  } else
  { HTRY(self->dataset=H5Dcreate(
                       self->file,name(self),
                       nd_to_hdf5_type(type_id),
                       make_space(self,ndim,shape),
                       H5P_DEFAULT,/*(rare) link creation props*/
                       dataset_creation_properties(
                          /*set_deflate*/(
                          set_chunk(self,ndim,shape))),
                       H5P_DEFAULT /*(rare) dataset access props*/
                       ));
    reverse_hsz_sz(ndim,sh,shape);
    *filespace=H5S_ALL;
  }
  HTRY(H5Dset_extent(self->dataset,sh));
  return self->dataset;
Error:
  return -1;
}

//hyperslab write
void hdf5IoDataModel::writeHyperslab(const QString &dataset_name, DataType type, quint64 *offset, quint64 *stride, quint64 *count, quint64 *block, quint64 *values_shape, void *values)
{
    hid_t dataset_id = d->datasetId(dataset_name);

    //the selection within the file dataset's dataspace
    hid_t file_dataspace = H5Dget_space(dataset_id);
    if(H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, stride, count, block)<0) {
        dtkError() << "ERROR selecting hyperslab" << dataset_name;
    }

    //set the dimensions of values. memory dataspace and the selection within it
    hid_t values_dataspace = H5Screate_simple(H5Sget_simple_extent_ndims(file_dataspace), values_shape, NULL);

    switch(type) {
    case dtkIoDataModel::Int:
        // TODO put d->prop_list_id instead of H5P_DEFAULT ????????
        d->status = H5Dwrite(dataset_id, H5T_NATIVE_INT, values_dataspace, file_dataspace, H5P_DEFAULT, values);
        break;
    case dtkIoDataModel::LongLongInt:
        d->status = H5Dwrite(dataset_id, H5T_NATIVE_LLONG, values_dataspace, file_dataspace, H5P_DEFAULT, values);
        break;
    case dtkIoDataModel::Double:
        d->status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, values_dataspace, file_dataspace, H5P_DEFAULT, values);
        break;
    default:
        dtkError() << "write method: Datatype not supported";
    };
    if(d->status<0) {
        dtkError() << "error writing hyperslab" << dataset_name;
    }

    H5Sclose(file_dataspace);
    H5Sclose(values_dataspace);

}

void BigArray<T>::getMatrix(unsigned long startingRow, unsigned long startingCol, unsigned long numRows, unsigned long numCols, T* result) const
{
    std::string errorString("Error reading matrix data");

    hsize_t dims[2] = {numCols, numRows};
    hid_t memspace = H5Screate_simple(2, dims, NULL);
    CHECK_HDF5_ERR(memspace, errorString)

    hsize_t	count[2] = {1, 1};
    hsize_t	stride[2] = {1, 1};
    hsize_t	block[2] = {dims[0], dims[1]};
    hsize_t	offset[2] = {startingCol, startingRow};


    // Select hyperslab in the file.
    hid_t filespace = H5Dget_space(dset_id);
    CHECK_HDF5_ERR(filespace, errorString)

    herr_t status;

    status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, stride, count, block);
    CHECK_HDF5_ERR(status, errorString)

    status = H5Dread(dset_id, getHdfType<T>(), memspace, filespace, plist_id, result);
    CHECK_HDF5_ERR(status, errorString)

    status = H5Sclose(memspace);
    CHECK_HDF5_ERR(status, errorString)

    status = H5Sclose(filespace);
    CHECK_HDF5_ERR(status, errorString)
}

 int64_t GWriteHDFFile::WriteBlock(std::string BlockName, int type, void *data, int partlen, uint32_t np_write, uint32_t begin)
 {
           herr_t herr;
           hid_t handle = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
           hid_t group = H5Gopen2(handle, g_name[type], H5P_DEFAULT);
           if(group < 0)
               return group;
           hsize_t size[2];
           int rank=1;
           //Get type
           char b_type = get_block_type(BlockName);
           hid_t dtype;
           if(b_type == 'f') {
               size[1] = partlen/sizeof(float);
               dtype=H5T_NATIVE_FLOAT;
           }else if (b_type == 'i') {
               size[1] = partlen/sizeof(int64_t);
               //Hopefully this is 64 bits; the HDF5 manual is not clear.
               dtype = H5T_NATIVE_LLONG;
           }
           else{
               return -1000;
           }
           if (size[1] > 1) {
                   rank = 2;
           }
           /* I don't totally understand why the below works (it is not clear to me from the documentation).
            * I gleaned it from a posting to the HDF5 mailing list and a related stack overflow thread here:
            * http://stackoverflow.com/questions/24883461/hdf5-updating-a-cell-in-a-table-of-integers
            * http://lists.hdfgroup.org/pipermail/hdf-forum_lists.hdfgroup.org/2014-July/007966.html
            * The important thing seems to be that we have a dataspace for the whole array and create a hyperslab on that dataspace.
            * Then we need another dataspace with the size of the stuff we want to write.*/
           //Make space in memory for the whole array
           //Create a hyperslab that we will write to
           size[0] = npart[type];
           hid_t full_space_id = H5Screate_simple(rank, size, NULL);
           //If this is the first write, create the dataset
           if (begin==0) {
               H5Dcreate2(group,BlockName.c_str(),dtype, full_space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
           }
           hid_t dset = H5Dopen2(group,BlockName.c_str(),H5P_DEFAULT);
           if (dset < 0)
               return dset;
           size[0] = np_write;
           hid_t space_id = H5Screate_simple(rank, size, NULL);
           hsize_t begins[2]={begin,0};
           //Select the hyperslab of elements we are about to write to
           H5Sselect_hyperslab(full_space_id, H5S_SELECT_SET, begins, NULL, size, NULL);
           /* Write to the dataset */
           herr = H5Dwrite(dset, dtype, space_id, full_space_id, H5P_DEFAULT, data);
           H5Dclose(dset);
           H5Sclose(space_id);
           H5Sclose(full_space_id);
           H5Gclose(group);
           H5Fclose(handle);
           if (herr < 0)
               return herr;
           return np_write;
 }

/*+++++++++++++++++++++++++
.IDENTifer   PYTABLE_write_records
.PURPOSE     Write records to an HDF5 array
.INPUT/OUTPUT
  call as    stat = PYTABLE_write_records( locID, dset_name, start, step,
			                   count, buffer );
     input:
            hid_t locID      :  HDF5 identifier of file or group
	    char *dset_name  :  name of dataset
	    hsize_t *start   :  index of first row to overwrite
	    hsize_t *step    :  
	    hsize_t *count   :  number of rows to write
	    void *buffer     :  data to write
	    
.RETURNS     A negative value is returned on failure. 
.COMMENTS    none
-------------------------*/
herr_t PYTABLE_write_records( hid_t locID, const char *dset_name, 
			      hsize_t *start, hsize_t *step,
			      hsize_t *count, const void *buffer )
{
     int      rank;

     hid_t    dataID;
     hid_t    spaceID = -1;
     hid_t    mem_spaceID = -1;
     hid_t    typeID = -1;

/* open the dataset. */
     if ( (dataID = H5Dopen( locID, dset_name, H5P_DEFAULT )) < 0 )
	  return -1;

/* get the dataspace handle */
     if ( (spaceID = H5Dget_space( dataID )) < 0 )
	  goto done;

/* get rank */
     if ( (rank = H5Sget_simple_extent_ndims( spaceID )) <= 0 )
	  goto done;

/* create a simple memory data space */
     if ( (mem_spaceID = H5Screate_simple( rank, count, NULL )) < 0 )
	  goto done;

/* define a hyperslab in the dataset */
     if ( H5Sselect_hyperslab( spaceID, H5S_SELECT_SET, start,
			       step, count, NULL ) < 0 )
	  goto done;

/* get an identifier for the datatype. */
     if ( (typeID = H5Dget_type( dataID )) < 0 ) goto done;

/* write data to hyperslap */
     if ( H5Dwrite( dataID, typeID, mem_spaceID, spaceID, 
		    H5P_DEFAULT, buffer ) < 0 )
	  goto done;

/* terminate access to the datatype */
     if ( H5Tclose( typeID ) < 0 ) goto done;

/* end access to the dataset */
     if ( H5Dclose( dataID ) ) goto done;

/* terminate access to the dataspace */
     if ( H5Sclose( mem_spaceID ) < 0 ) goto done;
     if ( H5Sclose( spaceID ) < 0 ) goto done;

     return 0;
 done:
     if ( typeID > 0 ) (void) H5Tclose( typeID );
     if ( spaceID > 0 ) (void) H5Sclose( spaceID );
     if ( mem_spaceID > 0 ) (void) H5Sclose( mem_spaceID );
     if ( dataID > 0 ) (void) H5Dclose( dataID );
     return -1;
}

int_f
nh5sselect_hyperslab_c ( hid_t_f *space_id , int_f *op, hsize_t_f *start, hsize_t_f *count, hsize_t_f *stride, hsize_t_f *block)
{
  int ret_value = -1;
  hid_t c_space_id;
  hsize_t *c_start = NULL;
  hsize_t *c_count = NULL;
  hsize_t *c_stride = NULL;
  hsize_t *c_block = NULL;

  H5S_seloper_t c_op;
  herr_t  status;
  int rank;
  int i;

  rank = H5Sget_simple_extent_ndims(*space_id);
  if (rank < 0 ) return ret_value;
  c_start = (hsize_t *)HDmalloc(sizeof(hsize_t)*rank);
  if (c_start == NULL) goto DONE;

  c_count = (hsize_t *)HDmalloc(sizeof(hsize_t)*rank);
  if (c_count == NULL) goto DONE;

  c_stride = (hsize_t *)HDmalloc(sizeof(hsize_t)*rank);
  if (c_stride == NULL) goto DONE;

  c_block = (hsize_t *)HDmalloc(sizeof(hsize_t)*rank);
  if (c_block == NULL) goto DONE;


  /*
   * Reverse dimensions due to C-FORTRAN storage order.
   */

  for (i=0; i < rank; i++) {
      int t= (rank - i) - 1;
      c_start[i] = (hsize_t)start[t];
      c_count[i] = (hsize_t)count[t];
      c_stride[i] = (hsize_t)stride[t];
      c_block[i] = (hsize_t)block[t];
  }

   c_op = (H5S_seloper_t)*op;
/*
  if (*op == H5S_SELECT_SET_F) c_op = H5S_SELECT_SET;
  if (*op == H5S_SELECT_OR_F)  c_op = H5S_SELECT_OR;
*/

  c_space_id = *space_id;
  status = H5Sselect_hyperslab(c_space_id, c_op, c_start, c_stride, c_count, c_block);
  if ( status >= 0  ) ret_value = 0;
DONE:
  if(c_start != NULL) HDfree(c_start);
  if(c_count != NULL) HDfree(c_count);
  if(c_stride!= NULL) HDfree(c_stride);
  if(c_block != NULL) HDfree(c_block);
  return ret_value;
}