void _io_write_prim_h5mpi(const char *fname, const char **pnames, const double *data)
// -----------------------------------------------------------------------------
// This function uses a collective MPI-IO procedure to write the contents of
// 'data' to the HDF5 file named 'fname', which is assumed to have been created
// already. The dataset with name 'dname', which is being written to, must not
// exist already. Chunking is enabled as per the module-wide ChunkSize variable,
// and is disabled by default. Recommended chunk size is local subdomain
// size. This will result in optimized read/write on the same decomposition
// layout, but poor performance for different access patterns, for example the
// slabs used by cluster-FFT functions.
//
// WARNING!
//
// All processors must define the same chunk size, the behavior of this function
// is not defined otherwise. This implies that chunking should be disabled when
// running on a strange number of cores, and subdomain sizes are non-uniform.
// -----------------------------------------------------------------------------
{
hsize_t ndp1 = n_dims + 1;
hsize_t *a_nint = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *l_ntot = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *l_strt = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *stride = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
int i;
for (i=0; i<n_dims; ++i) {
a_nint[i] = A_nint[i]; // Selection size, target and destination
l_ntot[i] = L_ntot[i]; // Memory space total size
l_strt[i] = L_strt[i]; // Memory space selection start
stride[i] = 1;
}
a_nint[ndp1 - 1] = 1;
l_ntot[ndp1 - 1] = n_prim;
stride[ndp1 - 1] = n_prim;
// Here we create the following property lists:
//
// file access property list ........ for the call to H5Fopen
// dset creation property list ........ for the call to H5Dcreate
// dset transfer property list ........ for the call to H5Dwrite
// ---------------------------------------------------------------------------
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
hid_t dxpl = H5Pcreate(H5P_DATASET_XFER);
// Here we define collective (MPI) access to the file with alignment
// properties optimized for the local file system, according to DiskBlockSize.
// ---------------------------------------------------------------------------
if (EnableChunking) {
H5Pset_chunk(dcpl, n_dims, ChunkSize);
}
if (EnableAlignment) {
H5Pset_alignment(fapl, AlignThreshold, DiskBlockSize);
}
H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
hid_t file = H5Fopen(fname, H5F_ACC_RDWR, fapl);
const int overwrite = H5Lexists(file, "prim", H5P_DEFAULT);
hid_t prim = overwrite ? H5Gopen(file, "prim", H5P_DEFAULT) :
H5Gcreate(file, "prim", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
hid_t mspc = H5Screate_simple(ndp1 , l_ntot, NULL);
hid_t fspc = H5Screate_simple(n_dims, G_ntot, NULL);
// Call signature to H5Sselect_hyperslab is (start, stride, count, chunk)
// ---------------------------------------------------------------------------
const clock_t start_all = clock();
for (i=0; i<n_prim; ++i) {
hid_t dset = overwrite ? H5Dopen(prim, pnames[i], H5P_DEFAULT) :
H5Dcreate(prim, pnames[i], H5T_NATIVE_DOUBLE, fspc,
H5P_DEFAULT, dcpl, H5P_DEFAULT);
l_strt[ndp1 - 1] = i;
H5Sselect_hyperslab(mspc, H5S_SELECT_SET, l_strt, stride, a_nint, NULL);
H5Sselect_hyperslab(fspc, H5S_SELECT_SET, G_strt, NULL, A_nint, NULL);
H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspc, fspc, dxpl, data);
H5Dclose(dset);
}
if (iolog) {
const double sec = (double)(clock() - start_all) / CLOCKS_PER_SEC;
fprintf(iolog, "[h5mpi] write to %s took %f minutes\n", fname, sec/60.0);
fflush(iolog);
}
free(a_nint);
free(l_ntot);
free(l_strt);
// Always close the hid_t handles in the reverse order they were opened in.
// ---------------------------------------------------------------------------
H5Sclose(fspc);
H5Sclose(mspc);
H5Gclose(prim);
H5Fclose(file);
H5Pclose(dxpl);
H5Pclose(dcpl);
H5Pclose(fapl);
}
开发者ID:darien0，项目名称:Mara，代码行数:100，代码来源:h5mpi.c
示例5: FTI_WriteHDF5Var
int FTI_WriteHDF5Var(FTIT_dataset *FTI_DataVar)
{
int j;
hsize_t dimLength[32];
char str[FTI_BUFS];
int res;
hid_t dcpl;
for (j = 0; j < FTI_DataVar->rank; j++) {
dimLength[j] = FTI_DataVar->dimLength[j];
}
dcpl = H5Pcreate (H5P_DATASET_CREATE);
res = H5Pset_fletcher32 (dcpl);
res = H5Pset_chunk (dcpl, FTI_DataVar->rank, dimLength);
hid_t dataspace = H5Screate_simple( FTI_DataVar->rank, dimLength, NULL);
hid_t dataset = H5Dcreate2 ( FTI_DataVar->h5group->h5groupID, FTI_DataVar->name,FTI_DataVar->type->h5datatype, dataspace, H5P_DEFAULT, dcpl , H5P_DEFAULT);
// If my data are stored in the CPU side
// Just store the data to the file and return;
#ifdef GPUSUPPORT
if ( !FTI_DataVar->isDevicePtr ){
#endif
res = H5Dwrite(dataset,FTI_DataVar->type->h5datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, FTI_DataVar->ptr);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Pclose (dcpl);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Dclose(dataset);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Sclose(dataspace);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
return FTI_SCES;
#ifdef GPUSUPPORT
}
// This code is only executed in the GPU case.
hsize_t *count = (hsize_t*) malloc (sizeof(hsize_t)*FTI_DataVar->rank);
hsize_t *offset= (hsize_t*) calloc (FTI_DataVar->rank,sizeof(hsize_t));
if ( !count|| !offset){
sprintf(str, "Could Not allocate count and offset regions");
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
hsize_t seperator;
hsize_t fetchBytes = FTI_getHostBuffSize();
fetchBytes = FTI_calculateCountDim(FTI_DataVar->eleSize, fetchBytes ,count, FTI_DataVar->rank, dimLength, &seperator);
sprintf(str,"GPU-Device Message: I Will Fetch %lld Bytes Per Stream Request", fetchBytes);
FTI_Print(str,FTI_DBUG);
FTIT_data_prefetch prefetcher;
prefetcher.fetchSize = fetchBytes;
prefetcher.totalBytesToFetch = FTI_DataVar->size;
prefetcher.isDevice = FTI_DataVar->isDevicePtr;
prefetcher.dptr = FTI_DataVar->devicePtr;
size_t bytesToWrite;
FTI_InitPrefetcher(&prefetcher);
unsigned char *basePtr = NULL;
if ( FTI_Try(FTI_getPrefetchedData(&prefetcher, &bytesToWrite, &basePtr), "Fetch next memory block from GPU to write to HDF5") != FTI_SCES){
return FTI_NSCS;
}
while( basePtr ){
res = FTI_WriteElements( dataspace, FTI_DataVar->type->h5datatype, dataset, count, offset, FTI_DataVar->rank , basePtr);
if (res != FTI_SCES ) {
free(offset);
free(count);
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
FTI_AdvanceOffset(seperator, offset,count, dimLength, FTI_DataVar->rank);
if ( FTI_Try(FTI_getPrefetchedData(&prefetcher, &bytesToWrite, &basePtr),
"Fetch next memory block from GPU to write to HDF5") != FTI_SCES){
//.........这里部分代码省略.........
开发者ID:leobago，项目名称:fti，代码行数:101，代码来源:hdf5.c
示例6: create_deflate_dsets_float
/*-------------------------------------------------------------------------
* Function: create_deflate_dsets_float
*
* Purpose: Create a dataset of FLOAT datatype with deflate filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 29 March 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_deflate_dsets_float(hid_t fid, hid_t fsid, hid_t msid)
{
#ifdef H5_HAVE_FILTER_DEFLATE
hid_t dataset = -1; /* dataset handles */
hid_t dcpl = -1;
float data[NX][NY]; /* data to write */
float fillvalue = -2.2f;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = ((float)(i + j + 1))/3;
}
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_deflate (dcpl, 6) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_FLOAT, &fillvalue) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace, little
* endian datatype and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME16, H5T_IEEE_F32LE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if((dataset = H5Dcreate2(fid, DATASETNAME17, H5T_IEEE_F32BE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
#else /* H5_HAVE_FILTER_DEFLATE */
const char *not_supported= "Deflate filter is not enabled. Can't create the dataset.";
puts(not_supported);
#endif /* H5_HAVE_FILTER_DEFLATE */
return 0;
#ifdef H5_HAVE_FILTER_DEFLATE
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
#endif /* H5_HAVE_FILTER_DEFLATE */
}
开发者ID:Starlink，项目名称:hdf5，代码行数:98，代码来源:gen_cross.c
示例7: main
//.........这里部分代码省略.........
int ncid, grpid, nvars, ngatts, ndims, unlimdimid, ngrps;
char name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
int ndims_in, natts_in, dimid_in[NDIMS];
/* nc_set_log_level(5);*/
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 2 || nvars != 0 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_grps(ncid, &ngrps, &grpid)) ERR;
if (ngrps != 1) ERR;
if (nc_inq(grpid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 0 || nvars != 1 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, VAR_NAME) || xtype_in != NC_SHORT || ndims_in != NDIMS ||
dimid_in[0] != 0 || dimid_in[1] != 1 || natts_in != 0) ERR;
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
printf("*** testing HDF5 compatibility with szip...");
{
#define DEFLATE_LEVEL 9
#define MAX_NAME 100
#define NUM_CD_ELEM 10
/* HDF5 defines this... */
#define DEFLATE_NAME "deflate"
#define DIM1_LEN 3000
#define GRP_NAME "George_Washington"
#define BATTLE_RECORD "Battle_Record"
hid_t fileid, grpid, spaceid, datasetid;
int data_out[DIM1_LEN], data_in[DIM1_LEN];
hsize_t dims[1] = {DIM1_LEN};
hid_t propid;
char name_in[MAX_NAME + 1];
int ncid, ndims_in, nvars_in, ngatts_in, unlimdimid_in, ngrps_in;
int nc_grpid;
int dimid_in[1], natts_in;
nc_type xtype_in;
int i;
for (i = 0; i < DIM1_LEN; i++)
data_out[i] = i;
/* Open file and create group. */
if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if ((grpid = H5Gcreate(fileid, GRP_NAME, 0)) < 0) ERR;
/* Write an array of bools, with szip compression. */
if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
if (H5Pset_chunk(propid, 1, dims)) ERR;
if (H5Pset_szip(propid, H5_SZIP_EC_OPTION_MASK, 32)) ERR;
if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
if ((datasetid = H5Dcreate(grpid, BATTLE_RECORD, H5T_NATIVE_INT,
spaceid, propid)) < 0) ERR;
if (H5Dwrite(datasetid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
data_out) < 0) ERR;
if (H5Dclose(datasetid) < 0 ||
H5Pclose(propid) < 0 ||
H5Sclose(spaceid) < 0 ||
H5Gclose(grpid) < 0 ||
H5Fclose(fileid) < 0)
ERR;
/* Open the file with netCDF and check it. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 0 || nvars_in != 0 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
if (nc_inq_grps(ncid, &ngrps_in, &nc_grpid)) ERR;
if (ngrps_in != 1) ERR;
if (nc_inq(nc_grpid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 1 || nvars_in != 1 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
/* Check the variable. */
if (nc_inq_var(nc_grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, BATTLE_RECORD) || xtype_in != NC_INT || ndims_in != 1 ||
dimid_in[0] != 0 || natts_in != 0) ERR;
/* Check the data. */
if (nc_get_var(nc_grpid, 0, data_in)) ERR;
for (i = 0; i < DIM1_LEN; i++)
if (data_in[i] != data_out[i]) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
FINAL_RESULTS;
}
客服电话
APP下载
官方微信
请发表评论