void ADIOI_SCI_WriteContig(ADIO_File fd, void *buf, int count, 
                   MPI_Datatype datatype, int file_ptr_type,
	           ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    int err=-1, datatype_size, len;
#ifndef PRINT_ERR_MSG
    static char myname[] = "ADIOI_SCI_WRITECONTIG";
#endif

    MPI_Type_size(datatype, &datatype_size);
    len = datatype_size * count;

    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
	if (fd->fp_sys_posn != offset)
	    lseek(fd->fd_sys, offset, SEEK_SET);
	err = write(fd->fd_sys, buf, len);
	fd->fp_sys_posn = offset + err;
	/* individual file pointer not updated */        
    } else { 
	/* write from curr. location of ind. file pointer */
	if (fd->fp_sys_posn != fd->fp_ind)
	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
	err = write(fd->fd_sys, buf, len);
	fd->fp_ind += err;
	fd->fp_sys_posn = fd->fp_ind;
    }

#ifdef HAVE_STATUS_SET_BYTES
    if (err != -1) 
	MPIR_Status_set_bytes(status, datatype, err);
#endif

#ifdef PRINT_ERR_MSG
    *error_code = (err == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
#else
    if (err == -1) {
	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
			      myname, "I/O Error", "%s", strerror(errno));
	ADIOI_Error(fd, *error_code, myname);
    }
    else *error_code = MPI_SUCCESS;
#endif
}

int ADIOI_PFS_ReadDone(ADIO_Request *request, ADIO_Status *status,
		       int *error_code)  
{
    int done=0;
    static char myname[] = "ADIOI_PFS_READDONE";

    if (*request == ADIO_REQUEST_NULL) {
        *error_code = MPI_SUCCESS;
        return 1;
    }

    if ((*request)->queued)
	done = _iodone(*((long *) (*request)->handle));
    else done = 1; /* ADIOI_Complete_Async completed this request, 
                      but request object was not freed. */

#ifdef HAVE_STATUS_SET_BYTES
    if ((done >= 0) && ((*request)->nbytes != -1))
	MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes);
#endif

    if (done >= 0) {
        /* if request is still queued in the system, it is also there
           on ADIOI_Async_list. Delete it from there. */
        if ((*request)->queued) ADIOI_Del_req_from_list(request);

        (*request)->fd->async_count--;
        if ((*request)->handle) ADIOI_Free((*request)->handle);
        ADIOI_Free_request((ADIOI_Req_node *) (*request));
        *request = ADIO_REQUEST_NULL;
    }
    
    if (done == -1 && errno != 0) {
	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
					   myname, __LINE__, MPI_ERR_IO,
					   "**io",
					   "**io %s", strerror(errno));
    }
    else *error_code = MPI_SUCCESS;
    return done;
}

void ADIOI_PFS_WriteContig(ADIO_File fd, void *buf, int count,
			   MPI_Datatype datatype, int file_ptr_type,
			   ADIO_Offset offset, ADIO_Status *status,
			   int *error_code)
{
    MPI_Count err=-1, datatype_size, len;
    static char myname[] = "ADIOI_PFS_WRITECONTIG";

    MPI_Type_size_x(datatype, &datatype_size);
    len = datatype_size * count;

    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
        if (fd->fp_sys_posn != offset) {
            lseek(fd->fd_sys, offset, SEEK_SET);
	}
        err = _cwrite(fd->fd_sys, buf, len);
        fd->fp_sys_posn = offset + err;
         /* individual file pointer not updated */
    }
    else { /* write from curr. location of ind. file pointer */
        if (fd->fp_sys_posn != fd->fp_ind) {
            lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
	}
        err = _cwrite(fd->fd_sys, buf, len);
        fd->fp_ind += err;
        fd->fp_sys_posn = fd->fp_ind;
    }

#ifdef HAVE_STATUS_SET_BYTES
    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
#endif

    if (err == -1) {
	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
					   myname, __LINE__, MPI_ERR_IO,
					   "**io",
					   "**io %s", strerror(errno));
    }
    else *error_code = MPI_SUCCESS;
}

int ADIOI_NTFS_ReadDone(ADIO_Request *request, ADIO_Status *status,
			int *error_code)
{
    DWORD ret_val;
    int done = 0;
    static char myname[] = "ADIOI_NTFS_ReadDone";

    if (*request == ADIO_REQUEST_NULL)
    {
	*error_code = MPI_SUCCESS;
	return 1;
    }

    if ((*request)->queued) 
    {
	(*request)->nbytes = 0;
	ret_val = GetOverlappedResult((*request)->fd, (*request)->handle, &(*request)->nbytes, FALSE);

	if (!ret_val)
	{
	    /* --BEGIN ERROR HANDLING-- */
	    ret_val = GetLastError();
	    if (ret_val == ERROR_IO_INCOMPLETE)
	    {
		done = 0;
		*error_code = MPI_SUCCESS;
	    }
	    else
	    {
		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io", "**io %s", ADIOI_NTFS_Strerror(ret_val));
	    }
	    /* --END ERROR HANDLING-- */
	}
	else 
	{
	    done = 1;		
	    *error_code = MPI_SUCCESS;
	}
    }
    else
    {
	done = 1;
	*error_code = MPI_SUCCESS;
    }
#ifdef HAVE_STATUS_SET_BYTES
    if (done && ((*request)->nbytes != -1))
	MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes);
#endif
    
    if (done) 
    {
	/* if request is still queued in the system, it is also there
	   on ADIOI_Async_list. Delete it from there. */
	if ((*request)->queued) ADIOI_Del_req_from_list(request);
	
	(*request)->fd->async_count--;
	if ((*request)->handle) 
	{
	    if (!CloseHandle(((OVERLAPPED*)((*request)->handle))->hEvent))
	    {
		ret_val = GetLastError();
		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io", "**io %s", ADIOI_NTFS_Strerror(ret_val));
	    }
	    ADIOI_Free((*request)->handle);
	}
	ADIOI_Free_request((ADIOI_Req_node *) (*request));
	*request = ADIO_REQUEST_NULL;
    }
    return done;
}

//.........这里部分代码省略.........
			/* in case last read list call fills max arrays */
			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
		    }

		    pvfs_write_list(fd->fd_sys ,mem_list_count, mem_offsets,
				   mem_lengths, file_list_count,
				   &file_offsets, &file_lengths);
		  
		    /* in the case of the last read list call, leave here */
		    if (b_blks_wrote == total_blks_to_write) break;

		    file_offsets += file_lengths;
		    file_lengths = 0;
		} 
	    } /* for (i=0; i<flat_buf->count; i++) */
	    j++;
	} /* while (b_blks_wrote < total_blks_to_write) */
	ADIOI_Free(mem_offsets);
	ADIOI_Free(mem_lengths);

	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;

	if (err_flag) {
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
					       MPIR_ERR_RECOVERABLE, myname,
					       __LINE__, MPI_ERR_IO, "**io",
					       "**io %s", strerror(errno));
	}
	else *error_code = MPI_SUCCESS;

	fd->fp_sys_posn = -1;   /* clear this. */

#ifdef HAVE_STATUS_SET_BYTES
	MPIR_Status_set_bytes(status, datatype, bufsize);
/* This is a temporary way of filling in status. The right way is to 
   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
#endif

	ADIOI_Delete_flattened(datatype);
	return;
    } /* if (!buftype_is_contig && filetype_is_contig) */

    /* already know that file is noncontiguous from above */
    /* noncontiguous in file */

/* filetype already flattened in ADIO_Open */
    flat_file = ADIOI_Flatlist;
    while (flat_file->type != fd->filetype) flat_file = flat_file->next;

    disp = fd->disp;

    /* for each case - ADIO_Individual pointer or explicit, find offset
       (file offset in bytes), n_filetypes (how many filetypes into file 
       to start), fwr_size (remaining amount of data in present file
       block), and st_index (start point in terms of blocks in starting
       filetype) */
    if (file_ptr_type == ADIO_INDIVIDUAL) {
        offset = fd->fp_ind; /* in bytes */
	n_filetypes = -1;
	flag = 0;
	while (!flag) {
	    n_filetypes++;
	    for (i=0; i<flat_file->count; i++) {
	        if (disp + flat_file->indices[i] + 
		    (ADIO_Offset) n_filetypes*filetype_extent +
		      flat_file->blocklens[i] >= offset) {

//.........这里部分代码省略.........
     * will be accessed by this process.
     * count_others_req_procs = number of processes whose requests (including
     * this process itself) will be accessed by this process
     * count_others_req_per_proc[i] indicates how many separate contiguous
     * requests of proc. i will be accessed by this process.
     */

    ADIOI_Calc_others_req(fd, count_my_req_procs, count_my_req_per_proc,
                          my_req, nprocs, myrank, &count_others_req_procs,
                          &others_req);
    ADIOI_Free(count_my_req_per_proc);

    /* exchange data and write in sizes of no more than stripe_size. */
    ADIOI_LUSTRE_Exch_and_write(fd, buf, datatype, nprocs, myrank,
                                others_req, my_req, offset_list, len_list,
                                contig_access_count, striping_info,
                                buf_idx, error_code);

    /* If this collective write is followed by an independent write,
     * it's possible to have those subsequent writes on other processes
     * race ahead and sneak in before the read-modify-write completes.
     * We carry out a collective communication at the end here so no one
     * can start independent i/o before collective I/O completes.
     *
     * need to do some gymnastics with the error codes so that if something
     * went wrong, all processes report error, but if a process has a more
     * specific error code, we can still have that process report the
     * additional information */

    old_error = *error_code;
    if (*error_code != MPI_SUCCESS)
	*error_code = MPI_ERR_IO;

    /* optimization: if only one process performing i/o, we can perform
     * a less-expensive Bcast  */
#ifdef ADIOI_MPE_LOGGING
    MPE_Log_event(ADIOI_MPE_postwrite_a, 0, NULL);
#endif
    if (fd->hints->cb_nodes == 1)
	MPI_Bcast(error_code, 1, MPI_INT,
		  fd->hints->ranklist[0], fd->comm);
    else {
	tmp_error = *error_code;
	MPI_Allreduce(&tmp_error, error_code, 1, MPI_INT,
		      MPI_MAX, fd->comm);
    }
#ifdef ADIOI_MPE_LOGGING
    MPE_Log_event(ADIOI_MPE_postwrite_b, 0, NULL);
#endif

    if ((old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO))
	*error_code = old_error;


    if (!buftype_is_contig)
	ADIOI_Delete_flattened(datatype);

    /* free all memory allocated for collective I/O */
    /* free others_req */
    for (i = 0; i < nprocs; i++) {
	if (others_req[i].count) {
	    ADIOI_Free(others_req[i].offsets);
	    ADIOI_Free(others_req[i].lens);
	    ADIOI_Free(others_req[i].mem_ptrs);
	}
    }
    ADIOI_Free(others_req);
    /* free my_req here */
    for (i = 0; i < nprocs; i++) {
	if (my_req[i].count) {
	    ADIOI_Free(my_req[i].offsets);
	    ADIOI_Free(my_req[i].lens);
	}
    }
    ADIOI_Free(my_req);
    for (i = 0; i < nprocs; i++) {
        ADIOI_Free(buf_idx[i]);
    }
    ADIOI_Free(buf_idx);
    ADIOI_Free(offset_list);
    ADIOI_Free(len_list);
    ADIOI_Free(st_offsets);
    ADIOI_Free(end_offsets);
    ADIOI_Free(striping_info);

#ifdef HAVE_STATUS_SET_BYTES
    if (status) {
	MPI_Count bufsize, size;
	/* Don't set status if it isn't needed */
	MPI_Type_size_x(datatype, &size);
	bufsize = size * count;
	MPIR_Status_set_bytes(status, datatype, bufsize);
    }
    /* This is a temporary way of filling in status. The right way is to
     * keep track of how much data was actually written during collective I/O.
     */
#endif

    fd->fp_sys_posn = -1;	/* set it to null. */
}

void ADIOI_XFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int *error_code)  
{
    int err;
#ifndef PRINT_ERR_MSG
    static char myname[] = "ADIOI_XFS_READCOMPLETE";
#endif

    if (*request == ADIO_REQUEST_NULL) {
	*error_code = MPI_SUCCESS;
	return;
    }

    if ((*request)->queued) {
	do {
	    err = aio_suspend64((const aiocb64_t **) &((*request)->handle), 1, 0);
	} while ((err == -1) && (errno == EINTR));

	if (err != -1) {
	    err = aio_return64((aiocb64_t *) (*request)->handle); 
	    (*request)->nbytes = err;
	    errno = aio_error64((aiocb64_t *) (*request)->handle);
	}
	else (*request)->nbytes = -1;

#ifdef PRINT_ERR_MSG
	*error_code = (err == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
#else
	if (err == -1) {
	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
			  myname, "I/O Error", "%s", strerror(errno));
	    ADIOI_Error((*request)->fd, *error_code, myname);	    
	}
	else *error_code = MPI_SUCCESS;
#endif
    }
    else *error_code = MPI_SUCCESS;

#ifdef HAVE_STATUS_SET_BYTES
    if ((*request)->nbytes != -1)
	MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes);
#endif

    if ((*request)->queued != -1) {

	/* queued = -1 is an internal hack used when the request must
	   be completed, but the request object should not be
	   freed. This is used in ADIOI_Complete_async, because the user
	   will call MPI_Wait later, which would require status to
	   be filled. Ugly but works. queued = -1 should be used only
	   in ADIOI_Complete_async. 
           This should not affect the user in any way. */

	/* if request is still queued in the system, it is also there
           on ADIOI_Async_list. Delete it from there. */
	if ((*request)->queued) ADIOI_Del_req_from_list(request);

	(*request)->fd->async_count--;
	if ((*request)->handle) ADIOI_Free((*request)->handle);
	ADIOI_Free_request((ADIOI_Req_node *) (*request));
	*request = ADIO_REQUEST_NULL;
    }
}

void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count,
                     MPI_Datatype datatype, int file_ptr_type,
		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    ssize_t err=-1;
    MPI_Count datatype_size, len;
    ADIO_Offset bytes_xfered=0;
    size_t rd_count;
    static char myname[] = "ADIOI_NFS_READCONTIG";
    char *p;

    MPI_Type_size_x(datatype, &datatype_size);
    len = datatype_size * count;

    if (file_ptr_type == ADIO_INDIVIDUAL) {
	offset = fd->fp_ind;
    }

    p = buf;
    while (bytes_xfered < len ) {
        rd_count = len - bytes_xfered;
        /* FreeBSD and Darwin workaround: bigger than INT_MAX is an error */
        if (rd_count > INT_MAX)
            rd_count = INT_MAX;
	if (fd->atomicity)
            ADIOI_WRITE_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
        else ADIOI_READ_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
#endif
        err = pread(fd->fd_sys, p, rd_count, offset+bytes_xfered);
        /* --BEGIN ERROR HANDLING-- */
        if (err == -1) {
            *error_code = MPIO_Err_create_code(MPI_SUCCESS,
        	    MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO,
        	    "**io", "**io %s", strerror(errno));
        }
        /* --END ERROR HANDLING-- */
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
#endif
        ADIOI_UNLOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
        if (err == 0) {
            /* end of file */
            break;
        }
        bytes_xfered += err;
        p += err;
    }

    fd->fp_sys_posn = offset + bytes_xfered;
    if (file_ptr_type == ADIO_INDIVIDUAL) {
        fd->fp_ind += bytes_xfered;
    }
    /* --END ERROR HANDLING-- */

#ifdef HAVE_STATUS_SET_BYTES
    if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
#endif

    *error_code = MPI_SUCCESS;
}

void ADIOI_GEN_ReadContig(ADIO_File fd, void *buf, int count, 
			  MPI_Datatype datatype, int file_ptr_type,
			  ADIO_Offset offset, ADIO_Status *status,
			  int *error_code)
{
    off_t err_lseek = -1;
    ssize_t err = -1;
    MPI_Count datatype_size;
    ADIO_Offset len, bytes_xfered=0;
    size_t rd_count;
    static char myname[] = "ADIOI_GEN_READCONTIG";
    char *p;

#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5034, 0, NULL);
#endif
    MPI_Type_size_x(datatype, &datatype_size);
    len = datatype_size * (ADIO_Offset)count;

    if (file_ptr_type == ADIO_INDIVIDUAL) {
	offset = fd->fp_ind;
    }

    if (fd->fp_sys_posn != offset) {
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
#endif
	err_lseek = lseek(fd->fd_sys, offset, SEEK_SET);
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
#endif
	/* --BEGIN ERROR HANDLING-- */
	if (err_lseek == -1) {
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
					       MPIR_ERR_RECOVERABLE,
					       myname, __LINE__,
					       MPI_ERR_IO, "**io",
					       "**io %s", strerror(errno));
	    fd->fp_sys_posn = -1;
	    return;
	}
	/* --END ERROR HANDLING-- */
    }

    p=buf;
    while (bytes_xfered < len) {
#ifdef ADIOI_MPE_LOGGING
	MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
#endif
	rd_count = len - bytes_xfered;
	err = read(fd->fd_sys, p, rd_count);
	/* --BEGIN ERROR HANDLING-- */
	if (err == -1) {
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
		    MPIR_ERR_RECOVERABLE,
		    myname, __LINE__,
		    MPI_ERR_IO, "**io",
		    "**io %s", strerror(errno));
	    fd->fp_sys_posn = -1;
	    return;
	}
	/* --END ERROR HANDLING-- */
	if (err == 0) {
	    /* end of file */
	    break;
	}

#ifdef ADIOI_MPE_LOGGING
	MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
#endif
	bytes_xfered += err;
	p += err;
    }

    fd->fp_sys_posn = offset + bytes_xfered;

    if (file_ptr_type == ADIO_INDIVIDUAL) {
	fd->fp_ind += bytes_xfered; 
    }

#ifdef HAVE_STATUS_SET_BYTES
    /* what if we only read half a datatype? */
    /* bytes_xfered could be larger than int */
    if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
#endif

    *error_code = MPI_SUCCESS;
#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5035, 0, NULL);
#endif
}

//.........这里部分代码省略.........
			{
			    cur_flat_file_reg_off = fd->fp_ind -
				(fd->disp + flat_file_p->indices[i] +
				 (num_filetypes * filetype_extent));
			    flag = 1;
			    break;
			}
			else
			    bytes_into_filetype += flat_file_p->blocklens[i];
		    }
		}
	    }
	    /* Impossible that we don't find it in this datatype */
	    assert(i != flat_file_p->count);
	    off = bytes_into_filetype + cur_flat_file_reg_off;
	}
    }
    else /* ADIO_EXPLICIT */
    {
	off = etype_size * offset;
    }

#ifdef DEBUG_DTYPE
    fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: (fd->fp_ind=%Ld,fd->disp=%Ld,"
	    " offset=%Ld),(pvfs_disp=%Ld,off=%Ld)\n",
	    fd->fp_ind, fd->disp, offset, pvfs_disp, off);
#endif


    /* Convert the MPI memory and file datatypes into
     * PVFS2 datatypes */
    ret = convert_mpi_pvfs2_dtype(&datatype, &tmp_mem_req);
    if (ret < 0)
    {
	goto error_state;
    }
    ret = convert_mpi_pvfs2_dtype(&(fd->filetype), &tmp_file_req);
    if (ret < 0)
    {
	goto error_state;
    }

    ret = PVFS_Request_contiguous(count, tmp_mem_req, &mem_req);
    if (ret != 0) /* TODO: convert this to MPIO error handling */
        fprintf(stderr, "ADIOI_PVFS2_stridedDtypeIO: error in final"
		" CONTIG memory type\n");
    PVFS_Request_free(&tmp_mem_req);

    /* pvfs_disp is used to offset the filetype */
    ret = PVFS_Request_hindexed(1, &pvfs_blk, &pvfs_disp,
                                tmp_file_req, &file_req);
    if (ret != 0)
        fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: error in final"
			" HINDEXED file type\n");
    PVFS_Request_free(&tmp_file_req);

    if (rw_type == READ)
	ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, off, buf,
			    mem_req, &(pvfs_fs->credentials), &resp_io);
    else
	ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, off, buf,
			     mem_req, &(pvfs_fs->credentials), &resp_io);

    if (ret != 0) {
	fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: Warning - PVFS_sys_"
		"read/write returned %d and completed %Ld bytes.\n",
		ret, (long long)resp_io.total_completed);
        *error_code = MPIO_Err_create_code(MPI_SUCCESS,
                                           MPIR_ERR_RECOVERABLE,
                                           myname, __LINE__,
                                           ADIOI_PVFS2_error_convert(ret),
                                           "Error in PVFS_sys_io \n", 0);
        goto error_state;
    }

    if (file_ptr_type == ADIO_INDIVIDUAL)
    {
        fd->fp_ind = off += resp_io.total_completed;
    }

  error_state:
    fd->fp_sys_posn = -1;   /* set it to null. */

    PVFS_Request_free(&mem_req);
    PVFS_Request_free(&file_req);

#ifdef DEBUG_DTYPE
    fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: "
            "resp_io.total_completed=%Ld,ret=%d\n",
	    resp_io.total_completed, ret);
#endif

#ifdef HAVE_STATUS_SET_BYTES
    MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
    /* This is a temporary way of filling in status. The right way is to
     * keep track of how much data was actually acccessed by
     * ADIOI_BUFFERED operations */
#endif
    return ret;
}

void ADIOI_XFS_WriteContig(ADIO_File fd, void *buf, int count, 
                     MPI_Datatype datatype, int file_ptr_type,
		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    int err=-1, datatype_size, len, diff, size, nbytes;
    void *newbuf;
    static char myname[] = "ADIOI_XFS_WRITECONTIG";

    MPI_Type_size(datatype, &datatype_size);
    len = datatype_size * count;

    fd->fp_sys_posn = -1; /* set it to null, since we are using pwrite */

    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;

    if (!(fd->direct_write))     /* direct I/O not enabled */
	err = pwrite(fd->fd_sys, buf, len, offset);
    else {       /* direct I/O enabled */

	/* (1) if mem_aligned && file_aligned 
                    use direct I/O to write up to correct io_size
                    use buffered I/O for remaining  */

	if (!(((long) buf) % fd->d_mem) && !(offset % fd->d_miniosz)) 
	    ADIOI_XFS_Aligned_Mem_File_Write(fd, buf, len, offset, &err);

        /* (2) if !file_aligned
                    use buffered I/O to write up to file_aligned
                    At that point, if still mem_aligned, use (1)
   		        else copy into aligned buf and then use (1) */
	else if (offset % fd->d_miniosz) {
	    diff = fd->d_miniosz - (offset % fd->d_miniosz);
	    diff = ADIOI_MIN(diff, len);
	    nbytes = pwrite(fd->fd_sys, buf, diff, offset);

	    buf = ((char *) buf) + diff;
	    offset += diff;
	    size = len - diff;
	    if (!(((long) buf) % fd->d_mem)) {
		ADIOI_XFS_Aligned_Mem_File_Write(fd, buf, size, offset, &err);
		nbytes += err;
	    }
	    else {
		newbuf = (void *) memalign(XFS_MEMALIGN, size);
		if (newbuf) {
		    memcpy(newbuf, buf, size);
		    ADIOI_XFS_Aligned_Mem_File_Write(fd, newbuf, size, offset, &err);
		    nbytes += err;
		    free(newbuf);
		}
		else nbytes += pwrite(fd->fd_sys, buf, size, offset);
	    }
	    err = nbytes;
	}

        /* (3) if !mem_aligned && file_aligned
    	            copy into aligned buf, then use (1)  */
	else {
	    newbuf = (void *) memalign(XFS_MEMALIGN, len);
	    if (newbuf) {
		memcpy(newbuf, buf, len);
		ADIOI_XFS_Aligned_Mem_File_Write(fd, newbuf, len, offset, &err);
		free(newbuf);
	    }
	    else err = pwrite(fd->fd_sys, buf, len, offset);
	}
    }

    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += err;

#ifdef HAVE_STATUS_SET_BYTES
    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
#endif

    if (err == -1) {
	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
					   myname, __LINE__, MPI_ERR_IO, "**io",
					   "**io %s", strerror(errno));
    }
    else *error_code = MPI_SUCCESS;
}

void ADIOI_PVFS_ReadContig(ADIO_File fd, void *buf, int count, 
                     MPI_Datatype datatype, int file_ptr_type,
		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    int err=-1, datatype_size, len;
    static char myname[] = "ADIOI_PVFS_READCONTIG";

    MPI_Type_size(datatype, &datatype_size);
    len = datatype_size * count;

    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
	if (fd->fp_sys_posn != offset) {
#ifdef ADIOI_MPE_LOGGING
            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
#endif
	    pvfs_lseek64(fd->fd_sys, offset, SEEK_SET);
#ifdef ADIOI_MPE_LOGGING
            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
#endif
        }
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
#endif
	err = pvfs_read(fd->fd_sys, buf, len);
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
#endif
	if (err>0)
		fd->fp_sys_posn = offset + err;
	/* individual file pointer not updated */        
    }
    else {  /* read from curr. location of ind. file pointer */
	if (fd->fp_sys_posn != fd->fp_ind) {
#ifdef ADIOI_MPE_LOGGING
            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
#endif
	    pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
#ifdef ADIOI_MPE_LOGGING
            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
#endif
        }
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
#endif
	err = pvfs_read(fd->fd_sys, buf, len);
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
#endif
	if (err > 0)
		fd->fp_ind += err; 
	fd->fp_sys_posn = fd->fp_ind;
    }         

#ifdef HAVE_STATUS_SET_BYTES
    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
#endif

    if (err == -1) {
	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
					   myname, __LINE__, MPI_ERR_IO,
					   "**io",
					   "**io %s", strerror(errno));
    }
    else *error_code = MPI_SUCCESS;
}

void ADIOI_PVFS_ReadStridedListIO(ADIO_File fd, void *buf, int count,
                       MPI_Datatype datatype, int file_ptr_type,
                       ADIO_Offset offset, ADIO_Status *status, int
                       *error_code)
{
/* offset is in units of etype relative to the filetype. */

    ADIOI_Flatlist_node *flat_buf, *flat_file;
    int i, j, k, l, brd_size, frd_size=0, st_index=0;
    int bufsize, sum, n_etypes_in_filetype, size_in_filetype;
    int n_filetypes, etype_in_filetype;
    ADIO_Offset abs_off_in_filetype=0;
    int filetype_size, etype_size, buftype_size;
    MPI_Aint filetype_extent, buftype_extent; 
    int buf_count, buftype_is_contig, filetype_is_contig;
    ADIO_Offset userbuf_off;
    ADIO_Offset off, disp, start_off;
    int flag, st_frd_size, st_n_filetypes;
    int new_brd_size, new_frd_size;

    int mem_list_count, file_list_count;
    char **mem_offsets;
    int64_t *file_offsets;
    int *mem_lengths;
    int32_t *file_lengths;
    int total_blks_to_read;

    int max_mem_list, max_file_list;

    int b_blks_read;
    int f_data_read;
    int size_read=0, n_read_lists, extra_blks;

    int end_brd_size, end_frd_size;
    int start_k, start_j, new_file_read, new_buffer_read;
    int start_mem_offset;

#define MAX_ARRAY_SIZE 1024

#ifndef PRINT_ERR_MESG
  static char myname[] = "ADIOI_PVFS_ReadStrided";
#endif

    *error_code = MPI_SUCCESS;  /* changed below if error */

    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
    MPI_Type_size(fd->filetype, &filetype_size);
    if ( ! filetype_size ) {
#ifdef HAVE_STATUS_SET_BYTES
	MPIR_Status_set_bytes(status, datatype, 0);
#endif
	*error_code = MPI_SUCCESS; 
	return;
    }

    MPI_Type_extent(fd->filetype, &filetype_extent);
    MPI_Type_size(datatype, &buftype_size);
    MPI_Type_extent(datatype, &buftype_extent);
    etype_size = fd->etype_size;

    bufsize = buftype_size * count;

    if (!buftype_is_contig && filetype_is_contig) {

/* noncontiguous in memory, contiguous in file. */
        int64_t file_offsets;
	int32_t file_lengths;

	ADIOI_Flatten_datatype(datatype);
	flat_buf = ADIOI_Flatlist;
	while (flat_buf->type != datatype) flat_buf = flat_buf->next;

	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
	    fd->disp + etype_size * offset;

	file_list_count = 1;
	file_offsets = off;
	file_lengths = 0;
	total_blks_to_read = count*flat_buf->count;
	b_blks_read = 0;

	/* allocate arrays according to max usage */
	if (total_blks_to_read > MAX_ARRAY_SIZE)
	    mem_list_count = MAX_ARRAY_SIZE;
	else mem_list_count = total_blks_to_read;
	mem_offsets = (char**)ADIOI_Malloc(mem_list_count*sizeof(char*));
	mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));

	j = 0;
	/* step through each block in memory, filling memory arrays */
	while (b_blks_read < total_blks_to_read) {
	    for (i=0; i<flat_buf->count; i++) {
		mem_offsets[b_blks_read % MAX_ARRAY_SIZE] = 
		    (char*)((char *)buf + j*buftype_extent + flat_buf->indices[i]);
		mem_lengths[b_blks_read % MAX_ARRAY_SIZE] = 
		    flat_buf->blocklens[i];
		file_lengths += flat_buf->blocklens[i];
		b_blks_read++;
		if (!(b_blks_read % MAX_ARRAY_SIZE) ||
//.........这里部分代码省略.........

//.........这里部分代码省略.........
	}
    if ( extent < count*btype_size )
	{
	    FPRINTF(stderr,"[%d/%d] %s error in computing extent -- extent %d is smaller than total bytes requested %d!\n",
		    myrank,nprocs,myname,extent,count*btype_size);
	    fflush(stderr);
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io",
		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
	    return;
	}
    end=start+(globus_off_t)extent;
    FPRINTF(stderr,"[%d/%d] %s writing %d bytes into extent of %d bytes starting at offset %Ld\n",
	    myrank,nprocs,myname,count*btype_size,extent,(long long)start);
    fflush(stderr);

    /* start up the globus partial write */
    globus_mutex_init(&writediscontig_ctl_lock, GLOBUS_NULL);
    globus_cond_init(&writediscontig_ctl_cond, GLOBUS_NULL);
    writediscontig_ctl_done=GLOBUS_FALSE;
    if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
					       fd->filename,
					       &(oattr[fd->fd_sys]),
					       GLOBUS_NULL,
					       start,
					       end,
					       writediscontig_ctl_cb,
					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
	{
	    globus_err_handler("globus_ftp_client_partial_get",myname,result);
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io",
		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
	    return;
	}

    /* Do all the actual I/Os */
    boff=0;
    nblks=0;
    while ( boff < (count*btype_size) )
	{
	    int i,blklen;

	    for (i=0;i<flat_file->count;i++)
		{
		    if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
			blklen=flat_file->blocklens[i];
		    else
			blklen=(count*btype_size)-boff;
		    if ( blklen > 0 )
			{
			    goff=start+nblks*ftype_extent+((globus_off_t)flat_file->indices[i]);
			    /*
			    FPRINTF(stderr,"[%d/%d] %s writing %d bytes from boff=%d at goff=%Ld\n",myrank,nprocs,myname,blklen,boff,goff);
			    */
			    if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
									  ((globus_byte_t *)buf)+boff,
									  (globus_size_t)blklen,
									  goff,
									  GLOBUS_TRUE,
									  writediscontig_data_cb,
									  (void *)(&bytes_written)))!=GLOBUS_SUCCESS )
				{
				    globus_err_handler("globus_ftp_client_register_write",myname,result);
				    *error_code=MPI_ERR_IO;
				    ADIOI_Error(fd,*error_code,myname);
				    return;
				}
			    boff+=blklen;
			    if ( boff>=(count*btype_size) )
				break;
			}
		}
	    nblks++;
	}

    
    /* The ctl callback won't start till the data callbacks complete, so it's
       safe to wait on just the ctl callback */
    globus_mutex_lock(&writediscontig_ctl_lock);
    while ( writediscontig_ctl_done!=GLOBUS_TRUE )
	globus_cond_wait(&writediscontig_ctl_cond,&writediscontig_ctl_lock);
    globus_mutex_unlock(&writediscontig_ctl_lock);
    globus_mutex_destroy(&writediscontig_ctl_lock);
    globus_cond_destroy(&writediscontig_ctl_cond);

#ifdef HAVE_STATUS_SET_BYTES
    MPIR_Status_set_bytes(status, datatype, bytes_written);
#endif
    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
    {
	fd->fp_ind += extent;
	fd->fp_sys_posn = fd->fp_ind;
    }
    else {
	fd->fp_sys_posn = offset + extent;
    }
}

void ADIOI_GRIDFTP_WriteContig(ADIO_File fd, void *buf, int count, 
			     MPI_Datatype datatype, int file_ptr_type,
			     ADIO_Offset offset, ADIO_Status *status, int
			     *error_code)
{
    char myname[]="ADIOI_GRIDFTP_WriteContig";
    int myrank, nprocs, datatype_size;
    globus_size_t len,bytes_written=0;
    globus_off_t goff;
    globus_result_t result;

    if ( fd->access_mode&ADIO_RDONLY )
	{
	    *error_code=MPI_ERR_AMODE;
	    return;
	}

    *error_code = MPI_SUCCESS;

    MPI_Comm_size(fd->comm, &nprocs);
    MPI_Comm_rank(fd->comm, &myrank);
    MPI_Type_size(datatype, &datatype_size);

    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
    {
	offset = fd->fp_ind;
    }

    /* Do the gridftp I/O transfer */
    goff = (globus_off_t)offset;
    len = ((globus_size_t)datatype_size)*((globus_size_t)count);

    globus_mutex_init(&writecontig_ctl_lock, GLOBUS_NULL);
    globus_cond_init(&writecontig_ctl_cond, GLOBUS_NULL);
    writecontig_ctl_done=GLOBUS_FALSE;
    if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
					       fd->filename,
					       &(oattr[fd->fd_sys]),
					       GLOBUS_NULL,
					       goff,
					       goff+(globus_off_t)len,
					       writecontig_ctl_cb,
					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
	{
	    globus_err_handler("globus_ftp_client_partial_put",myname,result);
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io",
		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
	    return;
	}
    if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
						  (globus_byte_t *)buf,
						  len,
						  goff,
						  GLOBUS_TRUE,
						  writecontig_data_cb,
						  (void *)(&bytes_written)))!=GLOBUS_SUCCESS )
	{
	    globus_err_handler("globus_ftp_client_register_write",myname,result);
	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
		    myname, __LINE__, MPI_ERR_IO,
		    "**io",
		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
	    return;
	}


    /* The ctl callback won't start till the data callbacks complete, so it's
       safe to wait on just the ctl callback */
    globus_mutex_lock(&writecontig_ctl_lock);
    while ( writecontig_ctl_done!=GLOBUS_TRUE )
	globus_cond_wait(&writecontig_ctl_cond,&writecontig_ctl_lock);
    globus_mutex_unlock(&writecontig_ctl_lock);

    globus_mutex_destroy(&writecontig_ctl_lock);
    globus_cond_destroy(&writecontig_ctl_cond);

#ifdef HAVE_STATUS_SET_BYTES
    MPIR_Status_set_bytes(status, datatype, bytes_written);
#endif
    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
    {
	offset = fd->fp_ind;
	fd->fp_ind += bytes_written;
	fd->fp_sys_posn = fd->fp_ind;
    }
    else {
	fd->fp_sys_posn = offset + bytes_written;
    }
}

//.........这里部分代码省略.........
     * the number of regions into which the range of bytes must be divided.
     * These regions are called "file domains", or FDs.
     *
     * When this function returns, fd_start, fd_end, fd_size, and
     * min_st_offset will be filled in.  fd_start holds the starting byte
     * location for each file domain.  fd_end holds the ending byte location.
     * min_st_offset holds the minimum byte location that will be accessed.
     *
     * Both fd_start[] and fd_end[] are indexed by an aggregator number; this
     * needs to be mapped to an actual rank in the communicator later.
     *
     */
    ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
			    nprocs_for_coll, &min_st_offset,
			    &fd_start, &fd_end, 
			    fd->hints->min_fdomain_size, &fd_size,
			    fd->hints->striping_unit);

    /* calculate where the portions of the access requests of this process 
     * are located in terms of the file domains.  this could be on the same
     * process or on other processes.  this function fills in:
     * count_my_req_procs - number of processes (including this one) for which
     *     this process has requests in their file domain
     * count_my_req_per_proc - count of requests for each process, indexed
     *     by rank of the process
     * my_req[] - array of data structures describing the requests to be
     *     performed by each process (including self).  indexed by rank.
     * buf_idx[] - array of locations into which data can be directly moved;
     *     this is only valid for contiguous buffer case
     */
    ADIOI_Calc_my_req(fd, offset_list, len_list, contig_access_count,
		      min_st_offset, fd_start, fd_end, fd_size,
		      nprocs, &count_my_req_procs, 
		      &count_my_req_per_proc, &my_req,
		      &buf_idx);

    /* perform a collective communication in order to distribute the
     * data calculated above.  fills in the following:
     * count_others_req_procs - number of processes (including this
     *     one) which have requests in this process's file domain.
     * count_others_req_per_proc[] - number of separate contiguous
     *     requests from proc i lie in this process's file domain.
     */
    ADIOI_Calc_others_req(fd, count_my_req_procs, 
			  count_my_req_per_proc, my_req, 
			  nprocs, myrank, &count_others_req_procs, 
			  &others_req); 

    /* my_req[] and count_my_req_per_proc aren't needed at this point, so 
     * let's free the memory 
     */
    ADIOI_Free(count_my_req_per_proc);
    for (i=0; i<nprocs; i++) {
	if (my_req[i].count) {
	    ADIOI_Free(my_req[i].offsets);
	    ADIOI_Free(my_req[i].lens);
	}
    }
    ADIOI_Free(my_req);


    /* read data in sizes of no more than ADIOI_Coll_bufsize, 
     * communicate, and fill user buf. 
     */
    ADIOI_Read_and_exch(fd, buf, datatype, nprocs, myrank,
                        others_req, offset_list,
			len_list, contig_access_count, min_st_offset,
			fd_size, fd_start, fd_end, buf_idx, error_code);

    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);

    /* free all memory allocated for collective I/O */
    for (i=0; i<nprocs; i++) {
	if (others_req[i].count) {
	    ADIOI_Free(others_req[i].offsets);
	    ADIOI_Free(others_req[i].lens);
	    ADIOI_Free(others_req[i].mem_ptrs);
	}
    }
    ADIOI_Free(others_req);

    ADIOI_Free(buf_idx);
    ADIOI_Free(offset_list);
    ADIOI_Free(len_list);
    ADIOI_Free(st_offsets);
    ADIOI_Free(end_offsets);
    ADIOI_Free(fd_start);
    ADIOI_Free(fd_end);

#ifdef HAVE_STATUS_SET_BYTES
    MPI_Type_size_x(datatype, &size);
    bufsize = size * count;
    MPIR_Status_set_bytes(status, datatype, bufsize);
/* This is a temporary way of filling in status. The right way is to 
   keep track of how much data was actually read and placed in buf 
   during collective I/O. */
#endif

    fd->fp_sys_posn = -1;   /* set it to null. */
}

void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count,
                     MPI_Datatype datatype, int file_ptr_type,
		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    int err=-1;
    MPI_Count datatype_size