MOAB: Mesh Oriented datABase
(version 5.2.1)
|
00001 /** \file ReadHDF5Dataset.cpp 00002 * \author Jason Kraftcheck 00003 * \date 2010-07-09 00004 */ 00005 00006 #include <assert.h> 00007 #include <string.h> 00008 #include <stdlib.h> 00009 #include <iostream> 00010 #include "moab/MOABConfig.h" 00011 #include "ReadHDF5Dataset.hpp" 00012 00013 #include "moab_mpe.h" 00014 00015 #include <H5Dpublic.h> 00016 #include <H5Tpublic.h> 00017 #include <H5Ppublic.h> 00018 #include <H5Spublic.h> 00019 #ifdef MOAB_HAVE_HDF5_PARALLEL 00020 #include <H5FDmpi.h> 00021 #include <H5FDmpio.h> 00022 #endif 00023 00024 #define HDF5_16API ( H5_VERS_MAJOR < 2 && H5_VERS_MINOR < 8 ) 00025 00026 namespace moab 00027 { 00028 00029 // Selection of hyperslabs appears to be superlinear. Don't try to select 00030 // more than a few thousand at a time or things start to get real slow. 00031 const size_t DEFAULT_HYPERSLAB_SELECTION_LIMIT = 200; 00032 size_t ReadHDF5Dataset::hyperslabSelectionLimit = DEFAULT_HYPERSLAB_SELECTION_LIMIT; 00033 void ReadHDF5Dataset::default_hyperslab_selection_limit() 00034 { 00035 hyperslabSelectionLimit = DEFAULT_HYPERSLAB_SELECTION_LIMIT; 00036 } 00037 00038 H5S_seloper_t ReadHDF5Dataset::hyperslabSelectOp = H5S_SELECT_OR; 00039 00040 #ifdef MOAB_HAVE_LIBMPE 00041 static std::pair< int, int > allocate_mpe_state( const char* name, const char* color ) 00042 { 00043 std::pair< int, int > result; 00044 result.first = MPE_Log_get_event_number(); 00045 result.second = MPE_Log_get_event_number(); 00046 MPE_Describe_state( result.first, result.second, name, color ); 00047 return result; 00048 } 00049 #else 00050 static std::pair< int, int > allocate_mpe_state( const char*, const char* ) 00051 { 00052 return std::pair< int, int >(); 00053 } 00054 #endif 00055 00056 bool ReadHDF5Dataset::haveMPEEvents = false; 00057 std::pair< int, int > ReadHDF5Dataset::mpeReadEvent; 00058 std::pair< int, int > ReadHDF5Dataset::mpeReduceEvent; 00059 00060 ReadHDF5Dataset::ReadHDF5Dataset( const char* debug_desc, bool parallel, const Comm* communicator ) 00061 : closeDataSet( false ), dataSet( -1 ), dataSpace( -1 ), dataType( -1 ), fileType( -1 ), ioProp( H5P_DEFAULT ), 00062 dataSpaceRank( 0 ), rowsInTable( 0 ), doConversion( false ), nativeParallel( parallel ), readCount( 0 ), 00063 bufferSize( 0 ), mpiComm( communicator ), mpeDesc( debug_desc ) 00064 { 00065 if( !haveMPEEvents ) 00066 { 00067 haveMPEEvents = true; 00068 mpeReadEvent = allocate_mpe_state( "ReadHDF5Dataset::read", "yellow" ); 00069 mpeReduceEvent = allocate_mpe_state( "ReadHDF5Dataset::all_reduce", "yellow" ); 00070 } 00071 00072 #ifndef MOAB_HAVE_HDF5_PARALLEL 00073 if( nativeParallel ) throw Exception( __LINE__ ); 00074 #else 00075 if( nativeParallel && !mpiComm ) throw Exception( __LINE__ ); 00076 00077 if( mpiComm ) 00078 { 00079 ioProp = H5Pcreate( H5P_DATASET_XFER ); 00080 H5Pset_dxpl_mpio( ioProp, H5FD_MPIO_COLLECTIVE ); 00081 } 00082 #endif 00083 } 00084 00085 ReadHDF5Dataset::ReadHDF5Dataset( const char* debug_desc, hid_t data_set_handle, bool parallel, 00086 const Comm* communicator, bool close_data_set ) 00087 : closeDataSet( close_data_set ), dataSet( data_set_handle ), dataSpace( -1 ), dataType( -1 ), fileType( -1 ), 00088 ioProp( H5P_DEFAULT ), dataSpaceRank( 0 ), rowsInTable( 0 ), doConversion( false ), nativeParallel( parallel ), 00089 readCount( 0 ), bufferSize( 0 ), mpiComm( communicator ), mpeDesc( debug_desc ) 00090 { 00091 if( !haveMPEEvents ) 00092 { 00093 haveMPEEvents = true; 00094 mpeReadEvent = allocate_mpe_state( "ReadHDF5Dataset::read", "yellow" ); 00095 mpeReduceEvent = allocate_mpe_state( "ReadHDF5Dataset::all_reduce", "yellow" ); 00096 } 00097 00098 init( data_set_handle, close_data_set ); 00099 00100 #ifndef MOAB_HAVE_HDF5_PARALLEL 00101 if( nativeParallel ) throw Exception( __LINE__ ); 00102 #else 00103 if( nativeParallel && !mpiComm ) throw Exception( __LINE__ ); 00104 00105 if( mpiComm ) 00106 { 00107 ioProp = H5Pcreate( H5P_DATASET_XFER ); 00108 H5Pset_dxpl_mpio( ioProp, H5FD_MPIO_COLLECTIVE ); 00109 } 00110 #endif 00111 } 00112 00113 void ReadHDF5Dataset::init( hid_t data_set_handle, bool close_data_set ) 00114 { 00115 closeDataSet = close_data_set; 00116 dataSet = data_set_handle; 00117 00118 fileType = H5Dget_type( data_set_handle ); 00119 if( fileType < 0 ) throw Exception( __LINE__ ); 00120 00121 dataSpace = H5Dget_space( dataSet ); 00122 if( dataSpace < 0 ) throw Exception( __LINE__ ); 00123 00124 dataSpaceRank = H5Sget_simple_extent_dims( dataSpace, dataSetCount, dataSetOffset ); 00125 if( dataSpaceRank < 0 ) throw Exception( __LINE__ ); 00126 rowsInTable = dataSetCount[0]; 00127 00128 for( int i = 0; i < dataSpaceRank; ++i ) 00129 dataSetOffset[i] = 0; 00130 00131 currOffset = rangeEnd = internalRange.end(); 00132 } 00133 00134 unsigned ReadHDF5Dataset::columns() const 00135 { 00136 if( dataSpaceRank == 1 ) 00137 return 1; 00138 else if( dataSpaceRank == 2 ) 00139 return dataSetCount[1]; 00140 00141 throw Exception( __LINE__ ); 00142 } 00143 00144 void ReadHDF5Dataset::set_column( unsigned column ) 00145 { 00146 if( dataSpaceRank != 2 || column >= dataSetCount[1] ) throw Exception( __LINE__ ); 00147 dataSetCount[1] = 1; 00148 dataSetOffset[1] = column; 00149 } 00150 00151 Range::const_iterator ReadHDF5Dataset::next_end( Range::const_iterator iter ) 00152 { 00153 size_t slabs_remaining = hyperslabSelectionLimit; 00154 size_t avail = bufferSize; 00155 while( iter != rangeEnd && slabs_remaining ) 00156 { 00157 size_t count = *( iter.end_of_block() ) - *iter + 1; 00158 if( count >= avail ) 00159 { 00160 iter += avail; 00161 break; 00162 } 00163 00164 avail -= count; 00165 iter += count; 00166 --slabs_remaining; 00167 } 00168 return iter; 00169 } 00170 00171 void ReadHDF5Dataset::set_file_ids( const Range& file_ids, EntityHandle start_id, hsize_t row_count, hid_t data_type ) 00172 { 00173 startID = start_id; 00174 currOffset = file_ids.begin(); 00175 rangeEnd = file_ids.end(); 00176 readCount = 0; 00177 bufferSize = row_count; 00178 00179 // if a) user specified buffer size and b) we're doing a true 00180 // parallel partial read and c) we're doing collective I/O, then 00181 // we need to know the maximum number of reads that will be done. 00182 #ifdef MOAB_HAVE_HDF5_PARALLEL 00183 if( nativeParallel ) 00184 { 00185 Range::const_iterator iter = currOffset; 00186 while( iter != rangeEnd ) 00187 { 00188 ++readCount; 00189 iter = next_end( iter ); 00190 } 00191 00192 MPE_Log_event( mpeReduceEvent.first, (int)readCount, mpeDesc.c_str() ); 00193 unsigned long recv = readCount, send = readCount; 00194 MPI_Allreduce( &send, &recv, 1, MPI_UNSIGNED_LONG, MPI_MAX, *mpiComm ); 00195 readCount = recv; 00196 MPE_Log_event( mpeReduceEvent.second, (int)readCount, mpeDesc.c_str() ); 00197 } 00198 #endif 00199 00200 dataType = data_type; 00201 htri_t equal = H5Tequal( fileType, dataType ); 00202 if( equal < 0 ) throw Exception( __LINE__ ); 00203 doConversion = !equal; 00204 00205 // We always read in the format of the file to avoid stupind HDF5 00206 // library behavior when reading in parallel. We call H5Tconvert 00207 // ourselves to do the data conversion. If the type we're reading 00208 // from the file is larger than the type we want in memory, then 00209 // we need to reduce num_rows so that we can read the larger type 00210 // from the file into the passed buffer mean to accomodate num_rows 00211 // of values of the smaller in-memory type. 00212 if( doConversion ) 00213 { 00214 size_t mem_size, file_size; 00215 mem_size = H5Tget_size( dataType ); 00216 file_size = H5Tget_size( fileType ); 00217 if( file_size > mem_size ) bufferSize = bufferSize * mem_size / file_size; 00218 } 00219 } 00220 00221 void ReadHDF5Dataset::set_all_file_ids( hsize_t row_count, hid_t data_type ) 00222 { 00223 internalRange.clear(); 00224 internalRange.insert( (EntityHandle)1, ( EntityHandle )( rowsInTable ) ); 00225 set_file_ids( internalRange, 1, row_count, data_type ); 00226 } 00227 00228 ReadHDF5Dataset::~ReadHDF5Dataset() 00229 { 00230 if( fileType >= 0 ) H5Tclose( fileType ); 00231 if( dataSpace >= 0 ) H5Sclose( dataSpace ); 00232 if( closeDataSet && dataSet >= 0 ) H5Dclose( dataSet ); 00233 dataSpace = dataSet = -1; 00234 if( ioProp != H5P_DEFAULT ) H5Pclose( ioProp ); 00235 } 00236 00237 void ReadHDF5Dataset::read( void* buffer, size_t& rows_read ) 00238 { 00239 herr_t err; 00240 rows_read = 0; 00241 00242 MPE_Log_event( mpeReadEvent.first, (int)readCount, mpeDesc.c_str() ); 00243 if( currOffset != rangeEnd ) 00244 { 00245 00246 // Build H5S hyperslab selection describing the portions of the 00247 // data set to read 00248 H5S_seloper_t sop = H5S_SELECT_SET; 00249 Range::iterator new_end = next_end( currOffset ); 00250 while( currOffset != new_end ) 00251 { 00252 size_t count = *( currOffset.end_of_block() ) - *currOffset + 1; 00253 if( new_end != rangeEnd && *currOffset + count > *new_end ) { count = *new_end - *currOffset; } 00254 rows_read += count; 00255 00256 dataSetOffset[0] = *currOffset - startID; 00257 dataSetCount[0] = count; 00258 err = H5Sselect_hyperslab( dataSpace, sop, dataSetOffset, NULL, dataSetCount, 0 ); 00259 if( err < 0 ) throw Exception( __LINE__ ); 00260 sop = hyperslabSelectOp; // subsequent calls to select_hyperslab append 00261 00262 currOffset += count; 00263 } 00264 00265 // Create a data space describing the memory in which to read the data 00266 dataSetCount[0] = rows_read; 00267 hid_t mem_id = H5Screate_simple( dataSpaceRank, dataSetCount, NULL ); 00268 if( mem_id < 0 ) throw Exception( __LINE__ ); 00269 00270 // Do the actual read 00271 err = H5Dread( dataSet, fileType, mem_id, dataSpace, ioProp, buffer ); 00272 H5Sclose( mem_id ); 00273 if( err < 0 ) throw Exception( __LINE__ ); 00274 00275 if( readCount ) --readCount; 00276 00277 if( doConversion ) 00278 { 00279 err = H5Tconvert( fileType, dataType, rows_read * columns(), buffer, 0, H5P_DEFAULT ); 00280 if( err < 0 ) throw Exception( __LINE__ ); 00281 } 00282 } 00283 else if( readCount ) 00284 { 00285 null_read(); 00286 --readCount; 00287 } 00288 MPE_Log_event( mpeReadEvent.second, (int)readCount, mpeDesc.c_str() ); 00289 } 00290 00291 void ReadHDF5Dataset::null_read() 00292 { 00293 herr_t err; 00294 err = H5Sselect_none( dataSpace ); 00295 if( err < 0 ) throw Exception( __LINE__ ); 00296 00297 //#if HDF5_16API 00298 hsize_t one = 1; 00299 hid_t mem_id = H5Screate_simple( 1, &one, NULL ); 00300 if( mem_id < 0 ) throw Exception( __LINE__ ); 00301 err = H5Sselect_none( mem_id ); 00302 if( err < 0 ) 00303 { 00304 H5Sclose( mem_id ); 00305 throw Exception( __LINE__ ); 00306 } 00307 //#else 00308 // hid_t mem_id = H5Screate(H5S_NULL); 00309 // if (mem_id < 0) 00310 // throw Exception(__LINE__); 00311 //#endif 00312 00313 err = H5Dread( dataSet, fileType, mem_id, dataSpace, ioProp, 0 ); 00314 H5Sclose( mem_id ); 00315 if( err < 0 ) throw Exception( __LINE__ ); 00316 } 00317 00318 } // namespace moab