Read variable-length data from 1-D array dataset. More...

#include <ReadHDF5VarLen.hpp>

Collaboration diagram for moab::ReadHDF5VarLen:

Public Member Functions
	ReadHDF5VarLen (DebugOutput &debug_output, void *buffer, size_t buffer_size)
	Constructor.
virtual	~ReadHDF5VarLen ()
ErrorCode	read_data (ReadHDF5Dataset &data_set, const Range &offsets, EntityHandle start_offset, hid_t data_type, const Range &file_ids, const std::vector< unsigned > &vals_per_ent, const Range &ranged_file_ids)
	Do actual read of data set.
ErrorCode	read_offsets (ReadHDF5Dataset &data_set, const Range &file_ids, EntityHandle start_file_id, EntityHandle nudge, Range &offsets_out, std::vector< unsigned > &counts_out)
	Read set description table or offset vector for var-len tags or old-format poly(gon\|hedra) connectivity.
ErrorCode	read (ReadHDF5Dataset &offset_data, ReadHDF5Dataset &value_data, const Range &file_ids, EntityHandle start_file_id, hid_t data_type, const Range *ranged=0)
Protected Member Functions
virtual ErrorCode	store_data (EntityHandle file_id, void *data, long num_data, bool ranged)=0
	Store data list for a single entity.
Protected Attributes
DebugOutput &	dbgOut
Static Private Member Functions
static bool	is_ranged (EntityHandle file_id, Range::const_iterator &ranged_iter, Range::const_iterator ranged_end)
	Test if passed file_id is value pointed to by ranged_iter, and if so, incremenet ranged_iter.
Private Attributes
void *const	dataBuffer
const size_t	bufferSize

Detailed Description

Read variable-length data from 1-D array dataset.

Utility class for reading variable-length data from an HDF5 dataset. Used for reading set contents, set parents, set children, polygon and polyhedron connectivity, and variable-length tag data.

This is an abstract class. The pure virtual store_data method must be implemented to create a concrete instance.

Definition at line 32 of file ReadHDF5VarLen.hpp.

Constructor & Destructor Documentation

moab::ReadHDF5VarLen::ReadHDF5VarLen	(	DebugOutput &	debug_output,
		void *	buffer,
		size_t	buffer_size
	)		`[inline]`

Constructor.

Parameters:

buffer	A temporary buffer to use during read
buffer_size	Size of `buffer`, in bytes.

Definition at line 68 of file ReadHDF5VarLen.hpp.

        : dbgOut( debug_output ), dataBuffer( buffer ), bufferSize( buffer_size )
    {
    }

virtual moab::ReadHDF5VarLen::~ReadHDF5VarLen ( ) [inline, virtual]

Definition at line 73 of file ReadHDF5VarLen.hpp.

{}

Member Function Documentation

bool moab::ReadHDF5VarLen::is_ranged	(	EntityHandle	file_id,
		Range::const_iterator &	ranged_iter,
		Range::const_iterator	ranged_end
	)		`[static, private]`

Test if passed file_id is value pointed to by ranged_iter, and if so, incremenet ranged_iter.

Definition at line 14 of file ReadHDF5VarLen.cpp.

Referenced by read_data().

{
    if( ranged_iter == range_end ) return false;

    assert( file_id <= *ranged_iter );
    if( *ranged_iter != file_id ) return false;

    ++ranged_iter;
    return true;
}

ErrorCode moab::ReadHDF5VarLen::read	(	ReadHDF5Dataset &	offset_data,
		ReadHDF5Dataset &	value_data,
		const Range &	file_ids,
		EntityHandle	start_file_id,
		hid_t	data_type,
		const Range *	ranged = `0`
	)		`[inline]`

Definition at line 135 of file ReadHDF5VarLen.hpp.

References ErrorCode, MB_SUCCESS, read_data(), and read_offsets().

    {
        ErrorCode rval;
        const EntityHandle nudge = 1;
        Range offsets;
        std::vector< unsigned > counts;
        rval = read_offsets( offset_data, file_ids, start_file_id, nudge, offsets, counts );
        if( MB_SUCCESS != rval ) return rval;
        Range empty;
        rval = read_data( value_data, offsets, nudge, data_type, file_ids, counts, ranged ? *ranged : empty );
        return rval;
    }

ErrorCode moab::ReadHDF5VarLen::read_data	(	ReadHDF5Dataset &	data_set,
		const Range &	offsets,
		EntityHandle	start_offset,
		hid_t	data_type,
		const Range &	file_ids,
		const std::vector< unsigned > &	vals_per_ent,
		const Range &	ranged_file_ids
	)

Do actual read of data set.

Parameters:

data_set	The data set to read.
file_ids	The file ids of the entities to read.
start_file_id	The file id corresponding to the first row of the dataset
data_type	The desired, in-memory data type for values
vals_per_ent	The number of values for each entity
ranged_file_ids	Those file ids for which the 'ranged' argument to `storedata` should be passed as `true`.

Definition at line 27 of file ReadHDF5VarLen.cpp.

References moab::Range::begin(), bufferSize, dataBuffer, dbgOut, moab::ReadHDF5Dataset::done(), moab::Range::end(), ErrorCode, moab::ReadHDF5Dataset::get_debug_desc(), moab::ReadHDF5Dataset::get_read_count(), is_ranged(), MB_SUCCESS, moab::DebugOutput::printf(), moab::ReadHDF5Dataset::read(), moab::ReadHDF5Dataset::set_file_ids(), moab::Range::size(), and store_data().

Referenced by read().

{
    ErrorCode rval;
    const size_t value_size          = H5Tget_size( data_type );
    const size_t buffer_size         = bufferSize / value_size;
    unsigned char* const data_buffer = reinterpret_cast< unsigned char* >( dataBuffer );
    std::vector< unsigned char > partial;  // for when we read only part of the contents of a set/entity
    Range::const_iterator fileid_iter                  = file_ids.begin();
    Range::const_iterator ranged_iter                  = ranged_file_ids.begin();
    std::vector< unsigned >::const_iterator count_iter = vals_per_ent.begin();
    size_t count, offset;
    bool ranged;
    int nn = 0;

    assert( file_ids.size() == vals_per_ent.size() );

    try
    {
        data_set.set_file_ids( offsets, start_offset, buffer_size, data_type );
    }
    catch( ReadHDF5Dataset::Exception& )
    {
        return MB_FAILURE;
    }

    dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );

    while( !data_set.done() )
    {
        dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
        try
        {
            data_set.read( data_buffer, count );
        }
        catch( ReadHDF5Dataset::Exception& )
        {
            return MB_FAILURE;
        }

        assert( 0 == count || fileid_iter != file_ids.end() );

        // Handle 'special' case where we read some, but not all
        // of the data for an entity during the last iteration.
        offset = 0;
        if( !partial.empty() )
        {  // didn't read all of previous entity
            assert( fileid_iter != file_ids.end() );
            assert( 0 == ( partial.size() % value_size ) );
            size_t num_prev = partial.size() / value_size;
            offset          = *count_iter - num_prev;
            if( offset > count )
            {  // still don't have all
                partial.insert( partial.end(), data_buffer, data_buffer + count * value_size );
                continue;
            }

            partial.insert( partial.end(), data_buffer, data_buffer + offset * value_size );

            ranged = is_ranged( *fileid_iter, ranged_iter, ranged_file_ids.end() );
            assert( partial.size() == *count_iter * value_size );
            rval = store_data( *fileid_iter, &partial[0], *count_iter, ranged );
            if( MB_SUCCESS != rval ) return rval;

            ++count_iter;
            ++fileid_iter;
            partial.clear();
        }

        // Process contents for all entities for which we
        // have read the complete list
        while( count_iter != vals_per_ent.end() && offset + *count_iter <= count )
        {
            assert( fileid_iter != file_ids.end() );
            ranged = is_ranged( *fileid_iter, ranged_iter, ranged_file_ids.end() );
            rval   = store_data( *fileid_iter, data_buffer + offset * value_size, *count_iter, ranged );
            if( MB_SUCCESS != rval ) return rval;

            offset += *count_iter;
            ++count_iter;
            ++fileid_iter;
        }

        // If we did not read all of the final entity,
        // store what we did read to be processed in the
        // next iteration
        if( offset < count )
        {
            assert( partial.empty() );
            partial.insert( partial.end(), data_buffer + offset * value_size, data_buffer + count * value_size );
        }
    }
    // NOTE: If the last set is empty, we will not process it here
    // assert(fileid_iter == file_ids.end());
#ifndef NDEBUG
    for( ; fileid_iter != file_ids.end(); ++fileid_iter )
    {
        assert( 0 == *count_iter );
        ++count_iter;
    }
#endif
    return MB_SUCCESS;
}

ErrorCode moab::ReadHDF5VarLen::read_offsets	(	ReadHDF5Dataset &	data_set,
		const Range &	file_ids,
		EntityHandle	start_file_id,
		EntityHandle	nudge,
		Range &	offsets_out,
		std::vector< unsigned > &	counts_out
	)

Read set description table or offset vector for var-len tags or old-format poly(gon|hedra) connectivity.

Parameters:

data_set	The data set to read.
file_ids	The file ids of the entities to read.
start_file_id	The file id corresponding to the first row of the dataset
num_columns	The number of columns of offsets in the dataset
indices	Array of length `num_columns` contaning the indices of the columns to read.
nudge	Amount by which to offset values in `offset_out` to avoid putting zeros in Range. Must be greater than 0. Probably 1.
offsets_out	An array of length `num_columns` which will be populated with the resulting list of offsets into the contents list calculated from reading the offsets from the passed data set.
counts_out	An array of length `num_columns` of std::vectors, where each vector will be filled with one value per file ID indicating the length of the data for the corresponding file ID.
ranged_file_ids	If non-null, the last column of the table will be read and tested for the ranged bit. For all file_ids for which the range bit is set, the file ID will be added to this list.

Definition at line 291 of file ReadHDF5VarLen.cpp.

References moab::Range::begin(), buffer, bufferSize, moab::Range::clear(), moab::Range::const_pair_begin(), moab::Range::const_pair_end(), dataBuffer, dbgOut, moab::ReadHDF5Dataset::done(), moab::Range::empty(), moab::Range::end(), moab::Range::front(), moab::ReadHDF5Dataset::get_debug_desc(), moab::ReadHDF5Dataset::get_read_count(), moab::Range::insert(), MB_SUCCESS, moab::DebugOutput::printf(), moab::ReadHDF5Dataset::read(), moab::ReadHDF5Dataset::set_file_ids(), moab::Range::size(), and moab::Range::const_iterator::start_of_block().

Referenced by read().

{

    // Use hints to make sure insertion into ranges is O(1)
    offsets_out.clear();
    counts_out.clear();
    counts_out.reserve( file_ids.size() );
    Range::iterator hint;

    // Calculate which rows we need to read from the offsets table
    Range rows;
    hint                            = rows.begin();
    Range::const_pair_iterator pair = file_ids.const_pair_begin();
    // special case if reading first entity in dataset, because
    // there is no previous end value.
    if( pair != file_ids.const_pair_end() && pair->first == start_file_id )
    {
        hint = rows.insert( nudge, pair->second - start_file_id + nudge );
        ++pair;
    }
    while( pair != file_ids.const_pair_end() )
    {
        hint = rows.insert( hint, pair->first - start_file_id + nudge - 1, pair->second - start_file_id + nudge );
        ++pair;
    }

    // set up read of offsets dataset
    hsize_t buffer_size = bufferSize / sizeof( hssize_t );
    hssize_t* buffer    = reinterpret_cast< hssize_t* >( dataBuffer );
    data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
    hssize_t prev_end;
    bool have_prev_end = false;
    // If we're reading the first row of the table, then the
    // previous end is implicitly -1.
    if( !file_ids.empty() && file_ids.front() == start_file_id )
    {
        prev_end      = -1;
        have_prev_end = true;
    }

    dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );

    // read offset table
    size_t count, offset;
    Range::const_iterator fiter = file_ids.begin();
    hint                        = offsets_out.begin();
    int nn                      = 0;
    while( !data_set.done() )
    {
        dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
        try
        {
            data_set.read( buffer, count );
        }
        catch( ReadHDF5Dataset::Exception& )
        {
            return MB_FAILURE;
        }
        if( !count )  // might have been NULL read for collective IO
            continue;

        // If the previous end values were read in the previous iteration,
        // then they're stored in prev_end.
        offset = 0;
        if( have_prev_end )
        {
            counts_out.push_back( buffer[0] - prev_end );
            hint = offsets_out.insert( hint, prev_end + 1 + nudge, buffer[0] + nudge );
            ++fiter;
            offset        = 1;
            have_prev_end = false;
        }

        while( offset < count )
        {
            assert( fiter != file_ids.end() );
            // whenever we get to a gap between blocks we need to
            // advance one step because we read an extra end id
            // preceding teah block
            if( fiter == fiter.start_of_block() )
            {
                if( offset == count - 1 ) break;
                ++offset;
            }

            size_t s = buffer[offset - 1] + 1;
            size_t e = buffer[offset];
            counts_out.push_back( e - s + 1 );
            hint = offsets_out.insert( hint, s + nudge, e + nudge );

            ++fiter;
            ++offset;
        }

        // If we did not end on the boundary between two blocks,
        // then we need to save the end indices for the final entry
        // for use in the next iteration.  Similarly, if we ended
        // with extra values that were read with the express intention
        // of getting the previous end values for a block, we need to
        // save them.  This case only arises if we hit the break in
        // the above loop.
        if( fiter != fiter.start_of_block() || offset < count )
        {
            assert( !have_prev_end );
            if( offset == count )
            {
                --offset;
                assert( fiter != fiter.start_of_block() );
            }
            else
            {
                assert( offset + 1 == count );
                assert( fiter == fiter.start_of_block() );
            }
            have_prev_end = true;
            prev_end      = buffer[offset];
        }
    }
    assert( !have_prev_end );
    assert( fiter == file_ids.end() );

    return MB_SUCCESS;
}

virtual ErrorCode moab::ReadHDF5VarLen::store_data	(	EntityHandle	file_id,
		void *	data,
		long	num_data,
		bool	ranged
	)		`[protected, pure virtual]`

Store data list for a single entity.

The is the pure virtual method that must be provided. It is responsible for storing the data read for a single entity.

This function will always be called in the order of the file_ids in the range passed to the read method.

Parameters:

file_id	The file ID for the entity
data	A pointer to the data for the entity
num_data	Number of values for the entity
ranged	For set contents, true if in ranged format.

Referenced by read_data().