WriteHDF5.cpp

Go to the documentation of this file.

/**
 * MOAB, a Mesh-Oriented datABase, is a software component for creating,
 * storing and accessing finite element mesh data.
 *
 * Copyright 2004 Sandia Corporation.  Under the terms of Contract
 * DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government
 * retains certain rights in this software.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 */

//-------------------------------------------------------------------------
// Filename      : WriteHDF5.cpp
//
// Purpose       : TSTT HDF5 Writer
//
// Special Notes : WriteSLAC used as template for this
//
// Creator       : Jason Kraftcheck
//
// Creation Date : 04/01/04
//-------------------------------------------------------------------------

#include <cassert>
#if defined( _MSC_VER )
typedef int id_t;
#elif defined( __MINGW32__ )
#include <sys/time.h>
#else
#include <ctime>
#endif

#include <cstdlib>
#include <cstring>
#include <cstdarg>
#include <limits>
#include <cstdio>
#include <iostream>
#include "WriteHDF5.hpp"
#include <H5Tpublic.h>
#include <H5Ppublic.h>
#include <H5Epublic.h>
#include "moab/Interface.hpp"
#include "Internals.hpp"
#include "MBTagConventions.hpp"
#include "moab/CN.hpp"
#include "moab/FileOptions.hpp"
#include "moab/CpuTimer.hpp"
#include "IODebugTrack.hpp"
#include "mhdf.h"

#ifndef MOAB_HAVE_HDF5
#error Attempt to compile WriteHDF5 with HDF5 support disabled
#endif

#undef BLOCKED_COORD_IO

#ifdef MOAB_HAVE_VALGRIND
#include <valgrind/memcheck.h>
#else
#ifndef VALGRIND_CHECK_MEM_IS_DEFINED
#define VALGRIND_CHECK_MEM_IS_DEFINED( a, b ) ( (void)0 )
#endif
#ifndef VALGRIND_CHECK_MEM_IS_ADDRESSABLE
#define VALGRIND_CHECK_MEM_IS_ADDRESSABLE( a, b ) ( (void)0 )
#endif
#ifndef VALGRIND_MAKE_MEM_UNDEFINED
#define VALGRIND_MAKE_MEM_UNDEFINED( a, b ) ( (void)0 )
#endif
#endif

namespace moab
{

template < typename T >
inline void VALGRIND_MAKE_VEC_UNDEFINED( std::vector< T >& v )
{
    (void)VALGRIND_MAKE_MEM_UNDEFINED( (T*)&v[0], v.size() * sizeof( T ) );
}

#define WRITE_HDF5_BUFFER_SIZE ( 40 * 1024 * 1024 )

static hid_t get_id_type()
{
    if( 8 == sizeof( WriteHDF5::wid_t ) )
    {
        if( 8 == sizeof( long ) )
            return H5T_NATIVE_ULONG;
        else
            return H5T_NATIVE_UINT64;
    }
    else if( 4 == sizeof( WriteHDF5::wid_t ) )
    {
        if( 4 == sizeof( int ) )
            return H5T_NATIVE_UINT;
        else
            return H5T_NATIVE_UINT32;
    }
    else
    {
        assert( 0 );
        return (hid_t)-1;
    }
}

// This is the HDF5 type used to store file IDs
const hid_t WriteHDF5::id_type = get_id_type();

// This function doesn't do anything useful. It's just a nice
// place to set a break point to determine why the writer fails.
static inline ErrorCode error( ErrorCode rval )
{
    return rval;
}

// Call \c error function during HDF5 library errors to make
// it easier to trap such errors in the debugger. This function
// gets registered with the HDF5 library as a callback. It
// works the same as the default (H5Eprint), except that it
// also calls the \c error function as a no-op.
#if defined( H5E_auto_t_vers ) && H5E_auto_t_vers > 1
static herr_t handle_hdf5_error( hid_t stack, void* data )
{
    WriteHDF5::HDF5ErrorHandler* h = reinterpret_cast< WriteHDF5::HDF5ErrorHandler* >( data );
    herr_t result                  = 0;
    if( h->func ) result = ( *h->func )( stack, h->data );
    error( MB_FAILURE );
    return result;
}
#else
static herr_t handle_hdf5_error( void* data )
{
    WriteHDF5::HDF5ErrorHandler* h = reinterpret_cast< WriteHDF5::HDF5ErrorHandler* >( data );
    herr_t result                  = 0;
    if( h->func ) result = ( *h->func )( h->data );
    error( MB_FAILURE );
    return result;
}
#endif

// Some macros to handle error checking. The
// CHK_MHDF__ERR* macros check the value of an mhdf_Status
// object. The CHK_MB_ERR_* check the value of an ErrorCode.
// The *_0 macros accept no other arguments. The *_1
// macros accept a single hdf5 handle to close on error.
// The *_2 macros accept an array of two hdf5 handles to
// close on error. The _*2C macros accept one hdf5 handle
// to close on error and a bool and an hdf5 handle where
// the latter handle is conditionally closed depending on
// the value of the bool. All macros contain a "return"
// statement.
#define CHK_MHDF_ERR_0( A )                            \
    do                                                 \
    {                                                  \
        if( mhdf_isError( &( A ) ) )                   \
        {                                              \
            MB_SET_ERR_CONT( mhdf_message( &( A ) ) ); \
            assert( 0 );                               \
            return error( MB_FAILURE );                \
        }                                              \
    } while( false )

#define CHK_MHDF_ERR_1( A, B )                         \
    do                                                 \
    {                                                  \
        if( mhdf_isError( &( A ) ) )                   \
        {                                              \
            MB_SET_ERR_CONT( mhdf_message( &( A ) ) ); \
            assert( 0 );                               \
            mhdf_closeData( filePtr, ( B ), &( A ) );  \
            return error( MB_FAILURE );                \
        }                                              \
    } while( false )

#define CHK_MHDF_ERR_2( A, B )                           \
    do                                                   \
    {                                                    \
        if( mhdf_isError( &( A ) ) )                     \
        {                                                \
            MB_SET_ERR_CONT( mhdf_message( &( A ) ) );   \
            assert( 0 );                                 \
            mhdf_closeData( filePtr, ( B )[0], &( A ) ); \
            mhdf_closeData( filePtr, ( B )[1], &( A ) ); \
            return error( MB_FAILURE );                  \
        }                                                \
    } while( false )

#define CHK_MHDF_ERR_3( A, B )                           \
    do                                                   \
    {                                                    \
        if( mhdf_isError( &( A ) ) )                     \
        {                                                \
            MB_SET_ERR_CONT( mhdf_message( &( A ) ) );   \
            assert( 0 );                                 \
            mhdf_closeData( filePtr, ( B )[0], &( A ) ); \
            mhdf_closeData( filePtr, ( B )[1], &( A ) ); \
            mhdf_closeData( filePtr, ( B )[2], &( A ) ); \
            return error( MB_FAILURE );                  \
        }                                                \
    } while( false )

#define CHK_MHDF_ERR_2C( A, B, C, D )                         \
    do                                                        \
    {                                                         \
        if( mhdf_isError( &( A ) ) )                          \
        {                                                     \
            MB_SET_ERR_CONT( mhdf_message( &( A ) ) );        \
            assert( 0 );                                      \
            mhdf_closeData( filePtr, ( B ), &( A ) );         \
            if( C ) mhdf_closeData( filePtr, ( D ), &( A ) ); \
            return error( MB_FAILURE );                       \
        }                                                     \
    } while( false )

#define CHK_MB_ERR_0( A )             \
    do                                \
    {                                 \
        if( MB_SUCCESS != ( A ) )     \
        {                             \
            MB_CHK_ERR_CONT( ( A ) ); \
            return error( A );        \
        }                             \
    } while( false )

#define CHK_MB_ERR_1( A, B, C )                       \
    do                                                \
    {                                                 \
        if( MB_SUCCESS != ( A ) )                     \
        {                                             \
            MB_CHK_ERR_CONT( ( A ) );                 \
            mhdf_closeData( filePtr, ( B ), &( C ) ); \
            assert( 0 );                              \
            return error( A );                        \
        }                                             \
    } while( false )

#define CHK_MB_ERR_2( A, B, C )                          \
    do                                                   \
    {                                                    \
        if( MB_SUCCESS != ( A ) )                        \
        {                                                \
            MB_CHK_ERR_CONT( ( A ) );                    \
            mhdf_closeData( filePtr, ( B )[0], &( C ) ); \
            mhdf_closeData( filePtr, ( B )[1], &( C ) ); \
            write_finished();                            \
            assert( 0 );                                 \
            return error( A );                           \
        }                                                \
    } while( false )

#define CHK_MB_ERR_3( A, B, C )                          \
    do                                                   \
    {                                                    \
        if( MB_SUCCESS != ( A ) )                        \
        {                                                \
            MB_CHK_ERR_CONT( ( A ) );                    \
            mhdf_closeData( filePtr, ( B )[0], &( C ) ); \
            mhdf_closeData( filePtr, ( B )[1], &( C ) ); \
            mhdf_closeData( filePtr, ( B )[2], &( C ) ); \
            write_finished();                            \
            assert( 0 );                                 \
            return error( A );                           \
        }                                                \
    } while( false )

#define CHK_MB_ERR_2C( A, B, C, D, E )                        \
    do                                                        \
    {                                                         \
        if( MB_SUCCESS != ( A ) )                             \
        {                                                     \
            MB_CHK_ERR_CONT( ( A ) );                         \
            mhdf_closeData( filePtr, ( B ), &( E ) );         \
            if( C ) mhdf_closeData( filePtr, ( D ), &( E ) ); \
            write_finished();                                 \
            assert( 0 );                                      \
            return error( A );                                \
        }                                                     \
    } while( false )

#define debug_barrier() debug_barrier_line( __LINE__ )
void WriteHDF5::debug_barrier_line( int ) {}

class CheckOpenWriteHDF5Handles
{
    int fileline;
    mhdf_FileHandle handle;
    int enter_count;

  public:
    CheckOpenWriteHDF5Handles( mhdf_FileHandle file, int line )
        : fileline( line ), handle( file ), enter_count( mhdf_countOpenHandles( file ) )
    {
    }

    ~CheckOpenWriteHDF5Handles()
    {
        int new_count = mhdf_countOpenHandles( handle );
        if( new_count != enter_count )
        {
            std::cout << "Leaked HDF5 object handle in function at " << __FILE__ << ":" << fileline << std::endl
                      << "Open at entrance: " << enter_count << std::endl
                      << "Open at exit:     " << new_count << std::endl;
        }
    }
};

MPEState WriteHDF5::topState;
MPEState WriteHDF5::subState;

#ifdef NDEBUG
#define CHECK_OPEN_HANDLES
#else
#define CHECK_OPEN_HANDLES CheckOpenWriteHDF5Handles check_open_handles_( filePtr, __LINE__ )
#endif

bool WriteHDF5::convert_handle_tag( const EntityHandle* source, EntityHandle* dest, size_t count ) const
{
    bool some_valid = false;
    for( size_t i = 0; i < count; ++i )
    {
        if( !source[i] )
            dest[i] = 0;
        else
        {
            dest[i] = idMap.find( source[i] );
            if( dest[i] ) some_valid = true;
        }
    }

    return some_valid;
}

bool WriteHDF5::convert_handle_tag( EntityHandle* data, size_t count ) const
{
    assert( sizeof( EntityHandle ) == sizeof( wid_t ) );
    return convert_handle_tag( data, data, count );
}

ErrorCode WriteHDF5::assign_ids( const Range& entities, wid_t id )
{
    Range::const_pair_iterator pi;
    for( pi = entities.const_pair_begin(); pi != entities.const_pair_end(); ++pi )
    {
        const EntityHandle n = pi->second - pi->first + 1;
        dbgOut.printf( 3, "Assigning %s %lu to %lu to file IDs [%lu,%lu]\n",
                       CN::EntityTypeName( TYPE_FROM_HANDLE( pi->first ) ),
                       (unsigned long)( ID_FROM_HANDLE( pi->first ) ),
                       (unsigned long)( ID_FROM_HANDLE( pi->first ) + n - 1 ), (unsigned long)id,
                       (unsigned long)( id + n - 1 ) );
        if( TYPE_FROM_HANDLE( pi->first ) == MBPOLYGON || TYPE_FROM_HANDLE( pi->first ) == MBPOLYHEDRON )
        {
            int num_vertices         = 0;
            const EntityHandle* conn = 0;
            iFace->get_connectivity( pi->first, conn, num_vertices );
            dbgOut.printf( 3, "  poly with %d verts/faces \n", num_vertices );
        }
        if( !idMap.insert( pi->first, id, n ).second ) return error( MB_FAILURE );
        id += n;
    }

    return MB_SUCCESS;
}

const char* WriteHDF5::ExportSet::name() const
{
    static char buffer[128];
    switch( type )
    {
        case MBVERTEX:
            return mhdf_node_type_handle();
        case MBENTITYSET:
            return mhdf_set_type_handle();
        default:
            sprintf( buffer, "%s%d", CN::EntityTypeName( type ), num_nodes );
            return buffer;
    }
}

WriterIface* WriteHDF5::factory( Interface* iface )
{
    return new WriteHDF5( iface );
}

WriteHDF5::WriteHDF5( Interface* iface )
    : bufferSize( WRITE_HDF5_BUFFER_SIZE ), dataBuffer( 0 ), iFace( iface ), writeUtil( 0 ), filePtr( 0 ),
      setContentsOffset( 0 ), setChildrenOffset( 0 ), setParentsOffset( 0 ), maxNumSetContents( 0 ),
      maxNumSetChildren( 0 ), maxNumSetParents( 0 ), writeSets( false ), writeSetContents( false ),
      writeSetChildren( false ), writeSetParents( false ), parallelWrite( false ), collectiveIO( false ),
      writeTagDense( false ), writeProp( H5P_DEFAULT ), dbgOut( "H5M", stderr ), debugTrack( false )
{
}

ErrorCode WriteHDF5::init()
{
    ErrorCode rval;

    if( writeUtil )  // init has already been called
        return MB_SUCCESS;
    /*
    #ifdef DEBUG
      H5Eset_auto(&hdf_error_handler, writeUtil); // HDF5 callback for errors
    #endif
    */
    // For known tag types, store the corresponding HDF5 in which
    // the tag data is to be written in the file.
    // register_known_tag_types(iFace);

    // Get the util interface
    rval = iFace->query_interface( writeUtil );
    CHK_MB_ERR_0( rval );

    idMap.clear();

#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
    herr_t err = H5Eget_auto( H5E_DEFAULT, &errorHandler.func, &errorHandler.data );
#else
    herr_t err = H5Eget_auto( &errorHandler.func, &errorHandler.data );
#endif
    if( err < 0 )
    {
        errorHandler.func = 0;
        errorHandler.data = 0;
    }
    else
    {
#if defined( H5Eset_auto_vers ) && H5Eset_auto_vers > 1
        err = H5Eset_auto( H5E_DEFAULT, &handle_hdf5_error, &errorHandler );
#else
        err = H5Eset_auto( &handle_hdf5_error, &errorHandler );
#endif
        if( err < 0 )
        {
            errorHandler.func = 0;
            errorHandler.data = 0;
        }
    }

    if( !topState.valid() ) topState = MPEState( "WriteHDF5", "yellow" );
    if( !subState.valid() ) subState = MPEState( "WriteHDF5 subevent", "cyan" );

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_finished()
{
    // Release memory allocated in lists
    exportList.clear();
    nodeSet.range.clear();
    setSet.range.clear();
    tagList.clear();
    idMap.clear();

    HDF5ErrorHandler handler;
#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
    herr_t err = H5Eget_auto( H5E_DEFAULT, &handler.func, &handler.data );
#else
    herr_t err = H5Eget_auto( &handler.func, &handler.data );
#endif
    if( err >= 0 && handler.func == &handle_hdf5_error )
    {
        assert( handler.data == &errorHandler );
#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
        H5Eset_auto( H5E_DEFAULT, errorHandler.func, errorHandler.data );
#else
        H5Eset_auto( errorHandler.func, errorHandler.data );
#endif
    }

    return MB_SUCCESS;
}

WriteHDF5::~WriteHDF5()
{
    if( !writeUtil )  // init() failed.
        return;

    iFace->release_interface( writeUtil );
}

ErrorCode WriteHDF5::write_file( const char* filename, bool overwrite, const FileOptions& opts,
                                 const EntityHandle* set_array, const int num_sets,
                                 const std::vector< std::string >& qa_records, const Tag* tag_list, int num_tags,
                                 int user_dimension )
{
    mhdf_Status status;

    parallelWrite = false;
    collectiveIO  = false;

    // Enable debug output
    int tmpval = 0;
    if( MB_SUCCESS == opts.get_int_option( "DEBUG_IO", 1, tmpval ) ) dbgOut.set_verbosity( tmpval );

    // writeTagDense = (MB_SUCCESS == opts.get_null_option("DENSE_TAGS"));
    writeTagDense = true;

    // Enable some extra checks for reads.  Note: amongst other things this
    // will print errors if the entire file is not read, so if doing a
    // partial read that is not a parallel read, this should be disabled.
    debugTrack = ( MB_SUCCESS == opts.get_null_option( "DEBUG_BINIO" ) );

    bufferSize = WRITE_HDF5_BUFFER_SIZE;
    int buf_size;
    ErrorCode rval = opts.get_int_option( "BUFFER_SIZE", buf_size );
    if( MB_SUCCESS == rval && buf_size >= 24 ) bufferSize = buf_size;

    // Allocate internal buffer to use when gathering data to write.
    dataBuffer = (char*)malloc( bufferSize );
    if( !dataBuffer ) return error( MB_MEMORY_ALLOCATION_FAILED );

    // Clear filePtr so we know if it is open upon failure
    filePtr = 0;

    // Do actual write.
    writeProp        = H5P_DEFAULT;
    ErrorCode result = write_file_impl( filename, overwrite, opts, set_array, num_sets, qa_records, tag_list, num_tags,
                                        user_dimension );
    // Close writeProp if it was opened
    if( writeProp != H5P_DEFAULT ) H5Pclose( writeProp );

    // Free memory buffer
    free( dataBuffer );
    dataBuffer = 0;

    // Close file
    bool created_file = false;
    if( filePtr )
    {
        created_file = true;
        mhdf_closeFile( filePtr, &status );
        filePtr = 0;
        if( mhdf_isError( &status ) )
        {
            MB_SET_ERR_CONT( mhdf_message( &status ) );
            if( MB_SUCCESS == result ) result = MB_FAILURE;
        }
    }

    // Release other resources
    if( MB_SUCCESS == result )
        result = write_finished();
    else
        write_finished();

    // If write failed, remove file unless KEEP option was specified
    if( MB_SUCCESS != result && created_file && MB_ENTITY_NOT_FOUND == opts.get_null_option( "KEEP" ) )
        remove( filename );

    return result;
}

ErrorCode WriteHDF5::write_file_impl( const char* filename, bool overwrite, const FileOptions& opts,
                                      const EntityHandle* set_array, const int num_sets,
                                      const std::vector< std::string >& qa_records, const Tag* tag_list, int num_tags,
                                      int user_dimension )
{
    ErrorCode result;
    std::list< TagDesc >::const_iterator t_itor;
    std::list< ExportSet >::iterator ex_itor;
    EntityHandle elem_count, max_id;
    double times[NUM_TIMES] = { 0 };

    if( MB_SUCCESS != init() ) return error( MB_FAILURE );

    // See if we need to report times
    bool cputime = false;
    result       = opts.get_null_option( "CPUTIME" );
    if( MB_SUCCESS == result ) cputime = true;

    CpuTimer timer;

    dbgOut.tprint( 1, "Gathering Mesh\n" );
    topState.start( "gathering mesh" );

    // Gather mesh to export
    exportList.clear();
    if( 0 == num_sets || ( 1 == num_sets && set_array[0] == 0 ) )
    {
        result = gather_all_mesh();
        topState.end( result );
        CHK_MB_ERR_0( result );
    }
    else
    {
        std::vector< EntityHandle > passed_export_list( set_array, set_array + num_sets );
        result = gather_mesh_info( passed_export_list );
        topState.end( result );
        CHK_MB_ERR_0( result );
    }

    times[GATHER_TIME] = timer.time_elapsed();

    // if (nodeSet.range.size() == 0)
    //  return error(MB_ENTITY_NOT_FOUND);

    dbgOut.tprint( 1, "Checking ID space\n" );

    // Make sure ID space is sufficient
    elem_count = nodeSet.range.size() + setSet.range.size();
    for( ex_itor = exportList.begin(); ex_itor != exportList.end(); ++ex_itor )
        elem_count += ex_itor->range.size();
    max_id = (EntityHandle)1 << ( 8 * sizeof( wid_t ) - 1 );
    if( elem_count > max_id )
    {
        MB_SET_ERR_CONT( "ID space insufficient for mesh size" );
        return error( result );
    }

    dbgOut.tprint( 1, "Creating File\n" );

    // Figure out the dimension in which to write the mesh.
    int mesh_dim;
    result = iFace->get_dimension( mesh_dim );
    CHK_MB_ERR_0( result );

    if( user_dimension < 1 ) user_dimension = mesh_dim;
    user_dimension = user_dimension > mesh_dim ? mesh_dim : user_dimension;

    // Create the file layout, including all tables (zero-ed) and
    // all structure and meta information.
    const char* optnames[] = { "WRITE_PART", "FORMAT", 0 };
    int junk;
    parallelWrite = ( MB_SUCCESS == opts.match_option( "PARALLEL", optnames, junk ) );
    if( parallelWrite )
    {
        // Just store Boolean value based on string option here.
        // parallel_create_file will set writeProp accordingly.
        // collectiveIO = (MB_SUCCESS == opts.get_null_option("COLLECTIVE"));
        // dbgOut.printf(2, "'COLLECTIVE' option = %s\n", collectiveIO ? "YES" : "NO");
        // Do this all the time, as it appears to be much faster than indep in some cases
        collectiveIO = true;
        result =
            parallel_create_file( filename, overwrite, qa_records, opts, tag_list, num_tags, user_dimension, times );
    }
    else
    {
        result = serial_create_file( filename, overwrite, qa_records, tag_list, num_tags, user_dimension );
    }
    if( MB_SUCCESS != result ) return error( result );

    times[CREATE_TIME] = timer.time_elapsed();

    dbgOut.tprint( 1, "Writing Nodes.\n" );
    // Write node coordinates
    if( !nodeSet.range.empty() || parallelWrite )
    {
        topState.start( "writing coords" );
        result = write_nodes();
        topState.end( result );
        if( MB_SUCCESS != result ) return error( result );
    }

    times[COORD_TIME] = timer.time_elapsed();

    dbgOut.tprint( 1, "Writing connectivity.\n" );

    // Write element connectivity
    for( ex_itor = exportList.begin(); ex_itor != exportList.end(); ++ex_itor )
    {
        topState.start( "writing connectivity for ", ex_itor->name() );
        result = write_elems( *ex_itor );
        topState.end( result );
        if( MB_SUCCESS != result ) return error( result );
    }
    times[CONN_TIME] = timer.time_elapsed();

    dbgOut.tprint( 1, "Writing sets.\n" );

    // Write meshsets
    result = write_sets( times );
    if( MB_SUCCESS != result ) return error( result );
    debug_barrier();

    times[SET_TIME] = timer.time_elapsed();
    dbgOut.tprint( 1, "Writing adjacencies.\n" );

    // Write adjacencies
    // Tim says don't save node adjacencies!
#ifdef MB_H5M_WRITE_NODE_ADJACENCIES
    result = write_adjacencies( nodeSet );
    if( MB_SUCCESS != result ) return error( result );
#endif
    for( ex_itor = exportList.begin(); ex_itor != exportList.end(); ++ex_itor )
    {
        topState.start( "writing adjacencies for ", ex_itor->name() );
        result = write_adjacencies( *ex_itor );
        topState.end( result );
        if( MB_SUCCESS != result ) return error( result );
    }
    times[ADJ_TIME] = timer.time_elapsed();

    dbgOut.tprint( 1, "Writing tags.\n" );

    // Write tags
    for( t_itor = tagList.begin(); t_itor != tagList.end(); ++t_itor )
    {
        std::string name;
        iFace->tag_get_name( t_itor->tag_id, name );
        topState.start( "writing tag: ", name.c_str() );
        result = write_tag( *t_itor, times );
        topState.end( result );
        if( MB_SUCCESS != result ) return error( result );
    }
    times[TAG_TIME] = timer.time_elapsed();

    times[TOTAL_TIME] = timer.time_since_birth();

    if( cputime ) { print_times( times ); }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::initialize_mesh( const Range ranges[5] )
{
    ErrorCode rval;

    if( !ranges[0].all_of_type( MBVERTEX ) ) return error( MB_FAILURE );
    nodeSet.range        = ranges[0];
    nodeSet.type         = MBVERTEX;
    nodeSet.num_nodes    = 1;
    nodeSet.max_num_ents = nodeSet.max_num_adjs = 0;

    if( !ranges[4].all_of_type( MBENTITYSET ) ) return error( MB_FAILURE );
    setSet.range        = ranges[4];
    setSet.type         = MBENTITYSET;
    setSet.num_nodes    = 0;
    setSet.max_num_ents = setSet.max_num_adjs = 0;
    maxNumSetContents = maxNumSetChildren = maxNumSetParents = 0;

    exportList.clear();
    std::vector< Range > bins( 1024 );  // Sort entities by connectivity length
                                        // Resize is expensive due to Range copy, so start big
    for( EntityType type = MBEDGE; type < MBENTITYSET; ++type )
    {
        ExportSet set;
        set.max_num_ents = set.max_num_adjs = 0;
        const int dim                       = CN::Dimension( type );

        // Group entities by connectivity length
        bins.clear();
        assert( dim >= 0 && dim <= 4 );
        std::pair< Range::const_iterator, Range::const_iterator > p = ranges[dim].equal_range( type );
        Range::const_iterator i                                     = p.first;
        while( i != p.second )
        {
            Range::const_iterator first = i;
            EntityHandle const* conn;
            int len, firstlen;

            // Dummy storage vector for structured mesh "get_connectivity" function
            std::vector< EntityHandle > storage;

            rval = iFace->get_connectivity( *i, conn, firstlen, false, &storage );
            if( MB_SUCCESS != rval ) return error( rval );

            for( ++i; i != p.second; ++i )
            {
                rval = iFace->get_connectivity( *i, conn, len, false, &storage );
                if( MB_SUCCESS != rval ) return error( rval );

                if( len != firstlen ) break;
            }

            if( firstlen >= (int)bins.size() ) bins.resize( firstlen + 1 );
            bins[firstlen].merge( first, i );
        }
        // Create ExportSet for each group
        for( std::vector< Range >::iterator j = bins.begin(); j != bins.end(); ++j )
        {
            if( j->empty() ) continue;

            set.range.clear();
            set.type      = type;
            set.num_nodes = j - bins.begin();
            exportList.push_back( set );
            exportList.back().range.swap( *j );
        }
    }

    return MB_SUCCESS;
}

// Gather the mesh to be written from a list of owning meshsets.
ErrorCode WriteHDF5::gather_mesh_info( const std::vector< EntityHandle >& export_sets )
{
    ErrorCode rval;

    int dim;
    Range range;      // Temporary storage
    Range ranges[5];  // Lists of entities to export, grouped by dimension

    // Gather list of all related sets
    std::vector< EntityHandle > stack( export_sets );
    std::copy( export_sets.begin(), export_sets.end(), stack.begin() );
    std::vector< EntityHandle > set_children;
    while( !stack.empty() )
    {
        EntityHandle meshset = stack.back();
        stack.pop_back();
        ranges[4].insert( meshset );

        // Get contained sets
        range.clear();
        rval = iFace->get_entities_by_type( meshset, MBENTITYSET, range );
        CHK_MB_ERR_0( rval );
        for( Range::iterator ritor = range.begin(); ritor != range.end(); ++ritor )
        {
            if( ranges[4].find( *ritor ) == ranges[4].end() ) stack.push_back( *ritor );
        }

        // Get child sets
        set_children.clear();
        rval = iFace->get_child_meshsets( meshset, set_children, 1 );
        CHK_MB_ERR_0( rval );
        for( std::vector< EntityHandle >::iterator vitor = set_children.begin(); vitor != set_children.end(); ++vitor )
        {
            if( ranges[4].find( *vitor ) == ranges[4].end() ) stack.push_back( *vitor );
        }
    }

    // Gather list of all mesh entities from list of sets,
    // grouped by dimension.
    for( Range::iterator setitor = ranges[4].begin(); setitor != ranges[4].end(); ++setitor )
    {
        for( dim = 0; dim < 4; ++dim )
        {
            range.clear();
            rval = iFace->get_entities_by_dimension( *setitor, dim, range, false );
            CHK_MB_ERR_0( rval );

            ranges[dim].merge( range );
        }
    }

    // For each list of elements, append adjacent children and
    // nodes to lists.
    for( dim = 3; dim > 0; --dim )
    {
        for( int cdim = 1; cdim < dim; ++cdim )
        {
            range.clear();
            rval = iFace->get_adjacencies( ranges[dim], cdim, false, range );
            CHK_MB_ERR_0( rval );
            ranges[cdim].merge( range );
        }
        range.clear();
        rval = writeUtil->gather_nodes_from_elements( ranges[dim], 0, range );
        CHK_MB_ERR_0( rval );
        ranges[0].merge( range );
    }

    return initialize_mesh( ranges );
}

// Gather all the mesh and related information to be written.
ErrorCode WriteHDF5::gather_all_mesh()
{
    ErrorCode rval;
    Range ranges[5];

    rval = iFace->get_entities_by_type( 0, MBVERTEX, ranges[0] );
    if( MB_SUCCESS != rval ) return error( rval );

    rval = iFace->get_entities_by_dimension( 0, 1, ranges[1] );
    if( MB_SUCCESS != rval ) return error( rval );

    rval = iFace->get_entities_by_dimension( 0, 2, ranges[2] );
    if( MB_SUCCESS != rval ) return error( rval );

    rval = iFace->get_entities_by_dimension( 0, 3, ranges[3] );
    if( MB_SUCCESS != rval ) return error( rval );

    rval = iFace->get_entities_by_type( 0, MBENTITYSET, ranges[4] );
    if( MB_SUCCESS != rval ) return error( rval );

    return initialize_mesh( ranges );
}

ErrorCode WriteHDF5::write_nodes()
{
    mhdf_Status status;
    int dim, mesh_dim;
    ErrorCode rval;
    hid_t node_table;
    long first_id, num_nodes;

    if( !nodeSet.total_num_ents ) return MB_SUCCESS;  // No nodes!

    CHECK_OPEN_HANDLES;

    rval = iFace->get_dimension( mesh_dim );
    CHK_MB_ERR_0( rval );

    debug_barrier();
    dbgOut.print( 3, "Opening Node Coords\n" );
    node_table = mhdf_openNodeCoords( filePtr, &num_nodes, &dim, &first_id, &status );
    CHK_MHDF_ERR_0( status );
    IODebugTrack track( debugTrack, "nodes", num_nodes );

    double* buffer = (double*)dataBuffer;
#ifdef BLOCKED_COORD_IO
    int chunk_size = bufferSize / sizeof( double );
#else
    int chunk_size = bufferSize / ( 3 * sizeof( double ) );
#endif

    long remaining  = nodeSet.range.size();
    long num_writes = ( remaining + chunk_size - 1 ) / chunk_size;
    if( nodeSet.max_num_ents )
    {
        assert( nodeSet.max_num_ents >= remaining );
        num_writes = ( nodeSet.max_num_ents + chunk_size - 1 ) / chunk_size;
    }
    long remaining_writes = num_writes;

    long offset                = nodeSet.offset;
    Range::const_iterator iter = nodeSet.range.begin();
    dbgOut.printf( 3, "Writing %ld nodes in %ld blocks of %d\n", remaining, ( remaining + chunk_size - 1 ) / chunk_size,
                   chunk_size );
    while( remaining )
    {
        (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );
        long count = chunk_size < remaining ? chunk_size : remaining;
        remaining -= count;
        Range::const_iterator end = iter;
        end += count;

#ifdef BLOCKED_COORD_IO
        for( int d = 0; d < dim; d++ )
        {
            if( d < mesh_dim )
            {
                rval = writeUtil->get_node_coords( d, iter, end, count, buffer );
                CHK_MB_ERR_1( rval, node_table, status );
            }
            else
                memset( buffer, 0, count * sizeof( double ) );

            dbgOut.printf( 3, " writing %c node chunk %ld of %ld, %ld values at %ld\n", (char)( 'X' + d ),
                           num_writes - remaining_writes + 1, num_writes, count, offset );
            mhdf_writeNodeCoordWithOpt( node_table, offset, count, d, buffer, writeProp, &status );
            CHK_MHDF_ERR_1( status, node_table );
        }
#else
        rval = writeUtil->get_node_coords( -1, iter, end, 3 * count, buffer );
        CHK_MB_ERR_1( rval, node_table, status );
        dbgOut.printf( 3, " writing node chunk %ld of %ld, %ld values at %ld\n", num_writes - remaining_writes + 1,
                       num_writes, count, offset );
        mhdf_writeNodeCoordsWithOpt( node_table, offset, count, buffer, writeProp, &status );
        CHK_MHDF_ERR_1( status, node_table );
#endif
        track.record_io( offset, count );

        iter = end;
        offset += count;
        --remaining_writes;
    }

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( remaining_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
#ifdef BLOCKED_COORD_IO
            for( int d = 0; d < dim; ++d )
            {
                dbgOut.printf( 3, " writing (empty) %c node chunk %ld of %ld.\n", (char)( 'X' + d ),
                               num_writes - remaining_writes, num_writes );
                mhdf_writeNodeCoordWithOpt( node_table, offset, 0, d, 0, writeProp, &status );
                CHK_MHDF_ERR_1( status, node_table );
            }
#else
            dbgOut.printf( 3, " writing (empty) node chunk %ld of %ld.\n", num_writes - remaining_writes, num_writes );
            mhdf_writeNodeCoordsWithOpt( node_table, offset, 0, 0, writeProp, &status );
            CHK_MHDF_ERR_1( status, node_table );
#endif
        }
    }

    mhdf_closeData( filePtr, node_table, &status );
    CHK_MHDF_ERR_0( status );

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_elems( ExportSet& elems )
{
    mhdf_Status status;
    ErrorCode rval;
    long first_id;
    int nodes_per_elem;
    long table_size;

    CHECK_OPEN_HANDLES;

    debug_barrier();
    dbgOut.printf( 2, "Writing %lu elements of type %s%d\n", (unsigned long)elems.range.size(),
                   CN::EntityTypeName( elems.type ), elems.num_nodes );
    dbgOut.print( 3, "Writing elements", elems.range );

    hid_t elem_table = mhdf_openConnectivity( filePtr, elems.name(), &nodes_per_elem, &table_size, &first_id, &status );
    CHK_MHDF_ERR_0( status );
    IODebugTrack track( debugTrack, elems.name() && strlen( elems.name() ) ? elems.name() : "<ANONYMOUS ELEM SET?>",
                        table_size );

    assert( (unsigned long)first_id <= elems.first_id );
    assert( (unsigned long)table_size >= elems.offset + elems.range.size() );

    EntityHandle* buffer = (EntityHandle*)dataBuffer;
    int chunk_size       = bufferSize / ( elems.num_nodes * sizeof( wid_t ) );
    long offset          = elems.offset;
    long remaining       = elems.range.size();
    long num_writes      = ( remaining + chunk_size - 1 ) / chunk_size;
    if( elems.max_num_ents )
    {
        assert( elems.max_num_ents >= remaining );
        num_writes = ( elems.max_num_ents + chunk_size - 1 ) / chunk_size;
    }
    long remaining_writes = num_writes;
    Range::iterator iter  = elems.range.begin();

    while( remaining )
    {
        (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );
        long count = chunk_size < remaining ? chunk_size : remaining;
        remaining -= count;

        Range::iterator next = iter;
        next += count;
        rval = writeUtil->get_element_connect( iter, next, elems.num_nodes, count * elems.num_nodes, buffer );
        CHK_MB_ERR_1( rval, elem_table, status );
        iter = next;

        for( long i = 0; i < count * nodes_per_elem; ++i )
        {
            buffer[i] = idMap.find( buffer[i] );
            if( 0 == buffer[i] )
            {
                MB_SET_ERR_CONT( "Invalid " << elems.name() << " element connectivity. Write Aborted" );
                mhdf_closeData( filePtr, elem_table, &status );
                return error( MB_FAILURE );
            }
        }

        dbgOut.printf( 3, " writing node connectivity %ld of %ld, %ld values at %ld\n",
                       num_writes - remaining_writes + 1, num_writes, count, offset );
        track.record_io( offset, count );
        mhdf_writeConnectivityWithOpt( elem_table, offset, count, id_type, buffer, writeProp, &status );
        CHK_MHDF_ERR_1( status, elem_table );

        offset += count;
        --remaining_writes;
    }

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( remaining_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
            dbgOut.printf( 3, " writing (empty) connectivity chunk %ld of %ld.\n", num_writes - remaining_writes + 1,
                           num_writes );
            mhdf_writeConnectivityWithOpt( elem_table, offset, 0, id_type, 0, writeProp, &status );
            CHK_MHDF_ERR_1( status, elem_table );
        }
    }

    mhdf_closeData( filePtr, elem_table, &status );
    CHK_MHDF_ERR_0( status );

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::get_set_info( EntityHandle set, long& num_entities, long& num_children, long& num_parents,
                                   unsigned long& flags )
{
    ErrorCode rval;
    int i;
    unsigned int u;

    rval = iFace->get_number_entities_by_handle( set, i, false );
    CHK_MB_ERR_0( rval );
    num_entities = i;

    rval = iFace->num_child_meshsets( set, &i );
    CHK_MB_ERR_0( rval );
    num_children = i;

    rval = iFace->num_parent_meshsets( set, &i );
    CHK_MB_ERR_0( rval );
    num_parents = i;

    rval = iFace->get_meshset_options( set, u );
    CHK_MB_ERR_0( rval );
    flags = u;

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_set_data( const WriteUtilIface::EntityListType which_data, const hid_t handle,
                                     IODebugTrack& track, Range* ranged, Range* null_stripped,
                                     std::vector< long >* set_sizes )
{
    // ranged must be non-null for CONTENTS and null for anything else
    assert( ( which_data == WriteUtilIface::CONTENTS ) == ( 0 != ranged ) );
    ErrorCode rval;
    mhdf_Status status;

    debug_barrier();

    // Function pointer type used to write set data
    void ( *write_func )( hid_t, long, long, hid_t, const void*, hid_t, mhdf_Status* );
    long max_vals;  // Max over all procs of number of values to write to data set
    long offset;    // Offset in HDF5 dataset at which to write next block of data
    switch( which_data )
    {
        case WriteUtilIface::CONTENTS:
            assert( ranged != 0 && null_stripped != 0 && set_sizes != 0 );
            write_func = &mhdf_writeSetDataWithOpt;
            max_vals   = maxNumSetContents;
            offset     = setContentsOffset;
            dbgOut.print( 2, "Writing set contents\n" );
            break;
        case WriteUtilIface::CHILDREN:
            assert( !ranged && !null_stripped && !set_sizes );
            write_func = &mhdf_writeSetParentsChildrenWithOpt;
            max_vals   = maxNumSetChildren;
            offset     = setChildrenOffset;
            dbgOut.print( 2, "Writing set child lists\n" );
            break;
        case WriteUtilIface::PARENTS:
            assert( !ranged && !null_stripped && !set_sizes );
            write_func = &mhdf_writeSetParentsChildrenWithOpt;
            max_vals   = maxNumSetParents;
            offset     = setParentsOffset;
            dbgOut.print( 2, "Writing set parent lists\n" );
            break;
        default:
            assert( false );
            return MB_FAILURE;
    }
    // assert(max_vals > 0); // Should have skipped this function otherwise

    // buffer to use for IO
    wid_t* buffer = reinterpret_cast< wid_t* >( dataBuffer );
    // number of handles that will fit in the buffer
    const size_t buffer_size = bufferSize / sizeof( EntityHandle );
    // the total number of write calls that must be made, including no-ops for collective io
    const size_t num_total_writes = ( max_vals + buffer_size - 1 ) / buffer_size;

    std::vector< SpecialSetData >::iterator si = specialSets.begin();

    std::vector< wid_t > remaining;         // data left over from prev iteration because it didn't fit in buffer
    size_t remaining_offset           = 0;  // avoid erasing from front of 'remaining'
    const EntityHandle* remaining_ptr = 0;  // remaining for non-ranged data
    size_t remaining_count            = 0;
    const wid_t* special_rem_ptr      = 0;
    Range::const_iterator i           = setSet.range.begin(), j, rhint, nshint;
    if( ranged ) rhint = ranged->begin();
    if( null_stripped ) nshint = null_stripped->begin();
    for( size_t w = 0; w < num_total_writes; ++w )
    {
        if( i == setSet.range.end() && !remaining.empty() && !remaining_ptr )
        {
            // If here, then we've written everything but we need to
            // make more write calls because we're doing collective IO
            // in parallel
            ( *write_func )( handle, 0, 0, id_type, 0, writeProp, &status );
            CHK_MHDF_ERR_0( status );
            continue;
        }

        // If we had some left-over data from a range-compacted set
        // from the last iteration, add it to the buffer now
        size_t count = 0;
        if( !remaining.empty() )
        {
            count = remaining.size() - remaining_offset;
            if( count > buffer_size )
            {
                memcpy( buffer, &remaining[remaining_offset], buffer_size * sizeof( wid_t ) );
                count = buffer_size;
                remaining_offset += buffer_size;
            }
            else
            {
                memcpy( buffer, &remaining[remaining_offset], count * sizeof( wid_t ) );
                remaining_offset = 0;
                remaining.clear();
            }
        }
        // If we had some left-over data from a non-range-compacted set
        // from the last iteration, add it to the buffer now
        else if( remaining_ptr )
        {
            if( remaining_count > buffer_size )
            {
                rval = vector_to_id_list( remaining_ptr, buffer, buffer_size );
                CHK_MB_ERR_0( rval );
                count = buffer_size;
                remaining_ptr += count;
                remaining_count -= count;
            }
            else
            {
                rval = vector_to_id_list( remaining_ptr, buffer, remaining_count );
                CHK_MB_ERR_0( rval );
                count           = remaining_count;
                remaining_ptr   = 0;
                remaining_count = 0;
            }
        }
        // If we had some left-over data from a "special" (i.e. parallel shared)
        // set.
        else if( special_rem_ptr )
        {
            if( remaining_count > buffer_size )
            {
                memcpy( buffer, special_rem_ptr, buffer_size * sizeof( wid_t ) );
                count = buffer_size;
                special_rem_ptr += count;
                remaining_count -= count;
            }
            else
            {
                memcpy( buffer, special_rem_ptr, remaining_count * sizeof( wid_t ) );
                count           = remaining_count;
                special_rem_ptr = 0;
                remaining_count = 0;
            }
        }

        // While there is both space remaining in the buffer and
        // more sets to write, append more set data to buffer.

        while( count < buffer_size && i != setSet.range.end() )
        {
            // Special case for "special" (i.e. parallel shared) sets:
            // we already have the data in a vector, just copy it.
            if( si != specialSets.end() && si->setHandle == *i )
            {
                std::vector< wid_t >& list = ( which_data == WriteUtilIface::CONTENTS )  ? si->contentIds
                                             : ( which_data == WriteUtilIface::PARENTS ) ? si->parentIds
                                                                                         : si->childIds;
                size_t append              = list.size();
                if( count + list.size() > buffer_size )
                {
                    append          = buffer_size - count;
                    special_rem_ptr = &list[append];
                    remaining_count = list.size() - append;
                }
                memcpy( buffer + count, &list[0], append * sizeof( wid_t ) );
                ++i;
                ++si;
                count += append;
                continue;
            }

            j = i;
            ++i;
            const EntityHandle* ptr;
            int len;
            unsigned char flags;
            rval = writeUtil->get_entity_list_pointers( j, i, &ptr, which_data, &len, &flags );
            if( MB_SUCCESS != rval ) return rval;
            if( which_data == WriteUtilIface::CONTENTS && !( flags & MESHSET_ORDERED ) )
            {
                bool compacted;
                remaining.clear();
                if( len == 0 )
                    compacted = false;
                else
                {
                    assert( !( len % 2 ) );
                    rval = range_to_blocked_list( ptr, len / 2, remaining, compacted );
                    if( MB_SUCCESS != rval ) return rval;
                }
                if( compacted )
                {
                    rhint = ranged->insert( rhint, *j );
                    set_sizes->push_back( remaining.size() );
                }
                else if( remaining.size() != (unsigned)len )
                {
                    nshint = null_stripped->insert( nshint, *j );
                    set_sizes->push_back( remaining.size() );
                }

                if( count + remaining.size() <= buffer_size )
                {
                    if( !remaining.empty() )
                        memcpy( buffer + count, &remaining[0], sizeof( wid_t ) * remaining.size() );
                    count += remaining.size();
                    remaining.clear();
                    remaining_offset = 0;
                }
                else
                {
                    remaining_offset = buffer_size - count;
                    memcpy( buffer + count, &remaining[0], sizeof( wid_t ) * remaining_offset );
                    count += remaining_offset;
                }
            }
            else
            {
                if( count + len > buffer_size )
                {
                    size_t append   = buffer_size - count;
                    remaining_ptr   = ptr + append;
                    remaining_count = len - append;
                    len             = append;
                }

                rval = vector_to_id_list( ptr, buffer + count, len );
                count += len;
            }
        }

        // Write the buffer.
        ( *write_func )( handle, offset, count, id_type, buffer, writeProp, &status );
        CHK_MHDF_ERR_0( status );
        track.record_io( offset, count );
        offset += count;
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_sets( double* times )
{
    mhdf_Status status;
    ErrorCode rval;
    long first_id, size;
    hid_t table;
    CpuTimer timer;

    CHECK_OPEN_HANDLES;
    /* If no sets, just return success */
    if( !writeSets ) return MB_SUCCESS;

    debug_barrier();
    dbgOut.printf( 2, "Writing %lu non-shared sets\n", (unsigned long)setSet.range.size() );
    dbgOut.print( 3, "Non-shared sets", setSet.range );

    /* Write set parents */
    if( writeSetParents )
    {
        topState.start( "writing parent lists for local sets" );
        table = mhdf_openSetParents( filePtr, &size, &status );
        CHK_MHDF_ERR_0( status );
        IODebugTrack track( debugTrack, "SetParents", size );

        rval = write_set_data( WriteUtilIface::PARENTS, table, track );
        topState.end( rval );
        CHK_MB_ERR_1( rval, table, status );

        mhdf_closeData( filePtr, table, &status );
        CHK_MHDF_ERR_0( status );

        times[SET_PARENT] = timer.time_elapsed();
        track.all_reduce();
    }

    /* Write set children */
    if( writeSetChildren )
    {
        topState.start( "writing child lists for local sets" );
        table = mhdf_openSetChildren( filePtr, &size, &status );
        CHK_MHDF_ERR_0( status );
        IODebugTrack track( debugTrack, "SetChildren", size );

        rval = write_set_data( WriteUtilIface::CHILDREN, table, track );
        topState.end( rval );
        CHK_MB_ERR_1( rval, table, status );

        mhdf_closeData( filePtr, table, &status );
        CHK_MHDF_ERR_0( status );

        times[SET_CHILD] = timer.time_elapsed();
        track.all_reduce();
    }

    /* Write set contents */
    Range ranged_sets, null_stripped_sets;
    std::vector< long > set_sizes;
    if( writeSetContents )
    {
        topState.start( "writing content lists for local sets" );
        table = mhdf_openSetData( filePtr, &size, &status );
        CHK_MHDF_ERR_0( status );
        IODebugTrack track( debugTrack, "SetContents", size );

        rval = write_set_data( WriteUtilIface::CONTENTS, table, track, &ranged_sets, &null_stripped_sets, &set_sizes );
        topState.end( rval );
        CHK_MB_ERR_1( rval, table, status );

        mhdf_closeData( filePtr, table, &status );
        CHK_MHDF_ERR_0( status );

        times[SET_CONTENT] = timer.time_elapsed();
        track.all_reduce();
    }
    assert( ranged_sets.size() + null_stripped_sets.size() == set_sizes.size() );

    /* Write set description table */

    debug_barrier();
    topState.start( "writing descriptions of local sets" );
    dbgOut.printf( 2, "Writing %lu non-shared sets\n", (unsigned long)setSet.range.size() );
    dbgOut.print( 3, "Non-shared sets", setSet.range );

    /* Open the table */
    table = mhdf_openSetMeta( filePtr, &size, &first_id, &status );
    CHK_MHDF_ERR_0( status );
    IODebugTrack track_meta( debugTrack, "SetMeta", size );

    /* Some debug stuff */
    debug_barrier();
    dbgOut.printf( 2, "Writing %lu non-shared sets\n", (unsigned long)setSet.range.size() );
    dbgOut.print( 3, "Non-shared sets", setSet.range );

    /* Counts and buffers and such */
    mhdf_index_t* const buffer     = reinterpret_cast< mhdf_index_t* >( dataBuffer );
    const size_t buffer_size       = bufferSize / ( 4 * sizeof( mhdf_index_t ) );
    const size_t num_local_writes  = ( setSet.range.size() + buffer_size - 1 ) / buffer_size;
    const size_t num_global_writes = ( setSet.max_num_ents + buffer_size - 1 ) / buffer_size;
    assert( num_local_writes <= num_global_writes );
    assert( num_global_writes > 0 );

    /* data about sets for which number of handles written is
     * not the same as the number of handles in the set
     * (range-compacted or null handles stripped out)
     */
    Range::const_iterator i                       = setSet.range.begin();
    Range::const_iterator r                       = ranged_sets.begin();
    Range::const_iterator s                       = null_stripped_sets.begin();
    std::vector< mhdf_index_t >::const_iterator n = set_sizes.begin();
    assert( ranged_sets.size() + null_stripped_sets.size() == set_sizes.size() );

    /* We write the end index for each list, rather than the count */
    mhdf_index_t prev_contents_end = setContentsOffset - 1;
    mhdf_index_t prev_children_end = setChildrenOffset - 1;
    mhdf_index_t prev_parents_end  = setParentsOffset - 1;

    /* While there is more data to write */
    size_t offset                                    = setSet.offset;
    std::vector< SpecialSetData >::const_iterator si = specialSets.begin();
    for( size_t w = 0; w < num_local_writes; ++w )
    {
        // Get a buffer full of data
        size_t count = 0;
        while( count < buffer_size && i != setSet.range.end() )
        {
            // Get set properties
            long num_ent, num_child, num_parent;
            unsigned long flags;
            if( si != specialSets.end() && si->setHandle == *i )
            {
                flags      = si->setFlags;
                num_ent    = si->contentIds.size();
                num_child  = si->childIds.size();
                num_parent = si->parentIds.size();
                ++si;
                if( r != ranged_sets.end() && *i == *r )
                {
                    assert( flags & mhdf_SET_RANGE_BIT );
                    ++r;
                    ++n;
                }
                else if( s != null_stripped_sets.end() && *i == *s )
                {
                    ++s;
                    ++n;
                }
            }
            else
            {
                assert( si == specialSets.end() || si->setHandle > *i );

                // Get set properties
                rval = get_set_info( *i, num_ent, num_child, num_parent, flags );
                CHK_MB_ERR_1( rval, table, status );

                // Check if size is something other than num handles in set
                if( r != ranged_sets.end() && *i == *r )
                {
                    num_ent = *n;
                    ++r;
                    ++n;
                    flags |= mhdf_SET_RANGE_BIT;
                }
                else if( s != null_stripped_sets.end() && *i == *s )
                {
                    num_ent = *n;
                    ++s;
                    ++n;
                }
            }

            // Put data in buffer
            mhdf_index_t* local = buffer + 4 * count;
            prev_contents_end += num_ent;
            prev_children_end += num_child;
            prev_parents_end += num_parent;
            local[0] = prev_contents_end;
            local[1] = prev_children_end;
            local[2] = prev_parents_end;
            local[3] = flags;

            // Iterate
            ++count;
            ++i;
        }

        // Write the data
        mhdf_writeSetMetaWithOpt( table, offset, count, MHDF_INDEX_TYPE, buffer, writeProp, &status );
        CHK_MHDF_ERR_1( status, table );
        track_meta.record_io( offset, count );
        offset += count;
    }
    assert( r == ranged_sets.end() );
    assert( s == null_stripped_sets.end() );
    assert( n == set_sizes.end() );

    /* If doing parallel write with collective IO, do null write
     * calls because other procs aren't done yet and write calls
     * are collective */
    for( size_t w = num_local_writes; w != num_global_writes; ++w )
    {
        mhdf_writeSetMetaWithOpt( table, 0, 0, MHDF_INDEX_TYPE, 0, writeProp, &status );
        CHK_MHDF_ERR_1( status, table );
    }

    topState.end();
    mhdf_closeData( filePtr, table, &status );
    CHK_MHDF_ERR_0( status );

    times[SET_META] = timer.time_elapsed();
    track_meta.all_reduce();

    return MB_SUCCESS;
}

template < class HandleRangeIter >
inline size_t count_num_handles( HandleRangeIter iter, HandleRangeIter end )
{
    size_t result = 0;
    for( ; iter != end; ++iter )
        result += iter->second - iter->first + 1;

    return result;
}

template < class HandleRangeIter >
inline ErrorCode range_to_id_list_templ( HandleRangeIter begin, HandleRangeIter end,
                                         const RangeMap< EntityHandle, WriteHDF5::wid_t >& idMap,
                                         WriteHDF5::wid_t* array )
{
    ErrorCode rval                                          = MB_SUCCESS;
    RangeMap< EntityHandle, WriteHDF5::wid_t >::iterator ri = idMap.begin();
    WriteHDF5::wid_t* i                                     = array;
    for( HandleRangeIter pi = begin; pi != end; ++pi )
    {
        EntityHandle h = pi->first;
        while( h <= pi->second )
        {
            ri = idMap.lower_bound( ri, idMap.end(), h );
            if( ri == idMap.end() || ri->begin > h )
            {
                rval = MB_ENTITY_NOT_FOUND;
                *i   = 0;
                ++i;
                ++h;
                continue;
            }

            // compute the last available value of the found target range (ri iterator)
            WriteHDF5::wid_t last_valid_input_value_in_current_map_range = ri->begin + ri->count - 1;
            // limit the number of steps we do on top of h so we do not overflow the output range
            // span
            WriteHDF5::wid_t step_until = std::min( last_valid_input_value_in_current_map_range, pi->second );
            WriteHDF5::wid_t n          = step_until - h + 1;
            assert( n > 0 );  // We must at least step 1

            WriteHDF5::wid_t id = ri->value + ( h - ri->begin );
            for( WriteHDF5::wid_t j = 0; j < n; ++i, ++j )
                *i = id + j;
            h += n;
        }
    }

    assert( i == array + count_num_handles( begin, end ) );
    return rval;
}

template < class HandleRangeIter >
inline ErrorCode range_to_blocked_list_templ( HandleRangeIter begin, HandleRangeIter end,
                                              const RangeMap< EntityHandle, WriteHDF5::wid_t >& idMap,
                                              std::vector< WriteHDF5::wid_t >& output_id_list, bool& ranged_list )
{
    output_id_list.clear();
    if( begin == end )
    {
        ranged_list = false;
        return MB_SUCCESS;
    }

    // First try ranged format, but give up if we reach the
    // non-range format size.
    RangeMap< EntityHandle, WriteHDF5::wid_t >::iterator ri = idMap.begin();

    const size_t num_handles = count_num_handles( begin, end );
    // If we end up with more than this many range blocks, then
    // we're better off just writing the set as a simple list
    size_t pairs_remaining = num_handles / 2;
    for( HandleRangeIter pi = begin; pi != end; ++pi )
    {
        EntityHandle h                              = pi->first;
        WriteHDF5::wid_t local_mapped_from_subrange = 0;
        while( h <= pi->second )
        {
            ri = idMap.lower_bound( ri, idMap.end(), h );
            if( ri == idMap.end() || ri->begin > h )
            {
                ++h;
                continue;
            }

            WriteHDF5::wid_t n = pi->second - pi->first + 1 - local_mapped_from_subrange;
            if( n > ri->count ) n = ri->count;

            WriteHDF5::wid_t id = ri->value + ( h - ri->begin );
            // see if we can go to the end of the range
            if( id + n > ri->value + ri->count )  // we have to reduce n, because we cannot go over next subrange
            {
                if( ri->value + ri->count - id > 0 ) n = ri->value + ri->count - id;
            }

            // See if we can append it to the previous range
            if( !output_id_list.empty() && output_id_list[output_id_list.size() - 2] + output_id_list.back() == id )
            {
                output_id_list.back() += n;
            }

            // If we ran out of space, (or set is empty) just do list format
            else if( !pairs_remaining )
            {
                ranged_list = false;
                output_id_list.resize( num_handles );
                range_to_id_list_templ( begin, end, idMap, &output_id_list[0] );
                output_id_list.erase( std::remove( output_id_list.begin(), output_id_list.end(), 0u ),
                                      output_id_list.end() );
                return MB_SUCCESS;
            }

            //
            else
            {
                --pairs_remaining;
                output_id_list.push_back( id );
                output_id_list.push_back( n );
            }
            local_mapped_from_subrange += n;  // we already mapped so many
            h += n;
        }
    }

    ranged_list = true;
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::range_to_blocked_list( const Range& input_range, std::vector< wid_t >& output_id_list,
                                            bool& ranged_list )
{
    return range_to_blocked_list_templ( input_range.const_pair_begin(), input_range.const_pair_end(), idMap,
                                        output_id_list, ranged_list );
}

ErrorCode WriteHDF5::range_to_blocked_list( const EntityHandle* array, size_t num_input_ranges,
                                            std::vector< wid_t >& output_id_list, bool& ranged_list )
{
    // We assume this in the cast on the following line
    typedef std::pair< EntityHandle, EntityHandle > mtype;
    assert( sizeof( mtype ) == 2 * sizeof( EntityHandle ) );
    const mtype* arr = reinterpret_cast< const mtype* >( array );
    return range_to_blocked_list_templ( arr, arr + num_input_ranges, idMap, output_id_list, ranged_list );
}

ErrorCode WriteHDF5::range_to_id_list( const Range& range, wid_t* array )
{
    return range_to_id_list_templ( range.const_pair_begin(), range.const_pair_end(), idMap, array );
}

ErrorCode WriteHDF5::vector_to_id_list( const EntityHandle* input, size_t input_len, wid_t* output, size_t& output_len,
                                        bool remove_zeros )
{
    const EntityHandle* i_iter = input;
    const EntityHandle* i_end  = input + input_len;
    wid_t* o_iter              = output;
    for( ; i_iter != i_end; ++i_iter )
    {
        wid_t id = idMap.find( *i_iter );
        if( !remove_zeros || id != 0 )
        {
            *o_iter = id;
            ++o_iter;
        }
    }
    output_len = o_iter - output;

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::vector_to_id_list( const std::vector< EntityHandle >& input, std::vector< wid_t >& output,
                                        bool remove_zeros )
{
    output.resize( input.size() );
    size_t output_size = 0;
    ErrorCode rval     = vector_to_id_list( &input[0], input.size(), &output[0], output_size, remove_zeros );
    output.resize( output_size );
    return rval;
}

ErrorCode WriteHDF5::vector_to_id_list( const EntityHandle* input, wid_t* output, size_t count )
{
    size_t output_len;
    return vector_to_id_list( input, count, output, output_len, false );
}

inline ErrorCode WriteHDF5::get_adjacencies( EntityHandle entity, std::vector< wid_t >& adj )
{
    const EntityHandle* adj_array;
    int num_adj;
    ErrorCode rval = writeUtil->get_adjacencies( entity, adj_array, num_adj );
    if( MB_SUCCESS != rval ) return error( rval );

    size_t j = 0;
    adj.resize( num_adj );
    for( int i = 0; i < num_adj; ++i )
        if( wid_t id = idMap.find( adj_array[i] ) ) adj[j++] = id;
    adj.resize( j );

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_adjacencies( const ExportSet& elements )
{
    ErrorCode rval;
    mhdf_Status status;
    Range::const_iterator iter;
    const Range::const_iterator end = elements.range.end();
    std::vector< wid_t > adj_list;

    CHECK_OPEN_HANDLES;

    debug_barrier();

    /* Count Adjacencies */
    long count = 0;
    // for (iter = elements.range.begin(); iter != end; ++iter) {
    //  adj_list.clear();
    //  rval = get_adjacencies(*iter, adj_list);CHK_MB_ERR_0(rval);
    //
    //  if (adj_list.size() > 0)
    //    count += adj_list.size() + 2;
    //}

    // if (count == 0)
    //  return MB_SUCCESS;

    long offset = elements.adj_offset;
    if( elements.max_num_adjs == 0 ) return MB_SUCCESS;

    /* Create data list */
    hid_t table = mhdf_openAdjacency( filePtr, elements.name(), &count, &status );
    CHK_MHDF_ERR_0( status );
    IODebugTrack track( debugTrack, "Adjacencies", count );

    /* Write data */
    wid_t* buffer   = (wid_t*)dataBuffer;
    long chunk_size = bufferSize / sizeof( wid_t );
    long num_writes = ( elements.max_num_adjs + chunk_size - 1 ) / chunk_size;
    (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );
    count = 0;
    for( iter = elements.range.begin(); iter != end; ++iter )
    {
        adj_list.clear();
        rval = get_adjacencies( *iter, adj_list );
        CHK_MB_ERR_1( rval, table, status );
        if( adj_list.size() == 0 ) continue;

        // If buffer is full, flush it
        if( count + adj_list.size() + 2 > (unsigned long)chunk_size )
        {
            dbgOut.print( 3, " writing adjacency chunk.\n" );
            track.record_io( offset, count );
            mhdf_writeAdjacencyWithOpt( table, offset, count, id_type, buffer, writeProp, &status );
            CHK_MHDF_ERR_1( status, table );
            (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );

            offset += count;
            count = 0;
        }

        buffer[count++] = idMap.find( *iter );
        buffer[count++] = adj_list.size();

        assert( adj_list.size() + 2 < (unsigned long)chunk_size );
        memcpy( buffer + count, &adj_list[0], adj_list.size() * sizeof( wid_t ) );
        count += adj_list.size();
    }

    if( count )
    {
        dbgOut.print( 2, " writing final adjacency chunk.\n" );
        mhdf_writeAdjacencyWithOpt( table, offset, count, id_type, buffer, writeProp, &status );
        CHK_MHDF_ERR_1( status, table );

        offset += count;
        count = 0;
        --num_writes;
    }

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( num_writes > 0 )
        {
            --num_writes;
            assert( writeProp != H5P_DEFAULT );
            dbgOut.print( 2, " writing empty adjacency chunk.\n" );
            mhdf_writeAdjacencyWithOpt( table, offset, 0, id_type, 0, writeProp, &status );
            CHK_MHDF_ERR_1( status, table );
        }
    }

    mhdf_closeData( filePtr, table, &status );
    CHK_MHDF_ERR_0( status );

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_tag( const TagDesc& tag_data, double* times )
{
    std::string name;
    ErrorCode rval = iFace->tag_get_name( tag_data.tag_id, name );
    if( MB_SUCCESS != rval ) return error( rval );

    CHECK_OPEN_HANDLES;
    debug_barrier();
    dbgOut.tprintf( 1, "Writing tag: \"%s\"\n", name.c_str() );

    int moab_size, elem_size, array_len;
    DataType moab_type;
    mhdf_TagDataType mhdf_type;
    hid_t hdf5_type;
    rval = get_tag_size( tag_data.tag_id, moab_type, moab_size, elem_size, array_len, mhdf_type, hdf5_type );
    if( MB_SUCCESS != rval ) return error( rval );

    CpuTimer timer;
    if( array_len == MB_VARIABLE_LENGTH && tag_data.write_sparse )
    {
        dbgOut.printf( 2, "Writing sparse data for var-len tag: \"%s\"\n", name.c_str() );
        rval = write_var_len_tag( tag_data, name, moab_type, hdf5_type, elem_size );
        times[VARLEN_TAG_TIME] += timer.time_elapsed();
    }
    else
    {
        int data_len = elem_size;
        if( moab_type != MB_TYPE_BIT ) data_len *= array_len;
        if( tag_data.write_sparse )
        {
            dbgOut.printf( 2, "Writing sparse data for tag: \"%s\"\n", name.c_str() );
            rval = write_sparse_tag( tag_data, name, moab_type, hdf5_type, data_len );
            times[SPARSE_TAG_TIME] += timer.time_elapsed();
        }
        for( size_t i = 0; MB_SUCCESS == rval && i < tag_data.dense_list.size(); ++i )
        {
            const ExportSet* set = find( tag_data.dense_list[i] );
            assert( 0 != set );
            debug_barrier();
            dbgOut.printf( 2, "Writing dense data for tag: \"%s\" on group \"%s\"\n", name.c_str(), set->name() );
            subState.start( "writing dense data for tag: ", ( name + ":" + set->name() ).c_str() );
            rval = write_dense_tag( tag_data, *set, name, moab_type, hdf5_type, data_len );
            subState.end( rval );
        }
        times[DENSE_TAG_TIME] += timer.time_elapsed();
    }

    H5Tclose( hdf5_type );
    return MB_SUCCESS == rval ? MB_SUCCESS : error( rval );
}

ErrorCode WriteHDF5::write_sparse_ids( const TagDesc& tag_data, const Range& range, hid_t id_table, size_t table_size,
                                       const char* name )
{
    ErrorCode rval;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    std::string tname( name ? name : "<UNKNOWN TAG?>" );
    tname += " - Ids";
    IODebugTrack track( debugTrack, tname, table_size );

    // Set up data buffer for writing IDs
    size_t chunk_size = bufferSize / sizeof( wid_t );
    wid_t* id_buffer  = (wid_t*)dataBuffer;

    // Write IDs of tagged entities.
    long remaining  = range.size();
    long offset     = tag_data.sparse_offset;
    long num_writes = ( remaining + chunk_size - 1 ) / chunk_size;
    if( tag_data.max_num_ents )
    {
        assert( tag_data.max_num_ents >= (unsigned long)remaining );
        num_writes = ( tag_data.max_num_ents + chunk_size - 1 ) / chunk_size;
    }
    Range::const_iterator iter = range.begin();
    while( remaining )
    {
        (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );

        // Write "chunk_size" blocks of data
        long count = (unsigned long)remaining > chunk_size ? chunk_size : remaining;
        remaining -= count;
        Range::const_iterator stop = iter;
        stop += count;
        Range tmp;
        ;
        tmp.merge( iter, stop );
        iter = stop;
        assert( tmp.size() == (unsigned)count );

        rval = range_to_id_list( tmp, id_buffer );
        CHK_MB_ERR_0( rval );

        // Write the data
        dbgOut.print( 3, " writing sparse tag entity chunk.\n" );
        track.record_io( offset, count );
        mhdf_writeSparseTagEntitiesWithOpt( id_table, offset, count, id_type, id_buffer, writeProp, &status );
        CHK_MHDF_ERR_0( status );

        offset += count;
        --num_writes;
    }  // while (remaining)

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( num_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
            dbgOut.print( 3, " writing empty sparse tag entity chunk.\n" );
            mhdf_writeSparseTagEntitiesWithOpt( id_table, offset, 0, id_type, 0, writeProp, &status );
            CHK_MHDF_ERR_0( status );
        }
    }

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_sparse_tag( const TagDesc& tag_data, const std::string& name, DataType mb_data_type,
                                       hid_t value_type, int value_type_size )
{
    ErrorCode rval;
    mhdf_Status status;
    hid_t tables[3];
    long table_size, data_size;

    CHECK_OPEN_HANDLES;

    // Get entities for which to write tag values
    Range range;
    rval = get_sparse_tagged_entities( tag_data, range );

    // Open tables to write info
    mhdf_openSparseTagData( filePtr, name.c_str(), &table_size, &data_size, tables, &status );
    CHK_MHDF_ERR_0( status );
    assert( range.size() + tag_data.sparse_offset <= (unsigned long)table_size );
    // Fixed-length tag
    assert( table_size == data_size );

    // Write IDs for tagged entities
    subState.start( "writing sparse ids for tag: ", name.c_str() );
    rval = write_sparse_ids( tag_data, range, tables[0], table_size, name.c_str() );
    subState.end( rval );
    CHK_MB_ERR_2( rval, tables, status );
    mhdf_closeData( filePtr, tables[0], &status );
    CHK_MHDF_ERR_1( status, tables[1] );

    // Set up data buffer for writing tag values
    IODebugTrack track( debugTrack, name + " Data", data_size );
    subState.start( "writing sparse values for tag: ", name.c_str() );
    rval = write_tag_values( tag_data.tag_id, tables[1], tag_data.sparse_offset, range, mb_data_type, value_type,
                             value_type_size, tag_data.max_num_ents, track );
    subState.end( rval );
    CHK_MB_ERR_0( rval );
    mhdf_closeData( filePtr, tables[1], &status );
    CHK_MHDF_ERR_0( status );

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_var_len_indices( const TagDesc& tag_data, const Range& range, hid_t idx_table,
                                            size_t table_size, int /*type_size*/, const char* name )
{
    ErrorCode rval;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    std::string tname( name ? name : "<UNKNOWN TAG?>" );
    tname += " - End Indices";
    IODebugTrack track( debugTrack, tname, table_size );

    // Set up data buffer for writing indices
    size_t chunk_size        = bufferSize / ( std::max( sizeof( void* ), sizeof( long ) ) + sizeof( int ) );
    mhdf_index_t* idx_buffer = (mhdf_index_t*)dataBuffer;
    const void** junk        = (const void**)dataBuffer;
    int* size_buffer         = (int*)( dataBuffer + chunk_size * std::max( sizeof( void* ), sizeof( mhdf_index_t ) ) );

    // Write IDs of tagged entities.
    long data_offset  = tag_data.var_data_offset - 1;  // Offset at which to write data buffer
    size_t remaining  = range.size();
    size_t offset     = tag_data.sparse_offset;
    size_t num_writes = ( remaining + chunk_size - 1 ) / chunk_size;
    if( tag_data.max_num_ents )
    {
        assert( tag_data.max_num_ents >= (unsigned long)remaining );
        num_writes = ( tag_data.max_num_ents + chunk_size - 1 ) / chunk_size;
    }
    Range::const_iterator iter = range.begin();
    while( remaining )
    {
        (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );

        // Write "chunk_size" blocks of data
        size_t count = remaining > chunk_size ? chunk_size : remaining;
        remaining -= count;
        Range::const_iterator stop = iter;
        stop += count;
        Range tmp;
        tmp.merge( iter, stop );
        iter = stop;
        assert( tmp.size() == (unsigned)count );

        rval = iFace->tag_get_by_ptr( tag_data.tag_id, tmp, junk, size_buffer );
        CHK_MB_ERR_0( rval );

        // Calculate end indices
        dbgOut.print( 3, " writing var-len tag offset chunk.\n" );
        track.record_io( offset, count );
        for( size_t i = 0; i < count; ++i )
        {
            data_offset += size_buffer[i];
            idx_buffer[i] = data_offset;
        }

        // Write
        mhdf_writeSparseTagIndicesWithOpt( idx_table, offset, count, MHDF_INDEX_TYPE, idx_buffer, writeProp, &status );
        CHK_MHDF_ERR_0( status );

        offset += count;
        --num_writes;
    }  // while (remaining)

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( num_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
            dbgOut.print( 3, " writing empty sparse tag entity chunk.\n" );
            mhdf_writeSparseTagIndicesWithOpt( idx_table, offset, 0, id_type, 0, writeProp, &status );
            CHK_MHDF_ERR_0( status );
        }
    }

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_var_len_data( const TagDesc& tag_data, const Range& range, hid_t table, size_t table_size,
                                         bool handle_tag, hid_t hdf_type, int type_size, const char* name )
{
    ErrorCode rval;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;
    assert( !handle_tag || sizeof( EntityHandle ) == type_size );

    std::string tname( name ? name : "<UNKNOWN TAG?>" );
    tname += " - Values";
    IODebugTrack track( debugTrack, tname, table_size );

    const size_t buffer_size = bufferSize / type_size;

    size_t num_writes = ( table_size + buffer_size - 1 ) / buffer_size;
    if( collectiveIO )
    {
        assert( tag_data.max_num_vals > 0 );
        num_writes = ( tag_data.max_num_vals + buffer_size - 1 ) / buffer_size;
    }

    unsigned char* buffer      = (unsigned char*)dataBuffer;
    const void* prev_data      = 0;  // Data left over from prev iteration
    size_t prev_len            = 0;
    Range::const_iterator iter = range.begin();
    long offset                = tag_data.var_data_offset;
    while( prev_data || iter != range.end() )
    {
        size_t count = 0;
        if( prev_data )
        {
            size_t len;
            const void* ptr = prev_data;
            if( prev_len <= buffer_size )
            {
                len       = prev_len;
                prev_data = 0;
                prev_len  = 0;
            }
            else
            {
                len       = buffer_size;
                prev_data = ( (const char*)prev_data ) + buffer_size * type_size;
                prev_len -= buffer_size;
            }

            if( handle_tag )
                convert_handle_tag( (const EntityHandle*)ptr, (EntityHandle*)buffer, len );
            else
                memcpy( buffer, ptr, len * type_size );
            count = len;
        }

        for( ; count < buffer_size && iter != range.end(); ++iter )
        {
            int len;
            const void* ptr;
            rval = iFace->tag_get_by_ptr( tag_data.tag_id, &*iter, 1, &ptr, &len );
            CHK_MB_ERR_0( rval );
            if( len + count > buffer_size )
            {
                prev_len  = len + count - buffer_size;
                len       = buffer_size - count;
                prev_data = ( (const char*)ptr ) + len * type_size;
            }

            if( handle_tag )
                convert_handle_tag( (const EntityHandle*)ptr, ( (EntityHandle*)buffer ) + count, len );
            else
                memcpy( buffer + count * type_size, ptr, len * type_size );
            count += len;
        }

        track.record_io( offset, count );
        mhdf_writeTagValuesWithOpt( table, offset, count, hdf_type, buffer, writeProp, &status );
        offset += count;
        CHK_MHDF_ERR_0( status );
        --num_writes;
    }

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( num_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
            dbgOut.print( 3, " writing empty var-len tag data chunk.\n" );
            mhdf_writeTagValuesWithOpt( table, 0, 0, hdf_type, 0, writeProp, &status );
            CHK_MHDF_ERR_0( status );
        }
    }

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_var_len_tag( const TagDesc& tag_data, const std::string& name, DataType mb_data_type,
                                        hid_t hdf_type, int type_size )
{
    ErrorCode rval;
    mhdf_Status status;
    hid_t tables[3];
    long table_size;
    long data_table_size;

    CHECK_OPEN_HANDLES;

    // Get entities for which to write tag values
    Range range;
    rval = get_sparse_tagged_entities( tag_data, range );

    // Open tables to write info
    mhdf_openSparseTagData( filePtr, name.c_str(), &table_size, &data_table_size, tables, &status );
    CHK_MHDF_ERR_0( status );
    assert( range.size() + tag_data.sparse_offset <= (unsigned long)table_size );

    // Write IDs for tagged entities
    subState.start( "writing ids for var-len tag: ", name.c_str() );
    rval = write_sparse_ids( tag_data, range, tables[0], table_size, name.c_str() );
    subState.end( rval );
    CHK_MB_ERR_2( rval, tables, status );
    mhdf_closeData( filePtr, tables[0], &status );
    CHK_MHDF_ERR_2( status, tables + 1 );

    // Write offsets for tagged entities
    subState.start( "writing indices for var-len tag: ", name.c_str() );
    rval = write_var_len_indices( tag_data, range, tables[2], table_size, type_size, name.c_str() );
    subState.end( rval );
    CHK_MB_ERR_1( rval, tables[1], status );
    mhdf_closeData( filePtr, tables[2], &status );
    CHK_MHDF_ERR_1( status, tables[1] );

    // Write the actual tag data
    subState.start( "writing values for var-len tag: ", name.c_str() );
    rval = write_var_len_data( tag_data, range, tables[1], data_table_size, mb_data_type == MB_TYPE_HANDLE, hdf_type,
                               type_size, name.c_str() );
    subState.end( rval );
    CHK_MB_ERR_0( rval );
    mhdf_closeData( filePtr, tables[1], &status );
    CHK_MHDF_ERR_0( status );

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_dense_tag( const TagDesc& tag_data, const ExportSet& elem_data, const std::string& name,
                                      DataType mb_data_type, hid_t value_type, int value_type_size )
{
    CHECK_OPEN_HANDLES;

    // Open tables to write info
    mhdf_Status status;
    long table_size;
    hid_t table = mhdf_openDenseTagData( filePtr, name.c_str(), elem_data.name(), &table_size, &status );
    CHK_MHDF_ERR_0( status );
    assert( elem_data.range.size() + elem_data.offset <= (unsigned long)table_size );

    IODebugTrack track( debugTrack, name + " " + elem_data.name() + " Data", table_size );
    ErrorCode rval = write_tag_values( tag_data.tag_id, table, elem_data.offset, elem_data.range, mb_data_type,
                                       value_type, value_type_size, elem_data.max_num_ents, track );
    CHK_MB_ERR_0( rval );
    mhdf_closeData( filePtr, table, &status );
    CHK_MHDF_ERR_0( status );

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_tag_values( Tag tag_id, hid_t data_table, unsigned long offset_in, const Range& range_in,
                                       DataType mb_data_type, hid_t value_type, int value_type_size,
                                       unsigned long max_num_ents, IODebugTrack& track )
{
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    // Set up data buffer for writing tag values
    size_t chunk_size = bufferSize / value_type_size;
    assert( chunk_size > 0 );
    char* tag_buffer = (char*)dataBuffer;

    // Write the tag values
    size_t remaining           = range_in.size();
    size_t offset              = offset_in;
    Range::const_iterator iter = range_in.begin();
    long num_writes            = ( remaining + chunk_size - 1 ) / chunk_size;
    if( max_num_ents )
    {
        assert( max_num_ents >= remaining );
        num_writes = ( max_num_ents + chunk_size - 1 ) / chunk_size;
    }
    while( remaining )
    {
        (void)VALGRIND_MAKE_MEM_UNDEFINED( dataBuffer, bufferSize );

        // Write "chunk_size" blocks of data
        long count = (unsigned long)remaining > chunk_size ? chunk_size : remaining;
        remaining -= count;
        memset( tag_buffer, 0, count * value_type_size );
        Range::const_iterator stop = iter;
        stop += count;
        Range range;
        range.merge( iter, stop );
        iter = stop;
        assert( range.size() == (unsigned)count );

        ErrorCode rval = iFace->tag_get_data( tag_id, range, tag_buffer );
        CHK_MB_ERR_0( rval );

        // Convert EntityHandles to file ids
        if( mb_data_type == MB_TYPE_HANDLE )
            convert_handle_tag( reinterpret_cast< EntityHandle* >( tag_buffer ),
                                count * value_type_size / sizeof( EntityHandle ) );

        // Write the data
        dbgOut.print( 2, " writing tag value chunk.\n" );
        track.record_io( offset, count );
        assert( value_type > 0 );
        mhdf_writeTagValuesWithOpt( data_table, offset, count, value_type, tag_buffer, writeProp, &status );
        CHK_MHDF_ERR_0( status );

        offset += count;
        --num_writes;
    }  // while (remaining)

    // Do empty writes if necessary for parallel collective IO
    if( collectiveIO )
    {
        while( num_writes-- )
        {
            assert( writeProp != H5P_DEFAULT );
            dbgOut.print( 2, " writing empty tag value chunk.\n" );
            assert( value_type > 0 );
            mhdf_writeTagValuesWithOpt( data_table, offset, 0, value_type, 0, writeProp, &status );
            CHK_MHDF_ERR_0( status );
        }
    }

    track.all_reduce();
    return MB_SUCCESS;
}

ErrorCode WriteHDF5::write_qa( const std::vector< std::string >& list )
{
    const char* app  = "MOAB";
    const char* vers = MOAB_VERSION;
    char date_str[64];
    char time_str[64];

    CHECK_OPEN_HANDLES;

    std::vector< const char* > strs( list.size() ? list.size() : 4 );
    if( list.size() == 0 )
    {
        time_t t = time( NULL );
        tm* lt   = localtime( &t );
#ifdef WIN32
        strftime( date_str, sizeof( date_str ), "%m/%d/%y", lt );  // VS 2008 does not support %D
        strftime( time_str, sizeof( time_str ), "%H:%M:%S", lt );  // VS 2008 does not support %T
#else
        strftime( date_str, sizeof( date_str ), "%D", lt );
        strftime( time_str, sizeof( time_str ), "%T", lt );
#endif

        strs[0] = app;
        strs[1] = vers;
        strs[2] = date_str;
        strs[3] = time_str;
    }
    else
    {
        for( unsigned int i = 0; i < list.size(); ++i )
            strs[i] = list[i].c_str();
    }

    mhdf_Status status;
    dbgOut.print( 2, " writing QA history.\n" );
    mhdf_writeHistory( filePtr, &strs[0], strs.size(), &status );
    CHK_MHDF_ERR_0( status );

    return MB_SUCCESS;
}

/*
ErrorCode WriteHDF5::register_known_tag_types(Interface* iface)
{
  hid_t int4, double16;
  hsize_t dim[1];
  int error = 0;
  ErrorCode rval;

  dim[0] = 4;
  int4 = H5Tarray_create(H5T_NATIVE_INT, 1, dim, NULL);

  dim[0] = 16;
  double16 = H5Tarray_create(H5T_NATIVE_DOUBLE, 1, dim, NULL);

  if (int4 < 0 || double16 < 0)
    error = 1;

  struct { const char* name; hid_t type; } list[] = {
    { GLOBAL_ID_TAG_NAME, H5T_NATIVE_INT } ,
    { MATERIAL_SET_TAG_NAME, H5T_NATIVE_INT },
    { DIRICHLET_SET_TAG_NAME, H5T_NATIVE_INT },
    { NEUMANN_SET_TAG_NAME, H5T_NATIVE_INT },
    { HAS_MID_NODES_TAG_NAME, int4 },
    { GEOM_DIMENSION_TAG_NAME, H5T_NATIVE_INT },
    { MESH_TRANSFORM_TAG_NAME, double16 },
    { 0, 0 } };

  for (int i = 0; list[i].name; ++i) {
    if (list[i].type < 1) {
      ++error;
      continue;
    }

    Tag handle;

    std::string name("__hdf5_tag_type_");
    name += list[i].name;

    rval = iface->tag_get_handle(name.c_str(), handle);
    if (MB_TAG_NOT_FOUND == rval) {
      rval = iface->tag_create(name.c_str(), sizeof(hid_t), MB_TAG_SPARSE, handle, NULL);
      if (MB_SUCCESS != rval) {
        ++error;
        continue;
      }

      hid_t copy_id = H5Tcopy(list[i].type);
      const EntityHandle mesh = 0;
      rval = iface->tag_set_data(handle, &mesh, 1, &copy_id);
      if (MB_SUCCESS != rval) {
        ++error;
        continue;
      }
    }
  }

  H5Tclose(int4);
  H5Tclose(double16);
  return error ? MB_FAILURE : MB_SUCCESS;
}
*/

ErrorCode WriteHDF5::gather_tags( const Tag* user_tag_list, int num_tags )
{
    ErrorCode result;
    std::vector< Tag > tag_list;
    std::vector< Tag >::iterator t_itor;
    Range range;

    // Get list of Tags to write
    result = writeUtil->get_tag_list( tag_list, user_tag_list, num_tags );
    CHK_MB_ERR_0( result );

    // Get list of tags
    for( t_itor = tag_list.begin(); t_itor != tag_list.end(); ++t_itor )
    {
        // Add tag to export list
        TagDesc tag_data;
        tag_data.write_sparse    = false;
        tag_data.tag_id          = *t_itor;
        tag_data.sparse_offset   = 0;
        tag_data.var_data_offset = 0;
        tag_data.max_num_ents    = 0;
        tag_data.max_num_vals    = 0;
        tagList.push_back( tag_data );
    }

    return MB_SUCCESS;
}

// If we support parallel, then this function will have been
// overridden with an alternate version in WriteHDF5Parallel
// that supports parallel I/O.  If we're here
// then MOAB was not built with support for parallel HDF5 I/O.
ErrorCode WriteHDF5::parallel_create_file( const char* /* filename */, bool /* overwrite */,
                                           const std::vector< std::string >& /* qa_records */,
                                           const FileOptions& /* opts */, const Tag* /* tag_list */, int /* num_tags */,
                                           int /* dimension */, double* /* times */ )
{
    MB_SET_ERR( MB_NOT_IMPLEMENTED, "WriteHDF5 does not support parallel writing" );
}

ErrorCode WriteHDF5::serial_create_file( const char* filename, bool overwrite,
                                         const std::vector< std::string >& qa_records, const Tag* user_tag_list,
                                         int num_user_tags, int dimension )
{
    long first_id;
    mhdf_Status status;
    hid_t handle;
    std::list< ExportSet >::iterator ex_itor;
    ErrorCode rval;

    topState.start( "creating file" );

    const char* type_names[MBMAXTYPE];
    memset( type_names, 0, MBMAXTYPE * sizeof( char* ) );
    for( EntityType i = MBEDGE; i < MBENTITYSET; ++i )
        type_names[i] = CN::EntityTypeName( i );

    // Create the file
    filePtr = mhdf_createFile( filename, overwrite, type_names, MBMAXTYPE, id_type, &status );
    CHK_MHDF_ERR_0( status );
    assert( !!filePtr );

    rval = write_qa( qa_records );
    CHK_MB_ERR_0( rval );

    // Create node table
    if( nodeSet.range.size() )
    {
        nodeSet.total_num_ents = nodeSet.range.size();
        handle = mhdf_createNodeCoords( filePtr, dimension, nodeSet.total_num_ents, &first_id, &status );
        CHK_MHDF_ERR_0( status );
        mhdf_closeData( filePtr, handle, &status );
        CHK_MHDF_ERR_0( status );
        nodeSet.first_id = (wid_t)first_id;
        rval             = assign_ids( nodeSet.range, nodeSet.first_id );
        CHK_MB_ERR_0( rval );
    }
    else
    {
        nodeSet.first_id = std::numeric_limits< wid_t >::max();
    }
    nodeSet.offset = 0;

    // Create element tables
    for( ex_itor = exportList.begin(); ex_itor != exportList.end(); ++ex_itor )
    {
        ex_itor->total_num_ents = ex_itor->range.size();
        rval                    = create_elem_table( *ex_itor, ex_itor->total_num_ents, first_id );
        CHK_MB_ERR_0( rval );

        ex_itor->first_id = (wid_t)first_id;
        ex_itor->offset   = 0;
        rval              = assign_ids( ex_itor->range, ex_itor->first_id );
        CHK_MB_ERR_0( rval );
    }
    // Create set tables
    writeSets = !setSet.range.empty();
    if( writeSets )
    {
        long contents_len, children_len, parents_len;

        setSet.total_num_ents = setSet.range.size();
        setSet.max_num_ents   = setSet.total_num_ents;
        rval                  = create_set_meta( setSet.total_num_ents, first_id );
        CHK_MB_ERR_0( rval );

        setSet.first_id = (wid_t)first_id;
        rval            = assign_ids( setSet.range, setSet.first_id );
        CHK_MB_ERR_0( rval );

        rval = count_set_size( setSet.range, contents_len, children_len, parents_len );
        CHK_MB_ERR_0( rval );

        rval = create_set_tables( contents_len, children_len, parents_len );
        CHK_MB_ERR_0( rval );

        setSet.offset     = 0;
        setContentsOffset = 0;
        setChildrenOffset = 0;
        setParentsOffset  = 0;
        writeSetContents  = !!contents_len;
        writeSetChildren  = !!children_len;
        writeSetParents   = !!parents_len;

        maxNumSetContents = contents_len;
        maxNumSetChildren = children_len;
        maxNumSetParents  = parents_len;
    }  // if (!setSet.range.empty())

    // Create adjacency table after set table, because sets do not have yet an id
    // some entities are adjacent to sets (exodus?)
    // Create node adjacency table
    wid_t num_adjacencies;
#ifdef MB_H5M_WRITE_NODE_ADJACENCIES
    rval = count_adjacencies( nodeSet.range, num_adjacencies );
    CHK_MB_ERR_0( rval );
    nodeSet.adj_offset   = 0;
    nodeSet.max_num_adjs = num_adjacencies;
    if( num_adjacencies > 0 )
    {
        handle = mhdf_createAdjacency( filePtr, mhdf_node_type_handle(), num_adjacencies, &status );
        CHK_MHDF_ERR_0( status );
        mhdf_closeData( filePtr, handle, &status );
    }
#endif

    // Create element adjacency tables
    for( ex_itor = exportList.begin(); ex_itor != exportList.end(); ++ex_itor )
    {
        rval = count_adjacencies( ex_itor->range, num_adjacencies );
        CHK_MB_ERR_0( rval );

        ex_itor->adj_offset   = 0;
        ex_itor->max_num_adjs = num_adjacencies;
        if( num_adjacencies > 0 )
        {
            handle = mhdf_createAdjacency( filePtr, ex_itor->name(), num_adjacencies, &status );
            CHK_MHDF_ERR_0( status );
            mhdf_closeData( filePtr, handle, &status );
        }
    }

    dbgOut.tprint( 1, "Gathering Tags\n" );

    rval = gather_tags( user_tag_list, num_user_tags );
    CHK_MB_ERR_0( rval );

    // Create the tags and tag data tables
    std::list< TagDesc >::iterator tag_iter = tagList.begin();
    for( ; tag_iter != tagList.end(); ++tag_iter )
    {
        // As we haven't yet added any ExportSets for which to write
        // dense tag data to the TagDesc struct pointed to by
        // tag_iter, this call will initially return all tagged entities
        // in the set of entities to be written.
        Range range;
        rval = get_sparse_tagged_entities( *tag_iter, range );
        CHK_MB_ERR_0( rval );

        int s;
        bool var_len = ( MB_VARIABLE_DATA_LENGTH == iFace->tag_get_length( tag_iter->tag_id, s ) );

        // Determine which ExportSets we want to write dense
        // data for. We never write dense data for variable-length
        // tag data.
        if( !var_len && writeTagDense )
        {
            // Check if we want to write this tag in dense format even if not
            // all of the entities have a tag value.  The criterion of this
            // is that the tag be dense, have a default value, and have at
            // least 2/3 of the entities tagged.
            bool prefer_dense = false;
            TagType type;
            rval = iFace->tag_get_type( tag_iter->tag_id, type );
            CHK_MB_ERR_0( rval );
            if( MB_TAG_DENSE == type )
            {
                const void* defval = 0;
                rval               = iFace->tag_get_default_value( tag_iter->tag_id, defval, s );
                if( MB_SUCCESS == rval ) prefer_dense = true;
            }

            if( check_dense_format_tag( nodeSet, range, prefer_dense ) )
            {
                range -= nodeSet.range;
                tag_iter->dense_list.push_back( nodeSet );
            }

            std::list< ExportSet >::const_iterator ex = exportList.begin();
            for( ; ex != exportList.end(); ++ex )
            {
                if( check_dense_format_tag( *ex, range, prefer_dense ) )
                {
                    range -= ex->range;
                    tag_iter->dense_list.push_back( *ex );
                }
            }

            if( check_dense_format_tag( setSet, range, prefer_dense ) )
            {
                range -= setSet.range;
                tag_iter->dense_list.push_back( setSet );
            }
        }

        tag_iter->write_sparse = !range.empty();

        unsigned long var_len_total = 0;
        if( var_len )
        {
            rval = get_tag_data_length( *tag_iter, range, var_len_total );
            CHK_MB_ERR_0( rval );
        }

        rval = create_tag( *tag_iter, range.size(), var_len_total );
        CHK_MB_ERR_0( rval );
    }  // for (tags)

    topState.end();
    return MB_SUCCESS;
}

bool WriteHDF5::check_dense_format_tag( const ExportSet& ents, const Range& all_tagged, bool prefer_dense )
{
    // If there are no tagged entities, then don't write anything
    if( ents.range.empty() ) return false;

    // If all of the entities are tagged, then write in dense format
    if( all_tagged.contains( ents.range ) ) return true;

    // Unless asked for more lenient choice of dense format, return false
    if( !prefer_dense ) return false;

    // If we're being lenient about choosing dense format, then
    // return true if at least 2/3 of the entities are tagged.
    Range xsect = intersect( setSet.range, all_tagged );
    if( 3 * xsect.size() >= 2 * setSet.range.size() ) return true;

    return false;
}

ErrorCode WriteHDF5::count_adjacencies( const Range& set, wid_t& result )
{
    ErrorCode rval;
    std::vector< wid_t > adj_list;
    Range::const_iterator iter      = set.begin();
    const Range::const_iterator end = set.end();
    result                          = 0;
    for( ; iter != end; ++iter )
    {
        adj_list.clear();
        rval = get_adjacencies( *iter, adj_list );
        CHK_MB_ERR_0( rval );

        if( adj_list.size() > 0 ) result += 2 + adj_list.size();
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::create_elem_table( const ExportSet& block, long num_entities, long& first_id_out )
{
    mhdf_Status status;
    hid_t handle;

    CHECK_OPEN_HANDLES;

    mhdf_addElement( filePtr, block.name(), block.type, &status );
    CHK_MHDF_ERR_0( status );

    handle = mhdf_createConnectivity( filePtr, block.name(), block.num_nodes, num_entities, &first_id_out, &status );
    CHK_MHDF_ERR_0( status );
    mhdf_closeData( filePtr, handle, &status );
    CHK_MHDF_ERR_0( status );

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::count_set_size( const Range& sets, long& contents_length_out, long& children_length_out,
                                     long& parents_length_out )
{
    ErrorCode rval;
    Range set_contents;
    long contents_length_set, children_length_set, parents_length_set;
    unsigned long flags;
    std::vector< wid_t > set_contents_ids;
    std::vector< SpecialSetData >::const_iterator si = specialSets.begin();

    contents_length_out = 0;
    children_length_out = 0;
    parents_length_out  = 0;

    for( Range::const_iterator iter = sets.begin(); iter != sets.end(); ++iter )
    {
        while( si != specialSets.end() && si->setHandle < *iter )
            ++si;

        if( si != specialSets.end() && si->setHandle == *iter )
        {
            contents_length_out += si->contentIds.size();
            children_length_out += si->childIds.size();
            parents_length_out += si->parentIds.size();
            ++si;
            continue;
        }

        rval = get_set_info( *iter, contents_length_set, children_length_set, parents_length_set, flags );
        CHK_MB_ERR_0( rval );

        // Check if can and should compress as ranges
        if( !( flags & MESHSET_ORDERED ) && contents_length_set )
        {
            set_contents.clear();
            rval = iFace->get_entities_by_handle( *iter, set_contents, false );
            CHK_MB_ERR_0( rval );

            bool blocked_list;
            rval = range_to_blocked_list( set_contents, set_contents_ids, blocked_list );
            CHK_MB_ERR_0( rval );

            if( blocked_list )
            {
                assert( set_contents_ids.size() % 2 == 0 );
                contents_length_set = set_contents_ids.size();
            }
        }

        contents_length_out += contents_length_set;
        children_length_out += children_length_set;
        parents_length_out += parents_length_set;
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::create_set_meta( long num_sets, long& first_id_out )
{
    hid_t handle;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    handle = mhdf_createSetMeta( filePtr, num_sets, &first_id_out, &status );
    CHK_MHDF_ERR_0( status );
    mhdf_closeData( filePtr, handle, &status );

    return MB_SUCCESS;
}

WriteHDF5::SpecialSetData* WriteHDF5::find_set_data( EntityHandle h )
{
    SpecialSetData tmp;
    tmp.setHandle = h;
    std::vector< SpecialSetData >::iterator i;
    i = std::lower_bound( specialSets.begin(), specialSets.end(), tmp, SpecSetLess() );
    return ( i == specialSets.end() || i->setHandle != h ) ? 0 : &*i;
}

ErrorCode WriteHDF5::create_set_tables( long num_set_contents, long num_set_children, long num_set_parents )
{
    hid_t handle;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    if( num_set_contents > 0 )
    {
        handle = mhdf_createSetData( filePtr, num_set_contents, &status );
        CHK_MHDF_ERR_0( status );
        mhdf_closeData( filePtr, handle, &status );
    }

    if( num_set_children > 0 )
    {
        handle = mhdf_createSetChildren( filePtr, num_set_children, &status );
        CHK_MHDF_ERR_0( status );
        mhdf_closeData( filePtr, handle, &status );
    }

    if( num_set_parents > 0 )
    {
        handle = mhdf_createSetParents( filePtr, num_set_parents, &status );
        CHK_MHDF_ERR_0( status );
        mhdf_closeData( filePtr, handle, &status );
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::get_tag_size( Tag tag, DataType& moab_type, int& num_bytes, int& type_size, int& array_length,
                                   mhdf_TagDataType& file_type, hid_t& hdf_type )
{
    ErrorCode rval;
    Tag type_handle;
    std::string tag_name, tag_type_name;

    CHECK_OPEN_HANDLES;

    // We return NULL for hdf_type if it can be determined from
    // the file_type.  The only case where it is non-zero is
    // if the user specified a specific type via a mesh tag.
    hdf_type            = (hid_t)0;
    bool close_hdf_type = false;

    rval = iFace->tag_get_data_type( tag, moab_type );
    CHK_MB_ERR_0( rval );
    rval = iFace->tag_get_length( tag, array_length );
    if( MB_VARIABLE_DATA_LENGTH == rval ) { array_length = MB_VARIABLE_LENGTH; }
    else if( MB_SUCCESS != rval )
        return error( rval );
    rval = iFace->tag_get_bytes( tag, num_bytes );
    if( MB_VARIABLE_DATA_LENGTH == rval )
        num_bytes = MB_VARIABLE_LENGTH;
    else if( MB_SUCCESS != rval )
        return error( rval );

    switch( moab_type )
    {
        case MB_TYPE_INTEGER:
            type_size      = sizeof( int );
            file_type      = mhdf_INTEGER;
            hdf_type       = H5T_NATIVE_INT;
            close_hdf_type = false;
            break;
        case MB_TYPE_DOUBLE:
            type_size      = sizeof( double );
            file_type      = mhdf_FLOAT;
            hdf_type       = H5T_NATIVE_DOUBLE;
            close_hdf_type = false;
            break;
        case MB_TYPE_BIT:
            type_size = sizeof( bool );
            file_type = mhdf_BITFIELD;
            assert( array_length <= 8 );
            hdf_type = H5Tcopy( H5T_NATIVE_B8 );
            H5Tset_precision( hdf_type, array_length );
            close_hdf_type = true;
            break;
        case MB_TYPE_HANDLE:
            type_size      = sizeof( EntityHandle );
            file_type      = mhdf_ENTITY_ID;
            hdf_type       = id_type;
            close_hdf_type = false;
            break;
        case MB_TYPE_OPAQUE:
            file_type = mhdf_OPAQUE;
            rval      = iFace->tag_get_name( tag, tag_name );
            CHK_MB_ERR_0( rval );
            tag_type_name = "__hdf5_tag_type_";
            tag_type_name += tag_name;
            rval = iFace->tag_get_handle( tag_type_name.c_str(), 0, MB_TYPE_OPAQUE, type_handle, MB_TAG_ANY );
            if( MB_TAG_NOT_FOUND == rval )
            {
                if( num_bytes == MB_VARIABLE_LENGTH )
                    type_size = 1;
                else
                    type_size = num_bytes;
                hdf_type       = H5Tcreate( H5T_OPAQUE, type_size );
                close_hdf_type = true;
            }
            else if( MB_SUCCESS == rval )
            {
                int hsize;
                rval = iFace->tag_get_bytes( type_handle, hsize );
                if( hsize != sizeof( hid_t ) ) return error( MB_FAILURE );

                const EntityHandle root = 0;
                rval                    = iFace->tag_get_data( type_handle, &root, 1, &hdf_type );
                if( rval != MB_SUCCESS ) return error( rval );

                type_size = H5Tget_size( hdf_type );
                if( type_size != num_bytes ) return error( MB_FAILURE );

                close_hdf_type = false;
            }
            else
                return error( rval );
            num_bytes    = array_length;
            array_length = ( num_bytes == MB_VARIABLE_LENGTH ) ? MB_VARIABLE_LENGTH : 1;
            break;
        default:
            break;
    }

    assert( num_bytes == MB_VARIABLE_LENGTH || ( moab_type == MB_TYPE_BIT && num_bytes == 1 ) ||
            array_length * type_size == num_bytes );

    if( num_bytes == MB_VARIABLE_LENGTH )
    {
        array_length = MB_VARIABLE_LENGTH;
        if( !close_hdf_type )
        {
            hdf_type = H5Tcopy( hdf_type );
            // close_hdf_type = true;
        }
    }
    else if( array_length > 1 && moab_type != MB_TYPE_BIT )
    {
        hsize_t len = array_length;
#if defined( H5Tarray_create_vers ) && ( H5Tarray_create_vers > 1 )
        hid_t temp_id = H5Tarray_create2( hdf_type, 1, &len );
#else
        hid_t temp_id = H5Tarray_create( hdf_type, 1, &len, NULL );
#endif
        if( close_hdf_type ) H5Tclose( hdf_type );
        hdf_type = temp_id;
    }
    else if( !close_hdf_type )
    {
        hdf_type = H5Tcopy( hdf_type );
        // close_hdf_type = true;
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::get_tag_data_length( const TagDesc& tag_info, const Range& range, unsigned long& result )
{
    ErrorCode rval;
    result = 0;

    // Split buffer into two pieces, one for pointers and one for sizes
    size_t step, remaining;
    step                    = bufferSize / ( sizeof( int ) + sizeof( void* ) );
    const void** ptr_buffer = reinterpret_cast< const void** >( dataBuffer );
    int* size_buffer        = reinterpret_cast< int* >( ptr_buffer + step );
    Range subrange;
    Range::const_iterator iter = range.begin();
    for( remaining = range.size(); remaining >= step; remaining -= step )
    {
        // Get subset of range containing 'count' entities
        Range::const_iterator end = iter;
        end += step;
        subrange.clear();
        subrange.merge( iter, end );
        iter = end;
        // Get tag sizes for entities
        rval = iFace->tag_get_by_ptr( tag_info.tag_id, subrange, ptr_buffer, size_buffer );
        if( MB_SUCCESS != rval ) return error( rval );
        // Sum lengths
        for( size_t i = 0; i < step; ++i )
            result += size_buffer[i];
    }
    // Process remaining
    subrange.clear();
    subrange.merge( iter, range.end() );
    assert( subrange.size() == remaining );
    rval = iFace->tag_get_by_ptr( tag_info.tag_id, subrange, ptr_buffer, size_buffer );
    if( MB_SUCCESS != rval ) return error( rval );
    for( size_t i = 0; i < remaining; ++i )
        result += size_buffer[i];

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::create_tag( const TagDesc& tag_data, unsigned long num_sparse_entities,
                                 unsigned long data_table_size )
{
    TagType mb_storage;
    DataType mb_type;
    mhdf_TagDataType mhdf_type;
    int tag_bytes, type_size, num_vals, storage;
    hid_t hdf_type = (hid_t)0;
    hid_t handles[3];
    std::string tag_name;
    ErrorCode rval;
    mhdf_Status status;

    CHECK_OPEN_HANDLES;

    // Get tag properties
    rval = iFace->tag_get_type( tag_data.tag_id, mb_storage );
    CHK_MB_ERR_0( rval );
    switch( mb_storage )
    {
        case MB_TAG_DENSE:
            storage = mhdf_DENSE_TYPE;
            break;
        case MB_TAG_SPARSE:
            storage = mhdf_SPARSE_TYPE;
            break;
        case MB_TAG_BIT:
            storage = mhdf_BIT_TYPE;
            break;
        case MB_TAG_MESH:
            storage = mhdf_MESH_TYPE;
            break;
        default:
            return error( MB_FAILURE );
    }
    rval = iFace->tag_get_name( tag_data.tag_id, tag_name );
    CHK_MB_ERR_0( rval );
    rval = get_tag_size( tag_data.tag_id, mb_type, tag_bytes, type_size, num_vals, mhdf_type, hdf_type );
    CHK_MB_ERR_0( rval );

    // Get default value
    const void *def_value, *mesh_value;
    int def_val_len, mesh_val_len;
    rval = iFace->tag_get_default_value( tag_data.tag_id, def_value, def_val_len );
    if( MB_ENTITY_NOT_FOUND == rval )
    {
        def_value   = 0;
        def_val_len = 0;
    }
    else if( MB_SUCCESS != rval )
    {
        H5Tclose( hdf_type );
        return error( rval );
    }

    // Get mesh value
    unsigned char byte;
    const EntityHandle root = 0;
    if( mb_storage == MB_TAG_BIT )
    {
        rval         = iFace->tag_get_data( tag_data.tag_id, &root, 1, &byte );
        mesh_value   = &byte;
        mesh_val_len = 1;
    }
    else
    {
        rval = iFace->tag_get_by_ptr( tag_data.tag_id, &root, 1, &mesh_value, &mesh_val_len );
    }
    if( MB_TAG_NOT_FOUND == rval )
    {
        mesh_value   = 0;
        mesh_val_len = 0;
    }
    else if( MB_SUCCESS != rval )
    {
        H5Tclose( hdf_type );
        return error( rval );
    }

    // For handle-type tags, need to convert from handles to file ids
    if( MB_TYPE_HANDLE == mb_type )
    {
        // Make sure there's room in the buffer for both
        assert( ( def_val_len + mesh_val_len ) * sizeof( long ) < (size_t)bufferSize );

        // Convert default value
        if( def_value )
        {
            memcpy( dataBuffer, def_value, def_val_len * sizeof( EntityHandle ) );
            convert_handle_tag( reinterpret_cast< EntityHandle* >( dataBuffer ), def_val_len );
            def_value = dataBuffer;
        }

        // Convert mesh value
        if( mesh_value )
        {
            EntityHandle* ptr = reinterpret_cast< EntityHandle* >( dataBuffer ) + def_val_len;
            memcpy( ptr, mesh_value, mesh_val_len * sizeof( EntityHandle ) );
            if( convert_handle_tag( ptr, mesh_val_len ) )
                mesh_value = ptr;
            else
                mesh_value = 0;
        }
    }

    if( MB_VARIABLE_LENGTH != tag_bytes )
    {
        // Write the tag description to the file
        mhdf_createTag( filePtr, tag_name.c_str(), mhdf_type, num_vals, storage, def_value, mesh_value, hdf_type,
                        mb_type == MB_TYPE_HANDLE ? id_type : 0, &status );
        CHK_MHDF_ERR_0( status );
        H5Tclose( hdf_type );

        // Create empty table for tag data
        if( num_sparse_entities )
        {
            mhdf_createSparseTagData( filePtr, tag_name.c_str(), num_sparse_entities, handles, &status );
            CHK_MHDF_ERR_0( status );
            mhdf_closeData( filePtr, handles[0], &status );
            mhdf_closeData( filePtr, handles[1], &status );
        }

        for( size_t i = 0; i < tag_data.dense_list.size(); ++i )
        {
            const ExportSet* ex = find( tag_data.dense_list[i] );
            assert( 0 != ex );
            handles[0] = mhdf_createDenseTagData( filePtr, tag_name.c_str(), ex->name(), ex->total_num_ents, &status );
            CHK_MHDF_ERR_0( status );
            mhdf_closeData( filePtr, handles[0], &status );
        }
    }
    else
    {
        mhdf_createVarLenTag( filePtr, tag_name.c_str(), mhdf_type, storage, def_value, def_val_len, mesh_value,
                              mesh_val_len, hdf_type, mb_type == MB_TYPE_HANDLE ? id_type : 0, &status );
        CHK_MHDF_ERR_0( status );
        H5Tclose( hdf_type );

        // Create empty table for tag data
        if( num_sparse_entities )
        {
            mhdf_createVarLenTagData( filePtr, tag_name.c_str(), num_sparse_entities, data_table_size, handles,
                                      &status );
            CHK_MHDF_ERR_0( status );
            mhdf_closeData( filePtr, handles[0], &status );
            mhdf_closeData( filePtr, handles[1], &status );
            mhdf_closeData( filePtr, handles[2], &status );
        }
    }

    return MB_SUCCESS;
}

ErrorCode WriteHDF5::get_num_sparse_tagged_entities( const TagDesc& tag, size_t& count )
{
    Range tmp;
    ErrorCode rval = get_sparse_tagged_entities( tag, tmp );
    count          = tmp.size();
    return rval;
}

ErrorCode WriteHDF5::get_sparse_tagged_entities( const TagDesc& tag, Range& results )
{
    results.clear();
    if( !tag.have_dense( setSet ) ) results.merge( setSet.range );
    std::list< ExportSet >::reverse_iterator e;
    for( e = exportList.rbegin(); e != exportList.rend(); ++e )
    {
        if( !tag.have_dense( *e ) ) results.merge( e->range );
    }
    if( !tag.have_dense( nodeSet ) ) results.merge( nodeSet.range );
    if( results.empty() ) return MB_SUCCESS;

    return iFace->get_entities_by_type_and_tag( 0, MBMAXTYPE, &tag.tag_id, 0, 1, results, Interface::INTERSECT );
}

void WriteHDF5::get_write_entities( Range& range )
{
    range.clear();
    range.merge( setSet.range );
    std::list< ExportSet >::reverse_iterator e;
    for( e = exportList.rbegin(); e != exportList.rend(); ++e )
        range.merge( e->range );
    range.merge( nodeSet.range );
}

void WriteHDF5::print_id_map() const
{
    print_id_map( std::cout, "" );
}

void WriteHDF5::print_id_map( std::ostream& s, const char* pfx ) const
{
    RangeMap< EntityHandle, wid_t >::const_iterator i;
    for( i = idMap.begin(); i != idMap.end(); ++i )
    {
        const char* n1 = CN::EntityTypeName( TYPE_FROM_HANDLE( i->begin ) );
        EntityID id    = ID_FROM_HANDLE( i->begin );
        if( 1 == i->count ) { s << pfx << n1 << " " << id << " -> " << i->value << std::endl; }
        else
        {
            const char* n2 = CN::EntityTypeName( TYPE_FROM_HANDLE( i->begin + i->count - 1 ) );
            if( n1 == n2 )
            {
                s << pfx << n1 << " " << id << "-" << id + i->count - 1 << " -> " << i->value << "-"
                  << i->value + i->count - 1 << std::endl;
            }
            else
            {
                s << pfx << n1 << " " << id << "-" << n1 << " " << ID_FROM_HANDLE( i->begin + i->count - 1 ) << " -> "
                  << i->value << "-" << i->value + i->count - 1 << std::endl;
            }
        }
    }
}

void WriteHDF5::print_times( const double* t ) const
{
    std::cout << "WriteHDF5:           " << t[TOTAL_TIME] << std::endl
              << "  gather mesh:       " << t[GATHER_TIME] << std::endl
              << "  create file:       " << t[CREATE_TIME] << std::endl
              << "    create nodes:    " << t[CREATE_NODE_TIME] << std::endl
              << "    negotiate types: " << t[NEGOTIATE_TYPES_TIME] << std::endl
              << "    create elem:     " << t[CREATE_ELEM_TIME] << std::endl
              << "    file id exch:    " << t[FILEID_EXCHANGE_TIME] << std::endl
              << "    create adj:      " << t[CREATE_ADJ_TIME] << std::endl
              << "    create set:      " << t[CREATE_SET_TIME] << std::endl
              << "      shared ids:    " << t[SHARED_SET_IDS] << std::endl
              << "      shared data:   " << t[SHARED_SET_CONTENTS] << std::endl
              << "      set offsets:   " << t[SET_OFFSET_TIME] << std::endl
              << "    create tags:     " << t[CREATE_TAG_TIME] << std::endl
              << "  coordinates:       " << t[COORD_TIME] << std::endl
              << "  connectivity:      " << t[CONN_TIME] << std::endl
              << "  sets:              " << t[SET_TIME] << std::endl
              << "    set descrip:     " << t[SET_META] << std::endl
              << "    set content:     " << t[SET_CONTENT] << std::endl
              << "    set parent:      " << t[SET_PARENT] << std::endl
              << "    set child:       " << t[SET_CHILD] << std::endl
              << "  adjacencies:       " << t[ADJ_TIME] << std::endl
              << "  tags:              " << t[TAG_TIME] << std::endl
              << "    dense data:      " << t[DENSE_TAG_TIME] << std::endl
              << "    sparse data:     " << t[SPARSE_TAG_TIME] << std::endl
              << "    var-len data:    " << t[VARLEN_TAG_TIME] << std::endl;
}

}  // namespace moab