iMeshP_MOAB.cpp

Go to the documentation of this file.

#include "moab/MOABConfig.h"
#include "iMeshP_extensions.h"
#include "iMesh_MOAB.hpp"
#include "moab/Core.hpp"
#include "moab/Range.hpp"
#include "moab/CN.hpp"
#include "moab/MeshTopoUtil.hpp"
#include "moab/FileOptions.hpp"
#include "moab/ParallelComm.hpp"
#include "MBParallelConventions.h"
#include "MBIter.hpp"

#define IS_BUILDING_MB
#include "Internals.hpp"
#undef IS_BUILDING_MB

#include <cassert>
#include <sstream>

#ifdef MOAB_HAVE_MPI
#include "moab_mpi.h"
#endif

using namespace moab;

/********************* Error Handling **************************/

#define FIXME printf( "Warning: function has incomplete implementation: %s\n", __func__ )

/******** Type-safe casting between MOAB and ITAPS types *********/

#ifndef MOAB_TEMPLATE_FUNC_SPECIALIZATION
// if no template specialization, disable some type checking
template < typename T, typename S >
inline T itaps_cast( S handle )
{
    assert( sizeof( S ) >= sizeof( T ) );
    return reinterpret_cast< T >( handle );
}
#else

// basic template method : only works to cast to equivalent types (no-op)
template < typename T, typename S >
inline T itaps_cast( S h )
{
    return h;
}
// verify size and do reinterpret cast
template < typename T >
inline T itaps_cast_internal_( EntityHandle h )
{
    assert( sizeof( T ) >= sizeof( EntityHandle ) );
    return reinterpret_cast< T >( h );
}
// verify size and do reinterpret cast
template < typename T >
inline EntityHandle* itaps_cast_ptr_( T* h )
{
    assert( sizeof( T ) == sizeof( EntityHandle ) );
    return reinterpret_cast< EntityHandle* >( h );
}
// verify size and do reinterpret cast
template < typename T >
inline const EntityHandle* itaps_cast_const_ptr_( const T* h )
{
    assert( sizeof( T ) == sizeof( EntityHandle ) );
    return reinterpret_cast< const EntityHandle* >( h );
}
// verify set-type handle before cast
template < typename T >
inline T itaps_set_cast_( EntityHandle h )
{
    assert( TYPE_FROM_HANDLE( h ) == MBENTITYSET );
    return itaps_cast_internal_< T >( h );
}

// define conversion routines between itaps handle and EntityHandle types
#define DECLARE_ALLOWED_ITAPS_CONVERSION( ITAPS_HANDLE_TYPE )                                       \
    template <>                                                                                     \
    inline ITAPS_HANDLE_TYPE itaps_cast< ITAPS_HANDLE_TYPE, EntityHandle >( EntityHandle h )        \
    {                                                                                               \
        return itaps_cast_internal_< ITAPS_HANDLE_TYPE >( h );                                      \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline EntityHandle itaps_cast< EntityHandle, ITAPS_HANDLE_TYPE >( ITAPS_HANDLE_TYPE handle )   \
    {                                                                                               \
        return reinterpret_cast< EntityHandle >( handle );                                          \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline EntityHandle* itaps_cast< EntityHandle*, ITAPS_HANDLE_TYPE* >( ITAPS_HANDLE_TYPE * ptr ) \
    {                                                                                               \
        return itaps_cast_ptr_( ptr );                                                              \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline const EntityHandle* itaps_cast< const EntityHandle*, const ITAPS_HANDLE_TYPE* >(         \
        const ITAPS_HANDLE_TYPE* ptr )                                                              \
    {                                                                                               \
        return itaps_cast_const_ptr_( ptr );                                                        \
    }

// define conversion routines between itaps handle and EntityHandle types
// but limit to EntityHandle for MBENTITYSET type.
#define DECLARE_ALLOWED_ITAPS_SET_CONVERSION( ITAPS_HANDLE_TYPE )                                   \
    template <>                                                                                     \
    inline ITAPS_HANDLE_TYPE itaps_cast< ITAPS_HANDLE_TYPE, EntityHandle >( EntityHandle h )        \
    {                                                                                               \
        return itaps_set_cast_< ITAPS_HANDLE_TYPE >( h );                                           \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline EntityHandle itaps_cast< EntityHandle, ITAPS_HANDLE_TYPE >( ITAPS_HANDLE_TYPE handle )   \
    {                                                                                               \
        return reinterpret_cast< EntityHandle >( handle );                                          \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline EntityHandle* itaps_cast< EntityHandle*, ITAPS_HANDLE_TYPE* >( ITAPS_HANDLE_TYPE * ptr ) \
    {                                                                                               \
        return itaps_cast_ptr_( ptr );                                                              \
    }                                                                                               \
                                                                                                    \
    template <>                                                                                     \
    inline const EntityHandle* itaps_cast< const EntityHandle*, const ITAPS_HANDLE_TYPE* >(         \
        const ITAPS_HANDLE_TYPE* ptr )                                                              \
    {                                                                                               \
        return itaps_cast_const_ptr_( ptr );                                                        \
    }

DECLARE_ALLOWED_ITAPS_SET_CONVERSION( iMeshP_PartitionHandle )
// DECLARE_ALLOWED_ITAPS_SET_CONVERSION( iMeshP_PartHandle )
DECLARE_ALLOWED_ITAPS_SET_CONVERSION( iBase_EntitySetHandle )
DECLARE_ALLOWED_ITAPS_CONVERSION( iBase_EntityHandle )

template <>
inline Tag itaps_cast< Tag, iBase_TagHandle >( iBase_TagHandle h )
{
    return reinterpret_cast< Tag >( h );
}
template <>
inline iBase_TagHandle itaps_cast< iBase_TagHandle, Tag >( Tag h )
{
    return reinterpret_cast< iBase_TagHandle >( h );
}

#endif

#define PCOMM ParallelComm::get_pcomm( MOABI, itaps_cast< EntityHandle >( partition_handle ) )

/*** static function implemented in iMesh_MOAB.cpp ***/

// Need a different function name for Tag because (currently)
// both Tag and iBase_EntityHandle are void**.
iBase_TagHandle itaps_tag_cast( Tag t )
{
    assert( sizeof( iBase_TagHandle ) >= sizeof( Tag ) );
    return reinterpret_cast< iBase_TagHandle >( t );
}

/********************* ITAPS arrays **************************/

// Handle returning Range in ITAPS array (do ALLOCATE_ARRAY and copy).
#define RANGE_TO_ITAPS_ARRAY( RANGE, NAME )                                                        \
    do                                                                                             \
    {                                                                                              \
        ALLOC_CHECK_ARRAY_NOFAIL( NAME, ( RANGE ).size() );                                        \
        std::copy( ( RANGE ).begin(), ( RANGE ).end(), itaps_cast< EntityHandle* >( *( NAME ) ) ); \
    } while( false )

static inline ErrorCode get_entities( Interface* iface, EntityHandle set, int type, int topology, Range& entities )
{
    if( topology != iMesh_ALL_TOPOLOGIES )
        return iface->get_entities_by_type( set, mb_topology_table[topology], entities );
    else if( type != iBase_ALL_TYPES )
        return iface->get_entities_by_dimension( set, type, entities );
    else
        return iface->get_entities_by_handle( set, entities );
}

/*
static inline ErrorCode remove_not_owned( ParallelComm* pcomm, Range& ents )
{
  ErrorCode rval;

  std::vector<unsigned char> pstatus(ents.size());
  rval = pcomm->get_moab()->tag_get_data(pcomm->pstatus_tag(), ents, &pstatus[0]);
  if (MB_SUCCESS != rval)
    return rval;

  Range::iterator i = ents.begin();
  std::vector<unsigned char>::const_iterator j;
  for (j = pstatus.begin(); j != pstatus.end(); ++j) {
    if (*j & PSTATUS_NOT_OWNED)
      i = ents.erase( i );
    else
      ++i;
  }

  return MB_SUCCESS;
}
*/

static inline ErrorCode count_owned( ParallelComm* pcomm, const Range& ents, int& n )
{
    ErrorCode rval;
    n = 0;

    std::vector< unsigned char > pstatus( ents.size() );
    rval = pcomm->get_moab()->tag_get_data( pcomm->pstatus_tag(), ents, &pstatus[0] );
    if( MB_SUCCESS != rval ) return rval;

    std::vector< unsigned char >::const_iterator j;
    for( j = pstatus.begin(); j != pstatus.end(); ++j )
        if( !( *j & PSTATUS_NOT_OWNED ) ) ++n;

    return MB_SUCCESS;
}

static void set_intersection_query( iMesh_Instance instance, iMeshP_PartHandle set1, iBase_EntitySetHandle set2,
                                    int type, int topo, Range& result, int* err )
{
    ErrorCode rval;

    if( !set1 )
    {
        rval = get_entities( MOABI, itaps_cast< EntityHandle >( set2 ), type, topo, result );CHKERR( rval, "Invalid Part handle" );
    }
    else if( !set2 )
    {
        rval = get_entities( MOABI, itaps_cast< EntityHandle >( set1 ), type, topo, result );CHKERR( rval, "Invalid set handle" );
    }
    else
    {
        Range r1, r2;
        rval = get_entities( MOABI, itaps_cast< EntityHandle >( set1 ), type, topo, r1 );CHKERR( rval, "Invalid Part handle" );
        rval = get_entities( MOABI, itaps_cast< EntityHandle >( set2 ), type, topo, r2 );CHKERR( rval, "Invalid set handle" );
        result.merge( intersect( r1, r2 ) );
    }

    RETURN( iBase_SUCCESS );
}

/********************* Iterators **************************/

static ErrorCode get_boundary_entities( ParallelComm* pcomm, EntityHandle part_handle, int entity_type,
                                        int entity_topology, int adj_part_id, Range& entities_out )
{
    int* adj_part_id_ptr = ( adj_part_id == iMeshP_ALL_PARTS ) ? 0 : &adj_part_id;

    Range iface_sets;
    ErrorCode rval = pcomm->get_interface_sets( part_handle, iface_sets, adj_part_id_ptr );
    if( MB_SUCCESS != rval ) return rval;

    for( Range::iterator i = iface_sets.begin(); i != iface_sets.end(); ++i )
    {
        rval = get_entities( pcomm->get_moab(), *i, entity_type, entity_topology, entities_out );
        if( MB_SUCCESS != rval ) return rval;
    }

    return MB_SUCCESS;
}

class PartBoundaryIter : public MBRangeIter
{
  private:
    ParallelComm* pComm;
    int adjPart;

  public:
    inline PartBoundaryIter( ParallelComm* pcomm, EntityHandle part_handle, iBase_EntityType entity_type,
                             iMesh_EntityTopology entity_topology, int adj_part_id, int array_sz )
        : MBRangeIter( entity_type, entity_topology, part_handle, array_sz ), pComm( pcomm ), adjPart( adj_part_id )
    {
    }

    virtual ErrorCode reset( Interface* )
    {
        iterData.clear();
        ErrorCode result = get_boundary_entities( pComm, entSet, entType, entTopo, adjPart, iterData );
        iterPos          = iterData.begin();
        return result;
    }
};

template < class Container >
class SetIntersectIter : public MBIter< Container >
{
  private:
    EntityHandle otherSet;

  public:
    SetIntersectIter( iBase_EntityType type, iMesh_EntityTopology topology, EntityHandle set, EntityHandle other_set,
                      int array_sz )
        : MBIter< Container >( type, topology, set, array_sz ), otherSet( other_set )
    {
    }
    virtual ~SetIntersectIter() {}

    inline ErrorCode intersect_with_set( Interface* mb, Range& range )
    {
        Range tmp;
        ErrorCode result;
        result = mb->get_entities_by_handle( otherSet, tmp );
        range  = intersect( range, tmp );
        return result;
    }

    inline ErrorCode intersect_with_set( Interface* mb, std::vector< EntityHandle >& list )
    {
        size_t w = 0;
        for( size_t r = 0; r < list.size(); ++r )
        {
            if( mb->contains_entities( otherSet, &list[r], 1 ) ) list[w++] = list[r];
        }
        list.resize( w );
        return MB_SUCCESS;
    }

    virtual ErrorCode reset( Interface* mb )
    {
        ErrorCode result = MBIter< Container >::reset( mb );
        if( MB_SUCCESS != result ) return result;

        result                       = intersect_with_set( mb, MBIter< Container >::iterData );
        MBIter< Container >::iterPos = MBIter< Container >::iterData.begin();
        return result;
    }
};

/********************* iMeshP API **************************/

#ifdef __cplusplus
extern "C" {
#endif

void iMeshP_createPartitionAll( iMesh_Instance instance,
                                /*in*/ MPI_Comm communicator,
                                /*out*/ iMeshP_PartitionHandle* partition_handle, int* err )
{
    *partition_handle = 0;

    Tag prtn_tag;
    ErrorCode rval = MOABI->tag_get_handle( PARALLEL_PARTITIONING_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag,
                                            MB_TAG_SPARSE | MB_TAG_CREAT );CHKERR( rval, "tag creation failed" );

    EntityHandle handle;
    rval = MOABI->create_meshset( MESHSET_SET, handle );CHKERR( rval, "set creation failed" );
    ParallelComm* pcomm = ParallelComm::get_pcomm( MOABI, handle, &communicator );
    if( !pcomm )
    {
        MOABI->delete_entities( &handle, 1 );
        RETURN( iBase_FAILURE );
    }

    // set the value of pcomm id, to the partitioning tag, although this is not used
    // we just need the tag to be set
    int pid = pcomm->get_id();
    rval    = MOABI->tag_set_data( prtn_tag, &handle, 1, &pid );CHKERR( rval, "tag creation failed" );

    *partition_handle = itaps_cast< iMeshP_PartitionHandle >( handle );
    RETURN( iBase_SUCCESS );
}

void iMeshP_destroyPartitionAll( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( pcomm ) delete pcomm;
    EntityHandle handle = itaps_cast< EntityHandle >( partition_handle );
    ErrorCode rval      = MOABI->delete_entities( &handle, 1 );CHKERR( rval, "entity deletion failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartIdFromPartHandle( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                     const iMeshP_PartHandle part_handle, iMeshP_Part* part_id, int* err )
{
    int junk1 = 1, junk2;
    iMeshP_getPartIdsFromPartHandlesArr( instance, partition_handle, &part_handle, 1, &part_id, &junk1, &junk2, err );
}

void iMeshP_getPartHandleFromPartId( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                     iMeshP_Part part_id, iMeshP_PartHandle* part_handle, int* err )
{
    int junk1 = 1, junk2;
    iMeshP_getPartHandlesFromPartsIdsArr( instance, partition_handle, &part_id, 1, &part_handle, &junk1, &junk2, err );
}

void iMeshP_getPartIdsFromPartHandlesArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                          const iMeshP_PartHandle* part_handles, const int part_handles_size,
                                          iMeshP_Part** part_ids, int* part_ids_allocated, int* part_ids_size,
                                          int* err )
{
    ErrorCode rval;
    ParallelComm* pcomm = PCOMM;
    ALLOC_CHECK_ARRAY( part_ids, part_handles_size );
    for( int i = 0; i < part_handles_size; ++i )
    {
        int id;
        rval             = pcomm->get_part_id( itaps_cast< EntityHandle >( part_handles[i] ), id );
        ( *part_ids )[i] = id;CHKERR( rval, "error getting part id" );
    }
    KEEP_ARRAY( part_ids );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartHandlesFromPartsIdsArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                           const iMeshP_Part* part_ids, const int part_ids_size,
                                           iMeshP_PartHandle** part_handles, int* part_handles_allocated,
                                           int* part_handles_size, int* err )
{
    ErrorCode rval;
    ParallelComm* pcomm = PCOMM;
    ALLOC_CHECK_ARRAY( part_handles, part_ids_size );
    for( int i = 0; i < part_ids_size; ++i )
    {
        EntityHandle handle;
        rval = pcomm->get_part_handle( part_ids[i], handle );CHKERR( rval, "error getting part handle" );
        ( *part_handles )[i] = itaps_cast< iMeshP_PartHandle >( handle );
    }
    KEEP_ARRAY( part_handles );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartitionComm( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle,
                              MPI_Comm* communicator_out, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) RETURN( iBase_FAILURE );
    *communicator_out = pcomm->proc_config().proc_comm();
    RETURN( iBase_SUCCESS );
}

void iMeshP_syncPartitionAll( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );
    ErrorCode rval = pcomm->collective_sync_partition();CHKERR( rval, "collective sync failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumPartitions( iMesh_Instance instance, int* num_partitions_out, int* err )
{
    std::vector< ParallelComm* > pcomms;
    ErrorCode rval = ParallelComm::get_all_pcomm( MOABI, pcomms );CHKERR( rval, "Internal error retreiving PComms" );

    std::vector< ParallelComm* >::iterator i;
    *num_partitions_out = 0;
    for( i = pcomms.begin(); i != pcomms.end(); ++i )
        if( ( *i )->get_partitioning() ) ( *num_partitions_out )++;

    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartitions( iMesh_Instance instance, iMeshP_PartitionHandle** partition_handle,
                           int* partition_handle_allocated, int* partition_handle_size, int* err )
{
    std::vector< ParallelComm* > pcomms;
    ErrorCode rval = ParallelComm::get_all_pcomm( MOABI, pcomms );CHKERR( rval, "Internal error retreiving PComms" );

    std::vector< ParallelComm* >::iterator i;
    int count = 0;
    for( i = pcomms.begin(); i != pcomms.end(); ++i )
        if( ( *i )->get_partitioning() ) ++count;
    ALLOC_CHECK_ARRAY_NOFAIL( partition_handle, count );

    *partition_handle_size = 0;
    for( i = pcomms.begin(); i != pcomms.end(); ++i )
        if( ( *i )->get_partitioning() )
            ( *partition_handle )[( *partition_handle_size )++] =
                itaps_cast< iMeshP_PartitionHandle >( ( *i )->get_partitioning() );

    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumGlobalParts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                               int* num_global_part, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    ErrorCode rval = pcomm->get_global_part_count( *num_global_part );CHKERR( rval, "PComm::get_global_part_count failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumLocalParts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                              int* num_local_part, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    *num_local_part = pcomm->partition_sets().size();
    RETURN( iBase_SUCCESS );
}

void iMeshP_getLocalParts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                           iMeshP_PartHandle** part_handles, int* part_handles_allocated, int* part_handles_size,
                           int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    RANGE_TO_ITAPS_ARRAY( pcomm->partition_sets(), part_handles );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getRankOfPart( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                           const iMeshP_Part part_id, int* rank, int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_getRankOfPartArr( instance, partition_handle, &part_id, 1, &rank, &junk1, &junk2, err );
}

void iMeshP_getRankOfPartArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                              const iMeshP_Part* part_ids, const int part_ids_size, int** rank, int* rank_allocated,
                              int* rank_size, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    ALLOC_CHECK_ARRAY( rank, part_ids_size );
    ErrorCode rval = MB_SUCCESS;
    for( int i = 0; i < part_ids_size; ++i )
    {
        rval = pcomm->get_part_owner( part_ids[i], ( *rank )[i] );CHKERR( rval, "PComm::get_part_owner failed" );
    }
    KEEP_ARRAY( rank );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumOfTypeAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                             const iBase_EntitySetHandle entity_set_handle, const int entity_type, int* num_type,
                             int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    Range entities;
    ErrorCode rval = get_entities( MOABI, itaps_cast< EntityHandle >( entity_set_handle ), entity_type,
                                   iMesh_ALL_TOPOLOGIES, entities );
    int count      = 0;
    if( MB_SUCCESS == rval ) rval = count_owned( pcomm, entities, count );

    int vals[2] = { count, rval }, sums[2];
    int ierr    = MPI_Allreduce( vals, sums, 2, MPI_INT, MPI_SUM, pcomm->proc_config().proc_comm() );
    assert( iBase_SUCCESS == 0 );
    if( ierr || sums[1] ) RETURN( iBase_FAILURE );

    *num_type = sums[0];
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumOfTopoAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                             const iBase_EntitySetHandle entity_set_handle, const int entity_topology, int* num_topo,
                             int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    Range entities;
    ErrorCode rval = get_entities( MOABI, itaps_cast< EntityHandle >( entity_set_handle ), iBase_ALL_TYPES,
                                   entity_topology, entities );
    int count      = 0;
    if( MB_SUCCESS == rval ) rval = count_owned( pcomm, entities, count );

    int vals[2] = { count, rval }, sums[2];
    int ierr    = MPI_Allreduce( vals, sums, 2, MPI_INT, MPI_SUM, pcomm->proc_config().proc_comm() );
    assert( iBase_SUCCESS == 0 );
    if( ierr || sums[1] ) RETURN( iBase_FAILURE );

    *num_topo = sums[0];
    RETURN( iBase_SUCCESS );
}

void iMeshP_createPart( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle,
                        iMeshP_PartHandle* part_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    EntityHandle h;
    ErrorCode rval = pcomm->create_part( h );CHKERR( rval, "Part creation failed" );
    *part_handle = itaps_cast< iMeshP_PartHandle >( h );
    RETURN( iBase_SUCCESS );
}

void iMeshP_destroyPart( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle,
                         iMeshP_PartHandle part_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    ErrorCode rval = pcomm->destroy_part( itaps_cast< EntityHandle >( part_handle ) );CHKERR( rval, "Part destruction failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumPartNbors( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle,
                             iMeshP_PartHandle part_handle, int entity_type, int* num_part_nbors, int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_getNumPartNborsArr( instance, partition_handle, &part_handle, 1, entity_type, &num_part_nbors, &junk1,
                               &junk2, err );
}

void iMeshP_getNumPartNborsArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                const iMeshP_PartHandle* part_handles, const int part_handles_size, int /*entity_type*/,
                                int** num_part_nbors, int* num_part_nbors_allocated, int* num_part_nbors_size,
                                int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    ALLOC_CHECK_ARRAY( num_part_nbors, part_handles_size );

    int n, neighbors[MAX_SHARING_PROCS];
    ErrorCode rval;
    for( int i = 0; i < part_handles_size; ++i )
    {
        EntityHandle h = itaps_cast< EntityHandle >( part_handles[i] );
        rval           = pcomm->get_part_neighbor_ids( h, neighbors, n );CHKERR( rval, "error getting neighbor ids" );
        ( *num_part_nbors )[i] = n;
    }

    KEEP_ARRAY( num_part_nbors );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartNbors( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                          const iMeshP_PartHandle part_handle, int entity_type, int* num_part_nbors,
                          iMeshP_Part** nbor_part_ids, int* nbor_part_ids_allocated, int* nbor_part_ids_size, int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_getPartNborsArr( instance, partition_handle, &part_handle, 1, entity_type, &num_part_nbors, &junk1, &junk2,
                            nbor_part_ids, nbor_part_ids_allocated, nbor_part_ids_size, err );
}

void iMeshP_getPartNborsArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                             const iMeshP_PartHandle* part_handles, const int part_handles_size, int /*entity_type*/,
                             int** num_part_nbors, int* num_part_nbors_allocated, int* num_part_nbors_size,
                             iMeshP_Part** nbor_part_ids, int* nbor_part_ids_allocated, int* nbor_part_ids_size,
                             int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    ALLOC_CHECK_ARRAY( num_part_nbors, part_handles_size );

    std::vector< int > all_neighbors;
    int n, pnbor[MAX_SHARING_PROCS];
    ErrorCode rval;
    for( int i = 0; i < part_handles_size; ++i )
    {
        EntityHandle h = itaps_cast< EntityHandle >( part_handles[i] );
        rval           = pcomm->get_part_neighbor_ids( h, pnbor, n );CHKERR( rval, "error getting neighbor ids" );
        ( *num_part_nbors )[i] = n;
        std::copy( pnbor, pnbor + n, std::back_inserter( all_neighbors ) );
    }

    ALLOC_CHECK_ARRAY_NOFAIL( nbor_part_ids, all_neighbors.size() );
    memcpy( *nbor_part_ids, &all_neighbors[0], sizeof( int ) * all_neighbors.size() );

    KEEP_ARRAY( num_part_nbors );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumPartBdryEnts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                const iMeshP_PartHandle part_handle, const int entity_type, const int entity_topology,
                                const iMeshP_Part target_part_id, int* num_entities, int* err )
{
    Range entities;
    ErrorCode rval = get_boundary_entities( PCOMM, itaps_cast< EntityHandle >( part_handle ), entity_type,
                                            entity_topology, target_part_id, entities );CHKERR( rval, "failed to get boundary entities" );
    *num_entities = entities.size();
    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartBdryEnts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                             const iMeshP_PartHandle part_handle, const int entity_type, const int entity_topology,
                             const iMeshP_Part target_part_id, iBase_EntityHandle** entity_handles,
                             int* entity_handles_allocated, int* entity_handles_size, int* err )
{
    Range entities;
    ErrorCode rval = get_boundary_entities( PCOMM, itaps_cast< EntityHandle >( part_handle ), entity_type,
                                            entity_topology, target_part_id, entities );CHKERR( rval, "failed to get boundary entities" );
    RANGE_TO_ITAPS_ARRAY( entities, entity_handles );
    RETURN( iBase_SUCCESS );
}

void iMeshP_initPartBdryEntIter( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                 const iMeshP_PartHandle part_handle, const int entity_type, const int entity_topology,
                                 const iMeshP_Part nbor_part_id, iBase_EntityIterator* entity_iterator, int* err )
{
    iMeshP_initPartBdryEntArrIter( instance, partition_handle, part_handle, entity_type, entity_topology, 1,
                                   nbor_part_id, reinterpret_cast< iBase_EntityArrIterator* >( entity_iterator ), err );
}

void iMeshP_initPartBdryEntArrIter( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                    const iMeshP_PartHandle part_handle, const int entity_type,
                                    const int entity_topology, const int array_size, const iMeshP_Part nbor_part_id,
                                    iBase_EntityArrIterator* entity_iterator, int* err )
{
    *entity_iterator =
        new PartBoundaryIter( PCOMM, itaps_cast< EntityHandle >( part_handle ), (iBase_EntityType)entity_type,
                              (iMesh_EntityTopology)entity_topology, nbor_part_id, array_size );
    ErrorCode result = ( *entity_iterator )->reset( MOABI );
    if( MB_SUCCESS != result ) delete *entity_iterator;CHKERR( result, "iMesh_initEntArrIter: ERROR getting entities of proper type or topology." );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumOfType( iMesh_Instance instance, const iMeshP_PartitionHandle, const iMeshP_PartHandle part_handle,
                          const iBase_EntitySetHandle entity_set_handle, const int entity_type, int* num_type,
                          int* err )
{
    Range r;
    set_intersection_query( instance, part_handle, entity_set_handle, entity_type, iMesh_ALL_TOPOLOGIES, r, err );
    *num_type = r.size();
}

void iMeshP_getNumOfTopo( iMesh_Instance instance, const iMeshP_PartitionHandle /*partition_handle*/,
                          const iMeshP_PartHandle part_handle, const iBase_EntitySetHandle entity_set_handle,
                          const int entity_topology, int* num_topo, int* err )
{
    Range r;
    set_intersection_query( instance, part_handle, entity_set_handle, iBase_ALL_TYPES, entity_topology, r, err );
    *num_topo = r.size();
}

void iMeshP_getAdjEntIndices( iMesh_Instance instance, iMeshP_PartitionHandle partition, iMeshP_PartHandle part,
                              iBase_EntitySetHandle entity_set_handle, int entity_type_requestor,
                              int entity_topology_requestor, int entity_type_requested,
                              iBase_EntityHandle** entity_handles, int* entity_handles_allocated,
                              int* entity_handles_size, iBase_EntityHandle** adj_entity_handles,
                              int* adj_entity_handles_allocated, int* adj_entity_handles_size, int** adj_entity_indices,
                              int* adj_entity_indices_allocated, int* adj_entity_indices_size, int** offset,
                              int* offset_allocated, int* offset_size, int* err )
{
    const int allocated_entity_handles = ( *entity_handles_allocated == 0 );
    const int allocated_indices        = ( *adj_entity_indices_allocated == 0 );
    const int allocated_offset         = ( *offset_allocated == 0 );

    // get source entities
    iMeshP_getEntities( instance, partition, part, entity_set_handle, entity_type_requestor, entity_topology_requestor,
                        entity_handles, entity_handles_allocated, entity_handles_size, err );
    if( iBase_SUCCESS != *err ) return;

    // get adjacencies
    iBase_EntityHandle* all_adj_handles = 0;
    int size = 0, alloc = 0;
    iMesh_getEntArrAdj( instance, *entity_handles, *entity_handles_size, entity_type_requested, &all_adj_handles,
                        &alloc, &size, offset, offset_allocated, offset_size, err );
    if( *err != iBase_SUCCESS )
    {
        if( allocated_entity_handles )
        {
            free( *entity_handles );
            *entity_handles           = 0;
            *entity_handles_allocated = 0;
        }
        return;
    }

    // allocate or check size of adj_entity_indices
    *adj_entity_indices_size = size;
    if( allocated_indices )
    {
        *adj_entity_indices = (int*)malloc( sizeof( iBase_EntityHandle ) * size );
        if( !*adj_entity_indices )
            *err = iBase_MEMORY_ALLOCATION_FAILED;
        else
            *adj_entity_indices_allocated = size;
    }
    else if( *adj_entity_indices_allocated < size )
    {
        *err = iBase_BAD_ARRAY_DIMENSION;
    }
    if( iBase_SUCCESS != *err )
    {
        free( all_adj_handles );
        if( allocated_entity_handles )
        {
            free( *entity_handles );
            *entity_handles           = 0;
            *entity_handles_allocated = 0;
        }
        if( allocated_offset )
        {
            free( *offset );
            *offset           = 0;
            *offset_allocated = 0;
        }
        return;
    }

    // Now create an array of unique sorted handles from all_adj_handles.
    // We need to create a copy because we still need all_adj_handles.  We
    // will eventually need to copy the resulting unique list into
    // adj_entity_handles, so if adj_entity_handles is already allocated and
    // of sufficient size, use it rather than allocating another temporary.
    iBase_EntityHandle* unique_adj = 0;
    if( *adj_entity_handles_allocated >= size ) { unique_adj = *adj_entity_handles; }
    else
    {
        unique_adj = (iBase_EntityHandle*)malloc( sizeof( iBase_EntityHandle ) * size );
    }
    std::copy( all_adj_handles, all_adj_handles + size, unique_adj );
    std::sort( unique_adj, unique_adj + size );
    *adj_entity_handles_size = std::unique( unique_adj, unique_adj + size ) - unique_adj;

    // If we created a temporary array for unique_adj rather than using
    // already allocated space in adj_entity_handles, allocate adj_entity_handles
    // and copy the unique handle list into it
    if( *adj_entity_handles != unique_adj )
    {
        if( !*adj_entity_handles_allocated )
        {
            *adj_entity_handles =
                (iBase_EntityHandle*)malloc( sizeof( iBase_EntityHandle ) * *adj_entity_handles_size );
            if( !*adj_entity_handles )
                *err = iBase_MEMORY_ALLOCATION_FAILED;
            else
                *adj_entity_handles_allocated = *adj_entity_handles_size;
        }
        else if( *adj_entity_handles_allocated < *adj_entity_handles_size )
            *err = iBase_BAD_ARRAY_DIMENSION;
        if( iBase_SUCCESS != *err )
        {
            free( unique_adj );
            free( all_adj_handles );
            if( allocated_entity_handles )
            {
                free( *entity_handles );
                *entity_handles           = 0;
                *entity_handles_allocated = 0;
            }
            if( allocated_offset )
            {
                free( *offset );
                *offset           = 0;
                *offset_allocated = 0;
            }
            if( allocated_indices )
            {
                free( *adj_entity_indices );
                *adj_entity_indices           = 0;
                *adj_entity_indices_allocated = 0;
            }
            return;
        }

        std::copy( unique_adj, unique_adj + *adj_entity_handles_size, *adj_entity_handles );
        free( unique_adj );
        unique_adj = *adj_entity_handles;
    }

    // convert from adjacency list to indices into unique_adj
    for( int i = 0; i < *adj_entity_indices_size; ++i )
        ( *adj_entity_indices )[i] =
            std::lower_bound( unique_adj, unique_adj + *adj_entity_handles_size, all_adj_handles[i] ) - unique_adj;
    free( all_adj_handles );
}

void iMeshP_getEntities( iMesh_Instance instance, const iMeshP_PartitionHandle, const iMeshP_PartHandle part_handle,
                         const iBase_EntitySetHandle entity_set_handle, const int entity_type,
                         const int entity_topology, iBase_EntityHandle** entity_handles, int* entity_handles_allocated,
                         int* entity_handles_size, int* err )
{
    Range r;
    set_intersection_query( instance, part_handle, entity_set_handle, entity_type, entity_topology, r, err );
    if( iBase_SUCCESS != *err ) return;

    RANGE_TO_ITAPS_ARRAY( r, entity_handles );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getAdjEntities( iMesh_Instance instance, const iMeshP_PartitionHandle /*partition_handle*/,
                            const iMeshP_PartHandle part_handle, const iBase_EntitySetHandle entity_set_handle,
                            const int entity_type_requestor, const int entity_topology_requestor,
                            const int entity_type_requested, iBase_EntityHandle** adj_entity_handles,
                            int* adj_entity_handles_allocated, int* adj_entity_handles_size, int** offset,
                            int* offset_allocated, int* offset_size, int** in_entity_set, int* in_entity_set_allocated,
                            int* in_entity_set_size, int* err )
{
    ErrorCode rval;
    Range r;
    set_intersection_query( instance, part_handle, entity_set_handle, entity_type_requestor, entity_topology_requestor,
                            r, err );
    if( iBase_SUCCESS != *err ) return;

    // count adjacencies
    std::vector< EntityHandle > tmp_storage;
    int num_adj = 0;
    int num_conn;
    const EntityHandle* conn_ptr;
    for( Range::iterator i = r.begin(); i != r.end(); ++i )
    {
        if( entity_type_requested || TYPE_FROM_HANDLE( *i ) == MBPOLYHEDRON )
        {
            tmp_storage.clear();
            rval = MOABI->get_adjacencies( &*i, 1, entity_type_requested, false, tmp_storage );CHKERR( rval, "get_adjacencies failed" );
            num_adj += tmp_storage.size();
        }
        else
        {
            rval = MOABI->get_connectivity( *i, conn_ptr, num_conn, false, &tmp_storage );CHKERR( rval, "get_connectivity failed" );
            num_adj += num_conn;
        }
    }

    // get adjacencies
    ALLOC_CHECK_ARRAY( adj_entity_handles, num_adj );
    ALLOC_CHECK_ARRAY( offset, r.size() );
    int arr_pos      = 0;
    int* offset_iter = *offset;
    for( Range::iterator i = r.begin(); i != r.end(); ++i )
    {
        *offset_iter = arr_pos;
        ++offset_iter;

        tmp_storage.clear();
        rval = MOABI->get_adjacencies( &*i, 1, entity_type_requested, false, tmp_storage );CHKERR( rval, "get_adjacencies failed" );
        for( std::vector< EntityHandle >::iterator j = tmp_storage.begin(); j != tmp_storage.end(); ++j )
        {
            ( *adj_entity_handles )[arr_pos] = itaps_cast< iBase_EntityHandle >( *j );
            ++arr_pos;
        }
    }

    // get in_entity_set
    iMesh_isEntArrContained( instance, entity_set_handle, *adj_entity_handles, *adj_entity_handles_size, in_entity_set,
                             in_entity_set_allocated, in_entity_set_size, err );

    if( iBase_SUCCESS == *err )
    {
        KEEP_ARRAY( adj_entity_handles );
        KEEP_ARRAY( offset );
    }
}

void iMeshP_initEntIter( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                         const iMeshP_PartHandle part_handle, const iBase_EntitySetHandle entity_set_handle,
                         const int requested_entity_type, const int requested_entity_topology,
                         iBase_EntityIterator* entity_iterator, int* err )
{
    iMeshP_initEntArrIter( instance, partition_handle, part_handle, entity_set_handle, requested_entity_type,
                           requested_entity_topology, 1,
                           reinterpret_cast< iBase_EntityArrIterator* >( entity_iterator ), err );
}

void iMeshP_initEntArrIter( iMesh_Instance instance, const iMeshP_PartitionHandle /*partition_handle*/,
                            const iMeshP_PartHandle part_handle, const iBase_EntitySetHandle entity_set_handle,
                            const int requested_entity_type, const int requested_entity_topology,
                            const int requested_array_size, iBase_EntityArrIterator* entArr_iterator, int* err )
{
    if( !entity_set_handle || entity_set_handle == part_handle )
    {
        iMesh_initEntArrIter( instance, part_handle, requested_entity_type, requested_entity_topology,
                              requested_array_size,
                              0,  // TODO: update this function for "resilient" arg
                              entArr_iterator, err );
    }
    else
    {
        unsigned flags;
        ErrorCode result = MOABI->get_meshset_options( itaps_cast< EntityHandle >( entity_set_handle ), flags );CHKERR( result, "Invalid entity set handle" );
        if( flags & MESHSET_ORDERED )
            *entArr_iterator = new SetIntersectIter< std::vector< EntityHandle > >(
                (iBase_EntityType)requested_entity_type, (iMesh_EntityTopology)requested_entity_topology,
                itaps_cast< EntityHandle >( entity_set_handle ), itaps_cast< EntityHandle >( part_handle ),
                requested_array_size );
        else
            *entArr_iterator =
                new SetIntersectIter< Range >( (iBase_EntityType)requested_entity_type,
                                               (iMesh_EntityTopology)requested_entity_topology,
                                               itaps_cast< EntityHandle >( entity_set_handle ),
                                               itaps_cast< EntityHandle >( part_handle ), requested_array_size );
        result = ( *entArr_iterator )->reset( MOABI );
        if( MB_SUCCESS != result ) delete *entArr_iterator;CHKERR( result, "iMesh_initEntArrIter: ERROR getting entities of proper type or topology." );
        RETURN( iBase_SUCCESS );
    }
}

void iMeshP_getEntOwnerPart( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                             const iBase_EntityHandle entity_handle, iMeshP_Part* part_id, int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_getEntOwnerPartArr( instance, partition_handle, &entity_handle, 1, &part_id, &junk1, &junk2, err );
}

void iMeshP_getEntOwnerPartArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                const iBase_EntityHandle* entity_handles, const int entity_handles_size,
                                iMeshP_Part** part_ids, int* part_ids_allocated, int* part_ids_size, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int id;
    ALLOC_CHECK_ARRAY( part_ids, entity_handles_size );
    ErrorCode rval = MB_SUCCESS;
    for( int i = 0; i < entity_handles_size; ++i )
    {
        EntityHandle h   = itaps_cast< EntityHandle >( entity_handles[i] );
        rval             = pcomm->get_owning_part( h, id );
        ( *part_ids )[i] = id;CHKERR( rval, "Failet get part owner" );
    }
    KEEP_ARRAY( part_ids );
    RETURN( iBase_SUCCESS );
}

void iMeshP_isEntOwner( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                        const iMeshP_PartHandle part_handle, const iBase_EntityHandle entity_handle, int* is_owner,
                        int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_isEntOwnerArr( instance, partition_handle, part_handle, &entity_handle, 1, &is_owner, &junk1, &junk2, err );
}

void iMeshP_isEntOwnerArr( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                           const iMeshP_PartHandle part_handle, const iBase_EntityHandle* entity_handles,
                           const int entity_handles_size, int** is_owner, int* is_owner_allocated, int* is_owner_size,
                           int* err )
{
    ErrorCode rval;
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int id;
    rval = pcomm->get_part_id( itaps_cast< EntityHandle >( part_handle ), id );CHKERR( rval, "error getting part id" );

    ALLOC_CHECK_ARRAY( is_owner, entity_handles_size );
    *is_owner_size = entity_handles_size;

    int owner;
    for( int i = 0; i < entity_handles_size; ++i )
    {
        rval = pcomm->get_owner( itaps_cast< EntityHandle >( entity_handles[i] ), owner );CHKERR( rval, "error getting owner" );
        ( *is_owner )[i] = ( owner == id );
    }

    KEEP_ARRAY( is_owner );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getEntStatus( iMesh_Instance instance,
                          /*in*/ const iMeshP_PartitionHandle partition_handle,
                          /*in*/ const iMeshP_PartHandle part_handle,
                          /*in*/ const iBase_EntityHandle entity_handle,
                          /*out*/ int* par_status,  // Values=INTERNAL,BOUNDARY,GHOST
                          int* err )
{
    int junk1 = 1, junk2 = 1;
    iMeshP_getEntStatusArr( instance, partition_handle, part_handle, &entity_handle, 1, &par_status, &junk1, &junk2,
                            err );
}

void iMeshP_getEntStatusArr( iMesh_Instance instance,
                             /*in*/ const iMeshP_PartitionHandle partition_handle,
                             /*in*/ const iMeshP_PartHandle /*part_handle*/,
                             /*in*/ const iBase_EntityHandle* entity_handles,
                             /*in*/ const int entity_handles_size,
                             /*inout*/ int** par_status,  // Values=INTERNAL,BOUNDARY,GHOST
                             /*inout*/ int* par_status_allocated,
                             /*inout*/ int* par_status_size, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    std::vector< unsigned char > pstatus( entity_handles_size );
    ErrorCode result = MOABI->tag_get_data( pcomm->pstatus_tag(), itaps_cast< const EntityHandle* >( entity_handles ),
                                            entity_handles_size, &pstatus[0] );CHKERR( result, "error getting pstatus_tag" );

    ALLOC_CHECK_ARRAY( par_status, entity_handles_size );
    for( int i = 0; i < entity_handles_size; i++ )
    {
        if( !pstatus[i] )
            ( *par_status )[i] = iMeshP_INTERNAL;
        else if( pstatus[i] & PSTATUS_GHOST )
            ( *par_status )[i] = iMeshP_GHOST;
        else if( pstatus[i] & PSTATUS_INTERFACE )
            ( *par_status )[i] = iMeshP_BOUNDARY;
    }

    KEEP_ARRAY( par_status );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getNumCopies( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                          const iBase_EntityHandle entity_handle, int* num_copies_ent, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int ids[MAX_SHARING_PROCS];
    ErrorCode rval = pcomm->get_sharing_parts( itaps_cast< EntityHandle >( entity_handle ), ids, *num_copies_ent );CHKERR( rval, "ParallelComm::get_sharing_parts failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getCopyParts( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                          const iBase_EntityHandle entity_handle, iMeshP_Part** part_ids, int* part_ids_allocated,
                          int* part_ids_size, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int ids[MAX_SHARING_PROCS], num_ids;
    ErrorCode rval = pcomm->get_sharing_parts( itaps_cast< EntityHandle >( entity_handle ), ids, num_ids );CHKERR( rval, "ParallelComm::get_sharing_parts failed" );
    ALLOC_CHECK_ARRAY_NOFAIL( part_ids, num_ids );
    std::copy( ids, ids + num_ids, *part_ids );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getCopies( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                       const iBase_EntityHandle entity_handle, iMeshP_Part** part_ids, int* part_ids_allocated,
                       int* part_ids_size, iBase_EntityHandle** copies_entity_handles,
                       int* copies_entity_handles_allocated, int* copies_entity_handles_size, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int ids[MAX_SHARING_PROCS], num_ids;
    EntityHandle handles[MAX_SHARING_PROCS];
    ErrorCode rval = pcomm->get_sharing_parts( itaps_cast< EntityHandle >( entity_handle ), ids, num_ids, handles );CHKERR( rval, "ParallelComm::get_sharing_parts failed" );
    ALLOC_CHECK_ARRAY_NOFAIL( part_ids, num_ids );
    ALLOC_CHECK_ARRAY_NOFAIL( copies_entity_handles, num_ids );
    for( int i = 0; i < num_ids; ++i )
    {
        ( *part_ids )[i]              = ids[i];
        ( *copies_entity_handles )[i] = itaps_cast< iBase_EntityHandle >( handles[i] );
    }
    RETURN( iBase_SUCCESS );
}

void iMeshP_getCopyOnPart( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                           const iBase_EntityHandle entity_handle, const iMeshP_Part part_id,
                           iBase_EntityHandle* copy_entity_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int ids[MAX_SHARING_PROCS], num_ids;
    EntityHandle handles[MAX_SHARING_PROCS];
    ErrorCode rval = pcomm->get_sharing_parts( itaps_cast< EntityHandle >( entity_handle ), ids, num_ids, handles );CHKERR( rval, "ParallelComm::get_sharing_parts failed" );
    int idx = std::find( ids, ids + num_ids, part_id ) - ids;
    if( idx == num_ids ) RETURN( iBase_FAILURE );

    *copy_entity_handle = itaps_cast< iBase_EntityHandle >( handles[idx] );
    RETURN( iBase_SUCCESS );
}

void iMeshP_getOwnerCopy( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                          const iBase_EntityHandle entity_handle, iMeshP_Part* owner_part_id,
                          iBase_EntityHandle* owner_entity_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    if( !pcomm ) ERROR( iBase_FAILURE, "No PComm" );

    int id;
    EntityHandle h;
    ErrorCode rval = pcomm->get_owning_part( itaps_cast< EntityHandle >( entity_handle ), id, &h );CHKERR( rval, "Failed to get owner" );
    *owner_part_id       = id;
    *owner_entity_handle = itaps_cast< iBase_EntityHandle >( h );
    RETURN( iBase_SUCCESS );
}

void iMeshP_waitForAnyRequest( iMesh_Instance instance, iMeshP_PartitionHandle, iMeshP_RequestHandle*, int, int*,
                               int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_waitForAllRequests( iMesh_Instance instance, iMeshP_PartitionHandle, iMeshP_RequestHandle*, int, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_waitForRequestEnt( iMesh_Instance instance, const iMeshP_PartitionHandle, iMeshP_RequestHandle,
                               iBase_EntityHandle**, int*, int*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_testRequest( iMesh_Instance instance, const iMeshP_PartitionHandle, iMeshP_RequestHandle, int*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_pollForRequests( iMesh_Instance instance, iMeshP_PartitionHandle, iMeshP_RequestHandle**, int*, int*,
                             int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_exchEntArrToPartsAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                                  const iBase_EntityHandle* entity_handles, const int entity_handles_size,
                                  const iMeshP_Part* target_part_ids, int command_code, int update_ghost,
                                  iMeshP_RequestHandle* /*request*/, int* err )
{
    if( command_code == 1 )
    {
        std::cerr << "Entity migration option is not supported." << std::endl;
        RETURN( iBase_NOT_SUPPORTED );
    }

    if( update_ghost == 1 )
    {
        std::cerr << "Gost update option is not supported." << std::endl;
        RETURN( iBase_NOT_SUPPORTED );
    }

    // make exchange entity and processor list
    ErrorCode rval;
    ParallelComm* pcomm = PCOMM;
    std::vector< unsigned int > exchange_procs;
    std::vector< Range* > exchange_ents;

    for( int i = 0; i < entity_handles_size; i++ )
    {
        int ind = -1;
        // iMeshP_Part target_p = target_part_ids[i];
        int target_p;
        rval = pcomm->get_part_owner( target_part_ids[i], target_p );CHKERR( rval, "ParallelComm::get_part_owner failed" );

        std::vector< unsigned int >::iterator vit = std::find( exchange_procs.begin(), exchange_procs.end(), target_p );
        if( vit == exchange_procs.end() )
        {
            ind = exchange_procs.size();
            exchange_procs.push_back( target_p );
            exchange_ents.push_back( new Range );
        }
        else
            ind = vit - exchange_procs.begin();

        exchange_ents[ind]->insert( itaps_cast< EntityHandle >( entity_handles[i] ) );
    }

    std::vector< MPI_Request > recv_ent_reqs, recv_remoteh_reqs;
    rval = pcomm->exchange_owned_meshs( exchange_procs, exchange_ents, recv_ent_reqs, recv_remoteh_reqs, true );CHKERR( rval, "ParallelComm::exchange_owned_meshs failed" );

    // delete exchange list
    std::vector< Range* >::iterator vit;
    for( vit = exchange_ents.begin(); vit != exchange_ents.end(); ++vit )
        delete( *vit );

    RETURN( iBase_SUCCESS );
}

void iMeshP_migrateEntity( iMesh_Instance instance, const iMeshP_PartitionHandle, const iMeshP_PartHandle,
                           const iBase_EntityHandle, iMeshP_RequestHandle*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_updateVtxCoords( iMesh_Instance instance, const iMeshP_PartitionHandle, const iBase_EntityHandle, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_replaceOnPartBdry( iMesh_Instance instance, const iMeshP_PartitionHandle, const iBase_EntityHandle*,
                               const int, const iBase_EntityHandle*, const int, const int*, const int, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_addGhostOf( iMesh_Instance instance, const iMeshP_PartitionHandle, const iMeshP_Part,
                        const iBase_EntityHandle, iMeshP_RequestHandle*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_rmvGhostOf( iMesh_Instance instance, const iMeshP_PartitionHandle, const iMeshP_Part,
                        const iBase_EntityHandle, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_syncMeshAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle, int* err )
{
    ParallelComm* pcomm = PCOMM;
    ErrorCode rval      = pcomm->resolve_shared_ents( itaps_cast< EntityHandle >( partition_handle ), -1, -1 );CHKERR( rval, "update failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_pushTags( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                      iBase_TagHandle source_tag, iBase_TagHandle dest_tag, int entity_type, int entity_topo, int* err )
{
    ParallelComm* pcomm = PCOMM;
    DimensionPair types;
    if( entity_topo != iMesh_ALL_TOPOLOGIES )
        types.first = types.second = mb_topology_table[entity_topo];
    else if( entity_type != iBase_ALL_TYPES )
        // types = CN::TypeDimensionMap[entity_type];
        types = CN::getDimPair( entity_type );
    else
    {
        types.first  = MBVERTEX;
        types.second = MBENTITYSET;
        --types.second;
    }

    std::vector< Tag > src_tags( 1, itaps_cast< Tag >( source_tag ) );
    std::vector< Tag > dst_tags( 1, itaps_cast< Tag >( dest_tag ) );

    ErrorCode rval;
    Range entities;
    for( EntityType t = types.first; t <= types.second; ++t )
    {
        rval = MOABI->get_entities_by_type_and_tag( 0, t, &src_tags[0], 0, 1, entities, Interface::UNION );CHKERR( rval, "error getting entities to push" );
    }

    rval = pcomm->exchange_tags( src_tags, dst_tags, entities );CHKERR( rval, "tag data communication failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_pushTagsEnt( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                         iBase_TagHandle source_tag, iBase_TagHandle dest_tag, const iBase_EntityHandle* entities,
                         int entities_size, int* err )
{

    Range range;
    const EntityHandle* ents = itaps_cast< const EntityHandle* >( entities );
    std::copy( ents, ents + entities_size, range_inserter( range ) );

    std::vector< Tag > src_tags( 1, itaps_cast< Tag >( source_tag ) );
    std::vector< Tag > dst_tags( 1, itaps_cast< Tag >( dest_tag ) );
    ParallelComm* pcomm = PCOMM;
    ErrorCode rval      = pcomm->exchange_tags( src_tags, dst_tags, range );CHKERR( rval, "tag data communication failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_iPushTags( iMesh_Instance instance, const iMeshP_PartitionHandle, iBase_TagHandle, iBase_TagHandle, int,
                       int, iMeshP_RequestHandle*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_iPushTagsEnt( iMesh_Instance instance, const iMeshP_PartitionHandle, iBase_TagHandle, iBase_TagHandle,
                          const iBase_EntityHandle*, int, iMeshP_RequestHandle*, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_createGhostEntsAll( iMesh_Instance instance, iMeshP_PartitionHandle partition_handle, int ghost_dim,
                                int bridge_dim, int num_layers, int include_copies, int* err )
{
    if( include_copies )
    {
        FIXME;
        RETURN( iBase_NOT_SUPPORTED );
    }

    ParallelComm* pcomm = PCOMM;
    ErrorCode rval;
    if( iBase_ALL_TYPES == ghost_dim ) ghost_dim = -1;
    rval = pcomm->exchange_ghost_cells( ghost_dim, bridge_dim, num_layers, 0, true );CHKERR( rval, "ghost exchange failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_deleteGhostEntsAll( iMesh_Instance instance, iMeshP_PartitionHandle, int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

void iMeshP_ghostEntInfo( const iMesh_Instance instance, const iMeshP_PartitionHandle, int*, int*, int**, int**, int**,
                          int* err )
{
    FIXME;
    RETURN( iBase_NOT_SUPPORTED );
}

// append the specified option if it isn't already there; returns whether this
// function actually appended it or not
static bool append_option( std::string& opt, const char* option, const char* default_value = 0 )
{
    std::string::size_type i;

    const char sep = ' ';

    if( strchr( option, sep ) || ( default_value && strchr( default_value, sep ) ) )
    {
        // options can't have a separator in them; XXX work around this
        return iBase_INVALID_ARGUMENT;
    }

    // search for the required option
    std::string search( &sep, 1 );
    search += option;
    const std::string::size_type sl = search.length();
    i                               = opt.find( search );
    while( i != std::string::npos )
    {
        std::string::size_type end = i + sl;
        if( end == opt.size() || opt[end] == sep || opt[end] == '=' ) break;
        i = end;
    }

    // if string already contained the option, just return
    if( i != std::string::npos ) return false;

    opt += search;
    if( default_value )
    {
        opt += "=";
        opt += default_value;
    }

    return true;
}

void iMeshP_loadAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition,
                     const iBase_EntitySetHandle entity_set_handle, const char* name, const char* options, int* err,
                     int name_len, int options_len )
{
    ErrorCode rval;

    // create partition set if user didn't give us one.
    EntityHandle partitioning;
    if( partition ) { partitioning = itaps_cast< EntityHandle >( partition ); }
    else
    {
        rval = MOABI->create_meshset( MESHSET_SET, partitioning );CHKERR( rval, "failed to create meshset" );
    }

    // get ParallelComm for partition
    MPI_Comm default_comm = MPI_COMM_WORLD;
    ParallelComm* pcomm   = ParallelComm::get_pcomm( MOABI, partitioning, &default_comm );
    if( !pcomm ) { RETURN( iBase_FAILURE ); }

    // add necessary values to options string
    std::string opt = std::string( options, options + options_len );

    if( append_option( opt, "moab:PARALLEL" ) )
    {
        // only append these other ones if the parallel option wasn't there originally
        append_option( opt, "moab:PARTITION_DISTRIBUTE" );
        append_option( opt, "moab:PARALLEL_RESOLVE_SHARED_ENTS" );
        std::ostringstream id;
        id << pcomm->get_id();
        append_option( opt, "moab:PCOMM", id.str().c_str() );
    }

    // load the file
    iMesh_load( instance, entity_set_handle, name, opt.c_str(), err, name_len, opt.length() );
    if( *err ) return;

    rval = pcomm->collective_sync_partition();CHKERR( rval, "collective sync failed" );
    RETURN( iBase_SUCCESS );
}

void iMeshP_saveAll( iMesh_Instance instance, const iMeshP_PartitionHandle partition,
                     const iBase_EntitySetHandle entity_set_handle, const char* name, const char* options, int* err,
                     const int name_len, int options_len )
{
    EntityHandle set;
    set = entity_set_handle ? itaps_cast< EntityHandle >( entity_set_handle ) : itaps_cast< EntityHandle >( partition );
    iMesh_save( instance, itaps_cast< iBase_EntitySetHandle >( set ), name, options, err, name_len, options_len );
}

//  Map from processes to parts:
//  Given a partition handle and a process rank,
//  return the part handles owned by the process.
//  COMMUNICATION:  None++.
void iMeshP_getPartsOnRank( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                            /*in*/ const int /*rank*/,
                            /*inout*/ iMeshP_PartHandle** part_handles,
                            /*inout*/ int* part_handles_allocated,
                            /*out*/ int* part_handles_size, int* err )
{
    EntityHandle p   = itaps_cast< EntityHandle >( partition_handle );
    ParallelComm* pc = ParallelComm::get_pcomm( MOABI, p );
    if( !pc ) RETURN( iBase_ERROR_MAP[MB_FAILURE] );

    Range part_sets;

    ALLOC_CHECK_ARRAY_NOFAIL( part_handles, pc->partition_sets().size() );
    Range::iterator rit;
    int i;
    for( i = 0, rit = pc->partition_sets().begin(); rit != pc->partition_sets().end(); ++rit, i++ )
        ( *part_handles )[i] = itaps_cast< iMeshP_PartHandle >( *rit );

    RETURN( iBase_SUCCESS );
}

void iMeshP_getPartsArrOnRank( iMesh_Instance instance, const iMeshP_PartitionHandle partition_handle,
                               /*in*/ const int* rank,
                               /*in*/ const int rank_size,
                               /*inout*/ iMeshP_PartHandle** part_handles,
                               /*inout*/ int* part_handles_allocated,
                               /*out*/ int* part_handles_size, int* err )
{
    EntityHandle p   = itaps_cast< EntityHandle >( partition_handle );
    ParallelComm* pc = ParallelComm::get_pcomm( MOABI, p );
    if( !pc ) RETURN( iBase_ERROR_MAP[MB_FAILURE] );

    if( rank[0] != (int)pc->proc_config().proc_rank() || rank_size > 1 )
    { RETURN( iBase_ERROR_MAP[MB_NOT_IMPLEMENTED] ); }

    iMeshP_getPartsOnRank( instance, partition_handle, rank[0], part_handles, part_handles_allocated, part_handles_size,
                           err );
}

/** \brief Assign a global id space to entities
 * Assign a global id space to entities and vertices, and optionally intermediate-dimension entities
 *
 *  COMMUNICATION:  Collective.
 */
void iMeshP_assignGlobalIds( iMesh_Instance instance, const iMeshP_PartitionHandle partition,
                             const iBase_EntitySetHandle this_set, const int dimension, const int start_id,
                             const int largest_dim_only, const int parallel, const int owned_only, int* err )
{
    ErrorCode rval;

    // get partition set
    EntityHandle partitionset = itaps_cast< EntityHandle >( partition );
    if( !partitionset )
    {
        rval = MB_FAILURE;CHKERR( rval, "failed to get partition set" );
    }

    EntityHandle this_mb_set = itaps_cast< EntityHandle >( this_set );

    // get ParallelComm for partition
    MPI_Comm default_comm;
    ParallelComm* pcomm = ParallelComm::get_pcomm( MOABI, partitionset, &default_comm );
    if( !pcomm ) { RETURN( iBase_FAILURE ); }

    rval = pcomm->assign_global_ids( this_mb_set, dimension, start_id, largest_dim_only, parallel, owned_only );

    RETURN( rval );
}

void iMeshP_getCommunicator( iMesh_Instance instance, int* fcomm, MPI_Comm* ccomm, int* err )
{
    *ccomm = MPI_Comm_f2c( *fcomm );
    RETURN( iBase_SUCCESS );
}

#ifdef __cplusplus
}  // extern "C"
#endif