MOAB_iMeshP_unit_tests.cpp

Go to the documentation of this file.

#include "iMeshP.h"
#include "moab_mpi.h"
#include <iostream>
#include <algorithm>
#include <vector>
#include <sstream>
#include <cassert>
#include <cmath>
#include <map>
#include <cstring>
#include <cstdio>  // remove()

#if !defined( _MSC_VER ) && !defined( __MINGW32__ )
#include <unistd.h>
#endif

#define STRINGIFY_( X ) #X
#define STRINGIFY( X )  STRINGIFY_( X )
const char* const FILENAME = "iMeshP_test_file";

/**************************************************************************
                              Error Checking
 **************************************************************************/

#define CHKERR                                                                                         \
    do                                                                                                 \
    {                                                                                                  \
        if( ierr )                                                                                     \
        {                                                                                              \
            std::cerr << "Error code  " << ierr << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
            return ierr;                                                                               \
        }                                                                                              \
    } while( false )

#define PCHECK                            \
    do                                    \
    {                                     \
        ierr = is_any_proc_error( ierr ); \
        CHKERR;                           \
    } while( false )

// Use my_rank instead of rank to avoid shadowed declaration
#define ASSERT( A )                                                                                                 \
    do                                                                                                              \
    {                                                                                                               \
        if( is_any_proc_error( !( A ) ) )                                                                           \
        {                                                                                                           \
            int my_rank = 0;                                                                                        \
            MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );                                                              \
            if( 0 == my_rank )                                                                                      \
                std::cerr << "Failed assertion: " #A << std::endl << "  at " __FILE__ ":" << __LINE__ << std::endl; \
            return 1;                                                                                               \
        }                                                                                                           \
    } while( false )

// Test if is_my_error is non-zero on any processor in MPI_COMM_WORLD
int is_any_proc_error( int is_my_error )
{
    int result;
    int err = MPI_Allreduce( &is_my_error, &result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD );
    return err || result;
}

/**************************************************************************
                           Test  Declarations
 **************************************************************************/

class PartMap;

/**\brief Consistency check for parallel load
 *
 * All other tests depend on this one.
 */
int test_load( iMesh_Instance, iMeshP_PartitionHandle prtn, PartMap& map, int comm_size );

/**\brief Test partition query methods
 *
 * Test:
 * - iMeshP_getPartitionComm
 * - iMeshP_getNumPartitions
 * - iMeshP_getPartitions
 */
int test_get_partitions( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test part quyery methods
 *
 * Test:
 * - iMeshP_getNumGlobalParts
 * - iMeshP_getNumLocalParts
 * - iMeshP_getLocalParts
 */
int test_get_parts( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test query by entity type
 *
 * Test:
 * - iMeshP_getNumOfTypeAll
 * - iMeshP_getNumOfType
 * - iMeshP_getEntities
 * -
 */
int test_get_by_type( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test query by entity topology
 *
 * Test:
 * - iMeshP_getNumOfTopoAll
 * - iMeshP_getNumOfTopo
 * - iMeshP_getEntities
 * -
 */
int test_get_by_topo( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test mapping from part id to part handle
 *
 * Test:
 * - iMeshP_getPartIdFromPartHandle
 * - iMeshP_getPartIdsFromPartHandlesArr
 * - iMeshP_getPartHandleFromPartId
 * - iMeshP_getPartHandlesFromPartsIdsArr
 * - iMeshP_getRankOfPart
 * - iMeshP_getRankOfPartArr
 */
int test_part_id_handle( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test get part rank
 *
 * Tests:
 * - iMeshP_getRankOfPart
 * - iMeshP_getRankOfPartArr
 */
int test_part_rank( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test querying of part neighbors
 *
 * Test:
 * - iMeshP_getNumPartNbors
 * - iMeshP_getNumPartNborsArr
 * - iMeshP_getPartNbors
 * - iMeshP_getPartNborsArr
 */
int test_get_neighbors( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test querying of part boundary entities
 *
 * Test:
 * - iMeshP_getNumPartBdryEnts
 * - iMeshP_getPartBdryEnts
 */
int test_get_part_boundary( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test querying of part boundary entities
 *
 * Test:
 * - iMeshP_initPartBdryEntIter
 * - iMeshP_initPartBdryEntArrIter
 */
int test_part_boundary_iter( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test adjacent entity query
 *
 * Test:
 * - iMeshP_getAdjEntities
 */
int test_get_adjacencies( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test entity iterators
 *
 * Test:
 * - iMeshP_initEntIter
 * - iMeshP_initEntArrIter
 */
int test_entity_iterator( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test entity owner queries
 *
 * Test:
 * - iMeshP_getEntOwnerPart
 * - iMeshP_getEntOwnerPartArr
 * - iMeshP_isEntOwner
 * - iMeshP_isEntOwnerArr
 */
int test_entity_owner( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test entity status
 *
 * Test:
 * - iMeshP_getEntStatus
 * - iMeshP_getEntStatusArr
 */
int test_entity_status( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test information about entity copies for interface entities
 *
 * Test:
 * - iMeshP_getNumCopies
 * - iMeshP_getCopyParts
 */
int test_entity_copy_parts( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test information about entity copies for interface entities
 *
 * Test:
 * - iMeshP_getCopies
 * - iMeshP_getCopyOnPart
 * - iMeshP_getOwnerCopy
 */
int test_entity_copies( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test creation of ghost entities
 *
 * Test:
 * - iMeshP_createGhostEntsAll
 */
int test_create_ghost_ents( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

/**\brief Test commuinication of tag data
 *
 * Test:
 * - iMeshP_pushTags
 * - iMeshP_pushTagsEnt
 */
int test_push_tag_data_iface( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );
int test_push_tag_data_ghost( iMesh_Instance, iMeshP_PartitionHandle prtn, const PartMap& );

int test_exchange_ents( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map );

/**************************************************************************
                              Helper Funcions
 **************************************************************************/

class PartMap
{
  public:
    int num_parts() const
    {
        return sortedPartList.size();
    }

    iMeshP_Part part_id_from_local_id( int local_id ) const
    {
        return sortedPartList[idx_from_local_id( local_id )];
    }

    int local_id_from_part_id( iMeshP_Part part ) const
    {
        return partLocalIds[idx_from_part_id( part )];
    }

    int rank_from_part_id( iMeshP_Part part ) const
    {
        return partRanks[idx_from_part_id( part )];
    }

    int rank_from_local_id( int id ) const
    {
        return partRanks[idx_from_local_id( id )];
    }

    int count_from_rank( int rank ) const
    {
        return std::count( partRanks.begin(), partRanks.end(), rank );
    }

    void part_id_from_rank( int rank, std::vector< iMeshP_Part >& parts ) const;

    void local_id_from_rank( int rank, std::vector< int >& ids ) const;

    const std::vector< iMeshP_Part >& get_parts() const
    {
        return sortedPartList;
    }

    const std::vector< int >& get_ranks() const
    {
        return partRanks;
    }

    int build_map( iMesh_Instance imesh, iMeshP_PartitionHandle partition, int num_expected_parts );

    static int part_from_coords( iMesh_Instance imesh, iMeshP_PartHandle part, int& id_out );

  private:
    inline int idx_from_part_id( iMeshP_Part id ) const
    {
        return std::lower_bound( sortedPartList.begin(), sortedPartList.end(), id ) - sortedPartList.begin();
    }
    inline int idx_from_local_id( int id ) const
    {
        return localIdReverseMap[id];
    }

    std::vector< iMeshP_Part > sortedPartList;
    std::vector< int > partRanks;
    std::vector< int > partLocalIds;
    std::vector< int > localIdReverseMap;
};

/**\brief Create mesh for use in parallel tests */
int create_mesh( const char* filename, int num_parts );

int create_mesh_in_memory( int rank, int size, iMesh_Instance imesh, iMeshP_PartitionHandle& prtn, PartMap& map );

/**\brief get unique identifier for each vertex */
int vertex_tag( iMesh_Instance imesh, iBase_EntityHandle vertex, int& tag );

int get_local_parts( iMesh_Instance instance,
                     iMeshP_PartitionHandle prtn,
                     std::vector< iMeshP_PartHandle >& handles,
                     std::vector< iMeshP_Part >* ids = 0 )
{
    iMeshP_PartHandle* arr = 0;
    int ierr, alloc = 0, size;
    iMeshP_getLocalParts( instance, prtn, &arr, &alloc, &size, &ierr );CHKERR;
    handles.resize( size );
    std::copy( arr, arr + size, handles.begin() );
    free( arr );
    if( !ids ) return iBase_SUCCESS;

    ids->resize( size );
    alloc            = size;
    iMeshP_Part* ptr = &( *ids )[0];
    iMeshP_getPartIdsFromPartHandlesArr( instance, prtn, &handles[0], handles.size(), &ptr, &alloc, &size, &ierr );CHKERR;
    assert( size == (int)ids->size() );
    assert( ptr == &( *ids )[0] );
    return iBase_SUCCESS;
}

static int get_entities( iMesh_Instance imesh,
                         iBase_EntitySetHandle set,
                         iBase_EntityType type,
                         iMesh_EntityTopology topo,
                         std::vector< iBase_EntityHandle >& entities )
{
    iBase_EntityHandle* array = 0;
    int junk = 0, size = 0, err;
    iMesh_getEntities( imesh, set, type, topo, &array, &junk, &size, &err );
    if( !err )
    {
        entities.clear();
        entities.resize( size );
        std::copy( array, array + size, entities.begin() );
        free( array );
    }
    return err;
}

static int get_part_quads_and_verts( iMesh_Instance imesh,
                                     iMeshP_PartHandle part,
                                     std::vector< iBase_EntityHandle >& elems,
                                     std::vector< iBase_EntityHandle >& verts )
{
    int ierr = get_entities( imesh, part, iBase_FACE, iMesh_QUADRILATERAL, elems );CHKERR;

    verts.resize( 4 * elems.size() );
    std::vector< int > junk( elems.size() + 1 );
    int junk1 = verts.size(), count, junk2 = junk.size(), junk3;
    iBase_EntityHandle* junk4 = &verts[0];
    int* junk5                = &junk[0];
    iMesh_getEntArrAdj( imesh, &elems[0], elems.size(), iBase_VERTEX, &junk4, &junk1, &count, &junk5, &junk2, &junk3,
                        &ierr );CHKERR;
    assert( junk1 == (int)verts.size() );
    assert( count == (int)( 4 * elems.size() ) );
    assert( junk2 == (int)junk.size() );
    assert( junk4 == &verts[0] );
    assert( junk5 == &junk[0] );
    std::sort( verts.begin(), verts.end() );
    verts.erase( std::unique( verts.begin(), verts.end() ), verts.end() );
    return iBase_SUCCESS;
}

static int get_coords( iMesh_Instance imesh, const iBase_EntityHandle* verts, int num_verts, double* coords )
{
    double* junk1 = coords;
    int junk2     = 3 * num_verts;
    int junk3;
    int ierr;
    iMesh_getVtxArrCoords( imesh, verts, num_verts, iBase_INTERLEAVED, &junk1, &junk2, &junk3, &ierr );
    if( iBase_SUCCESS != ierr ) return ierr;
    assert( junk1 == coords );
    assert( junk2 == 3 * num_verts );
    assert( junk3 == 3 * num_verts );
    return iBase_SUCCESS;
}

/**************************************************************************
                              Main Method
 **************************************************************************/

#define RUN_TEST( A ) run_test( &( A ), #A )

int run_test( int ( *func )( iMesh_Instance, iMeshP_PartitionHandle, const PartMap& ), const char* func_name )
{
    int rank, size, ierr;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );
    iMesh_Instance imesh;
    iMesh_newMesh( 0, &imesh, &ierr, 0 );
    PCHECK;

    iMeshP_PartitionHandle prtn;
    iMeshP_createPartitionAll( imesh, MPI_COMM_WORLD, &prtn, &ierr );
    PCHECK;

    PartMap map;

#ifdef MOAB_HAVE_HDF5
    if( rank == 0 )
    {
        ierr = create_mesh( FILENAME, size );
    }
    MPI_Bcast( &ierr, 1, MPI_INT, 0, MPI_COMM_WORLD );
    if( ierr )
    {
        if( rank == 0 )
        {
            std::cerr << "Failed to create input test file on root processor.  Aborting." << std::endl;
        }
        abort();
    }

    ierr = test_load( imesh, prtn, map, size );
    if( ierr )
    {
        if( rank == 0 )
        {
            std::cerr << "Failed to load input mesh." << std::endl
                      << "Cannot run further tests." << std::endl
                      << "ABORTING" << std::endl;
        }
        abort();
    }
#else
    // so we have MPI and no HDF5; in order to run the test we need to create the
    // model in memory, and then call sync to resolve shared ents, as if it was read
    ierr = create_mesh_in_memory( rank, size, imesh, prtn, map );
    MPI_Bcast( &ierr, 1, MPI_INT, 0, MPI_COMM_WORLD );
    if( ierr )
    {
        if( rank == 0 )
        {
            std::cerr << "Failed to create mesh.  Aborting." << std::endl;
        }
        abort();
    }

#endif
    int result = ( *func )( imesh, prtn, map );
    int is_err = is_any_proc_error( result );
    if( rank == 0 )
    {
        if( is_err )
            std::cout << func_name << " : FAILED!!" << std::endl;
        else
            std::cout << func_name << " : success" << std::endl;
    }

    iMesh_dtor( imesh, &ierr );CHKERR;
    return is_err;
}

int main( int argc, char* argv[] )
{
    MPI_Init( &argc, &argv );
    int size, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    if( argc > 2 && !strcmp( argv[1], "-p" ) )
    {
#if !defined( _MSC_VER ) && !defined( __MINGW32__ )
        std::cout << "Processor " << rank << " of " << size << " with PID " << getpid() << std::endl;
        std::cout.flush();
#endif
        // loop forever on requested processor, giving the user time
        // to attach a debugger.  Once the debugger in attached, user
        // can change 'pause'.  E.g. on gdb do "set var pause = 0"
        if( atoi( argv[2] ) == rank )
        {
            volatile int pause = 1;
            while( pause )
                ;
        }
        MPI_Barrier( MPI_COMM_WORLD );
    }

    int num_errors = 0;
    num_errors += RUN_TEST( test_get_partitions );
    num_errors += RUN_TEST( test_get_parts );
    num_errors += RUN_TEST( test_get_by_type );
    num_errors += RUN_TEST( test_get_by_topo );
    num_errors += RUN_TEST( test_part_id_handle );
    num_errors += RUN_TEST( test_part_rank );
    num_errors += RUN_TEST( test_get_neighbors );
    num_errors += RUN_TEST( test_get_part_boundary );
    num_errors += RUN_TEST( test_part_boundary_iter );
    num_errors += RUN_TEST( test_get_adjacencies );
    num_errors += RUN_TEST( test_entity_iterator );
    num_errors += RUN_TEST( test_entity_owner );
    num_errors += RUN_TEST( test_entity_status );
    num_errors += RUN_TEST( test_entity_copy_parts );
    num_errors += RUN_TEST( test_entity_copies );
    num_errors += RUN_TEST( test_push_tag_data_iface );
    num_errors += RUN_TEST( test_push_tag_data_ghost );
    num_errors += RUN_TEST( test_create_ghost_ents );
    num_errors += RUN_TEST( test_exchange_ents );

    // wait until all procs are done before writing summary data
    std::cout.flush();
    MPI_Barrier( MPI_COMM_WORLD );

#ifdef MOAB_HAVE_HDF5
    // clean up output file
    if( rank == 0 ) remove( FILENAME );
#endif

    if( rank == 0 )
    {
        if( !num_errors )
            std::cout << "All tests passed" << std::endl;
        else
            std::cout << num_errors << " TESTS FAILED!" << std::endl;
    }

    MPI_Finalize();
    return num_errors;
}

// Create a mesh
//
//
// Groups of four quads will be arranged into parts as follows:
// +------+------+------+------+------+-----
// |             |             |
// |             |             |
// +    Part 0   +    Part 2   +    Part 4
// |             |             |
// |             |             |
// +------+------+------+------+------+-----
// |             |             |
// |             |             |
// +    Part 1   +    Part 3   +    Part 5
// |             |             |
// |             |             |
// +------+------+------+------+------+-----
//
// Vertices will be enumerated as follows:
// 1------6-----11-----16-----21-----26-----
// |             |             |
// |             |             |
// 2      7     12     17     22     27
// |             |             |
// |             |             |
// 3------8-----13-----18-----23-----28-----
// |             |             |
// |             |             |
// 4      9     14     19     24     29
// |             |             |
// |             |             |
// 5-----10-----15-----20-----25-----30-----
//
// Element IDs will be [4*rank+1,4*rank+5]
template < int size >
struct EHARR
{
    iBase_EntityHandle h[size];
    iBase_EntityHandle& operator[]( int i )
    {
        return h[i];
    }
    operator iBase_EntityHandle*()
    {
        return h;
    }
};
int create_mesh( const char* filename, int num_parts )
{
    const char* tagname = "GLOBAL_ID";
    int ierr;

    iMesh_Instance imesh;
    iMesh_newMesh( 0, &imesh, &ierr, 0 );CHKERR;

    const int num_full_cols  = 2 * ( num_parts / 2 );
    const int need_half_cols = num_parts % 2;
    const int num_cols       = num_full_cols + 2 * need_half_cols;
    const int num_vtx        = 5 + 5 * num_cols - 4 * ( num_parts % 2 );
    std::vector< EHARR< 5 > > vertices( num_cols + 1 );
    std::vector< EHARR< 4 > > elements( num_cols );
    std::vector< int > vertex_ids( num_vtx );
    std::vector< iBase_EntityHandle > vertex_list( num_vtx );
    for( int i = 0; i < num_vtx; ++i )
        vertex_ids[i] = i + 1;

    // create vertices
    int vl_pos = 0;
    for( int i = 0; i <= num_cols; ++i )
    {
        double coords[15]       = { static_cast< double >( i ), 0, 0, static_cast< double >( i ), 1, 0,
                              static_cast< double >( i ), 2, 0, static_cast< double >( i ), 3, 0,
                              static_cast< double >( i ), 4, 0 };
        iBase_EntityHandle* ptr = vertices[i];
        const int n             = ( num_full_cols == num_cols || i <= num_full_cols ) ? 5 : 3;
        int junk1 = n, junk2 = n;
        iMesh_createVtxArr( imesh, n, iBase_INTERLEAVED, coords, 3 * n, &ptr, &junk1, &junk2, &ierr );CHKERR;
        assert( ptr == vertices[i] );
        assert( junk1 == n );
        assert( junk2 == n );
        for( int j = 0; j < n; ++j )
            vertex_list[vl_pos++] = vertices[i][j];
    }

    // create elements
    for( int i = 0; i < num_cols; ++i )
    {
        iBase_EntityHandle conn[16];
        for( int j = 0; j < 4; ++j )
        {
            conn[4 * j]     = vertices[i][j];
            conn[4 * j + 1] = vertices[i][j + 1];
            conn[4 * j + 2] = vertices[i + 1][j + 1];
            conn[4 * j + 3] = vertices[i + 1][j];
        }
        iBase_EntityHandle* ptr = elements[i];
        const int n             = ( i < num_full_cols ) ? 4 : 2;
        int junk1 = n, junk2 = n, junk3 = n, junk4 = n;
        int stat[4];
        int* ptr2 = stat;
        iMesh_createEntArr( imesh, iMesh_QUADRILATERAL, conn, 4 * n, &ptr, &junk1, &junk2, &ptr2, &junk3, &junk4,
                            &ierr );CHKERR;
        assert( ptr == elements[i] );
        assert( junk1 == n );
        assert( junk2 == n );
        assert( ptr2 == stat );
        assert( junk3 == n );
        assert( junk4 == n );
    }

    // create partition
    iMeshP_PartitionHandle partition;
    iMeshP_createPartitionAll( imesh, MPI_COMM_SELF, &partition, &ierr );CHKERR;
    for( int i = 0; i < num_parts; ++i )
    {
        iMeshP_PartHandle part;
        iMeshP_createPart( imesh, partition, &part, &ierr );CHKERR;
        iBase_EntityHandle quads[] = { elements[2 * ( i / 2 )][2 * ( i % 2 )],
                                       elements[2 * ( i / 2 ) + 1][2 * ( i % 2 )],
                                       elements[2 * ( i / 2 )][2 * ( i % 2 ) + 1],
                                       elements[2 * ( i / 2 ) + 1][2 * ( i % 2 ) + 1] };
        iMesh_addEntArrToSet( imesh, quads, 4, part, &ierr );CHKERR;
    }

    // assign global ids to vertices
    iBase_TagHandle id_tag = 0;
    iMesh_getTagHandle( imesh, tagname, &id_tag, &ierr, strlen( tagname ) );
    if( iBase_SUCCESS == ierr )
    {
        int tag_size, tag_type;
        iMesh_getTagSizeValues( imesh, id_tag, &tag_size, &ierr );CHKERR;
        if( tag_size != 1 ) return iBase_TAG_ALREADY_EXISTS;
        iMesh_getTagType( imesh, id_tag, &tag_type, &ierr );CHKERR;
        if( tag_type != iBase_INTEGER ) return iBase_TAG_ALREADY_EXISTS;
    }
    else
    {
        iMesh_createTag( imesh, tagname, 1, iBase_INTEGER, &id_tag, &ierr, strlen( tagname ) );CHKERR;
    }
    iMesh_setIntArrData( imesh, &vertex_list[0], num_vtx, id_tag, &vertex_ids[0], num_vtx, &ierr );CHKERR;

    // write file
    iBase_EntitySetHandle root_set;
    iMesh_getRootSet( imesh, &root_set, &ierr );
    iMeshP_saveAll( imesh, partition, root_set, filename, 0, &ierr, strlen( filename ), 0 );CHKERR;

    iMesh_dtor( imesh, &ierr );CHKERR;

    return 0;
}
int create_mesh_in_memory( int rank,
                           int num_parts,
                           iMesh_Instance imesh,
                           iMeshP_PartitionHandle& partition,
                           PartMap& map )
{
    const char* tagname = "GLOBAL_ID";
    int ierr;

    const int num_cols = 2;
    const int num_vtx  = 9;
    // we are on the top or botton row
    int bottom = rank % 2;  // 0  2
                            // 1  3
    std::vector< EHARR< 3 > > vertices( 3 );
    std::vector< EHARR< 2 > > elements( 2 );  // 4 elements per process
    std::vector< int > vertex_ids( num_vtx );
    std::vector< iBase_EntityHandle > vertex_list( num_vtx );
    int start = 1 + 2 * bottom + 10 * ( rank / 2 );

    for( int i = 0; i < 3; ++i )
        for( int j = 0; j < 3; ++j )
        {
            vertex_ids[i + 3 * j] = start + i + 5 * j;
        }

    // create vertices
    int vl_pos = 0;
    int startI = 2 * ( rank / 2 );  // so it will be 0, 0, 2, 2, ...)
    for( int i = 0; i <= 2; ++i )
    {
        double coords[9] = {
            static_cast< double >( i + startI ), 2. * bottom,     0,
            static_cast< double >( i + startI ), 1 + 2. * bottom, 0,
            static_cast< double >( i + startI ), 2 + 2. * bottom, 0,
        };
        iBase_EntityHandle* ptr = vertices[i];
        const int n             = 3;
        int junk1 = n, junk2 = n;
        iMesh_createVtxArr( imesh, n, iBase_INTERLEAVED, coords, 3 * n, &ptr, &junk1, &junk2, &ierr );CHKERR;
        assert( ptr == vertices[i] );
        assert( junk1 == n );
        assert( junk2 == n );
        for( int j = 0; j < n; ++j )
            vertex_list[vl_pos++] = vertices[i][j];
    }

    // create elements
    for( int i = 0; i < num_cols; ++i )
    {
        iBase_EntityHandle conn[8];
        for( int j = 0; j < 2; ++j )
        {
            conn[4 * j]     = vertices[i][j];
            conn[4 * j + 1] = vertices[i][j + 1];
            conn[4 * j + 2] = vertices[i + 1][j + 1];
            conn[4 * j + 3] = vertices[i + 1][j];
        }
        iBase_EntityHandle* ptr = elements[i];
        const int n             = 2;
        int junk1 = n, junk2 = n, junk3 = n, junk4 = n;
        int stat[4];
        int* ptr2 = stat;
        iMesh_createEntArr( imesh, iMesh_QUADRILATERAL, conn, 4 * n, &ptr, &junk1, &junk2, &ptr2, &junk3, &junk4,
                            &ierr );CHKERR;
        assert( ptr == elements[i] );
        assert( junk1 == n );
        assert( junk2 == n );
        assert( ptr2 == stat );
        assert( junk3 == n );
        assert( junk4 == n );
    }

    // create partition
    iMeshP_createPartitionAll( imesh, MPI_COMM_WORLD, &partition, &ierr );CHKERR;

    iMeshP_PartHandle part;
    iMeshP_createPart( imesh, partition, &part, &ierr );CHKERR;
    iBase_EntityHandle quads[] = { elements[0][0], elements[0][1], elements[1][0], elements[1][1] };
    iMesh_addEntArrToSet( imesh, quads, 4, part, &ierr );CHKERR;

    // assign global ids to vertices
    iBase_TagHandle id_tag = 0;
    iMesh_getTagHandle( imesh, tagname, &id_tag, &ierr, strlen( tagname ) );
    if( iBase_SUCCESS == ierr )
    {
        int tag_size, tag_type;
        iMesh_getTagSizeValues( imesh, id_tag, &tag_size, &ierr );CHKERR;
        if( tag_size != 1 ) return iBase_TAG_ALREADY_EXISTS;
        iMesh_getTagType( imesh, id_tag, &tag_type, &ierr );CHKERR;
        if( tag_type != iBase_INTEGER ) return iBase_TAG_ALREADY_EXISTS;
    }
    else
    {
        iMesh_createTag( imesh, tagname, 1, iBase_INTEGER, &id_tag, &ierr, strlen( tagname ) );CHKERR;
    }
    iMesh_setIntArrData( imesh, &vertex_list[0], num_vtx, id_tag, &vertex_ids[0], num_vtx, &ierr );CHKERR;

    // some mesh sync
    iMeshP_syncPartitionAll( imesh, partition, &ierr );CHKERR;
    iMeshP_syncMeshAll( imesh, partition, &ierr );CHKERR;

    ierr = map.build_map( imesh, partition, num_parts );CHKERR;
    return 0;
}
// generate unique for each vertex from coordinates.
// Assume integer coordinate values with x in [0,inf] and y in [0,4]
// as generated by create_mean(..).
int vertex_tag( iMesh_Instance imesh, iBase_EntityHandle vertex, int& tag )
{
    int ierr;
    double x, y, z;
    iMesh_getVtxCoord( imesh, vertex, &x, &y, &z, &ierr );CHKERR;

    int xc = (int)round( x );
    int yc = (int)round( y );
    tag    = 5 * xc + yc + 1;
    return ierr;
}

/**************************************************************************
                           Test  Implementations
 **************************************************************************/

int test_load( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, PartMap& map, int proc_size )
{
    int ierr;

    iBase_EntitySetHandle root_set;
    iMesh_getRootSet( imesh, &root_set, &ierr );
    const char* opt = "moab:PARTITION=PARALLEL_PARTITION";
    iMeshP_loadAll( imesh, prtn, root_set, FILENAME, opt, &ierr, strlen( FILENAME ), strlen( opt ) );
    PCHECK;

    ierr = map.build_map( imesh, prtn, proc_size );CHKERR;
    return iBase_SUCCESS;
}

/**\brief Test partition query methods
 *
 * Test:
 * - iMeshP_getPartitionComm
 * - iMeshP_getNumPartitions
 * - iMeshP_getPartitions
 */
int test_get_partitions( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& )
{
    int ierr;

    // test iMeshP_getPartitionCom
    MPI_Comm comm = MPI_COMM_SELF;
    iMeshP_getPartitionComm( imesh, prtn, &comm, &ierr );
    PCHECK;
    ASSERT( comm == MPI_COMM_WORLD );

    // test iMeshP_getPartitions
    iMeshP_PartitionHandle* array = 0;
    int alloc = 0, size = -1;
    iMeshP_getPartitions( imesh, &array, &alloc, &size, &ierr );
    PCHECK;
    ASSERT( array != 0 );
    ASSERT( alloc == size );
    ASSERT( size > 0 );
    int idx = std::find( array, array + size, prtn ) - array;
    free( array );
    ASSERT( idx < size );

    // test iMesP_getNumPartitions
    int size2 = -1;
    iMeshP_getNumPartitions( imesh, &size2, &ierr );
    PCHECK;
    ASSERT( size2 == size );
    return 0;
}

/**\brief Test part quyery methods
 *
 * Test:
 * - iMeshP_getNumGlobalParts
 * - iMeshP_getNumLocalParts
 * - iMeshP_getLocalParts
 */
int test_get_parts( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int size, rank, ierr;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    int num_part_g;
    iMeshP_getNumGlobalParts( imesh, prtn, &num_part_g, &ierr );
    PCHECK;
    ASSERT( num_part_g == map.num_parts() );

    int num_part_l;
    iMeshP_getNumLocalParts( imesh, prtn, &num_part_l, &ierr );
    PCHECK;
    ASSERT( num_part_l == map.count_from_rank( rank ) );

    std::vector< iMeshP_PartHandle > parts( num_part_l );
    iMeshP_PartHandle* ptr = &parts[0];
    int junk1 = num_part_l, count = -1;
    iMeshP_getLocalParts( imesh, prtn, &ptr, &junk1, &count, &ierr );
    PCHECK;
    assert( ptr == &parts[0] );
    assert( junk1 == num_part_l );
    ASSERT( count == num_part_l );

    return iBase_SUCCESS;
}

static int test_get_by_type_topo_all( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, bool test_type, int num_parts )
{
    // calculate number of quads and vertices in entire mesh
    // from number of parts (see create_mesh(..) function.)
    const int expected_global_quad_count = 4 * num_parts;
    const int num_col                    = 2 * ( num_parts / 2 + num_parts % 2 );
    const int expected_global_vtx_count  = num_parts == 1 ? 9 : num_parts % 2 ? 1 + 5 * num_col : 5 + 5 * num_col;

    // test getNumOf*All for root set
    int ierr, count;
    iBase_EntitySetHandle root;
    iMesh_getRootSet( imesh, &root, &ierr );
    if( test_type )
        iMeshP_getNumOfTypeAll( imesh, prtn, root, iBase_VERTEX, &count, &ierr );
    else
        iMeshP_getNumOfTopoAll( imesh, prtn, root, iMesh_POINT, &count, &ierr );
    PCHECK;
    ASSERT( count == expected_global_vtx_count );
    if( test_type )
        iMeshP_getNumOfTypeAll( imesh, prtn, root, iBase_FACE, &count, &ierr );
    else
        iMeshP_getNumOfTopoAll( imesh, prtn, root, iMesh_QUADRILATERAL, &count, &ierr );
    PCHECK;
    ASSERT( count == expected_global_quad_count );

    // create an entity set containing half of the quads
    std::vector< iBase_EntityHandle > all_quads, half_quads;
    ierr = get_entities( imesh, root, iBase_FACE, iMesh_QUADRILATERAL, all_quads );
    assert( 0 == all_quads.size() % 2 );
    half_quads.resize( all_quads.size() / 2 );
    for( size_t i = 0; i < all_quads.size() / 2; ++i )
        half_quads[i] = all_quads[2 * i];
    iBase_EntitySetHandle set;
    iMesh_createEntSet( imesh, 1, &set, &ierr );CHKERR;
    iMesh_addEntArrToSet( imesh, &half_quads[0], half_quads.size(), set, &ierr );CHKERR;

    // test getNumOf*All with defined set
    if( test_type )
        iMeshP_getNumOfTypeAll( imesh, prtn, set, iBase_VERTEX, &count, &ierr );
    else
        iMeshP_getNumOfTopoAll( imesh, prtn, set, iMesh_POINT, &count, &ierr );
    PCHECK;
    ASSERT( count == 0 );
    if( test_type )
        iMeshP_getNumOfTypeAll( imesh, prtn, set, iBase_FACE, &count, &ierr );
    else
        iMeshP_getNumOfTopoAll( imesh, prtn, set, iMesh_QUADRILATERAL, &count, &ierr );
    PCHECK;
    ASSERT( count == expected_global_quad_count / 2 );

    return 0;
}

static int test_get_by_type_topo_local( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, bool test_type )
{
    int ierr;
    iBase_EntitySetHandle root;
    iMesh_getRootSet( imesh, &root, &ierr );

    // select a single part
    std::vector< iMeshP_PartHandle > parts;
    ierr = get_local_parts( imesh, prtn, parts );CHKERR;
    iMeshP_PartHandle part = parts.front();

    // get the entities contained in the part
    std::vector< iBase_EntityHandle > part_quads, part_all;
    ierr = get_entities( imesh, part, iBase_FACE, iMesh_QUADRILATERAL, part_quads );CHKERR;
    ierr = get_entities( imesh, part, iBase_ALL_TYPES, iMesh_ALL_TOPOLOGIES, part_all );CHKERR;

    // compare local counts (using root set)

    int count;
    if( test_type )
        iMeshP_getNumOfType( imesh, prtn, part, root, iBase_FACE, &count, &ierr );
    else
        iMeshP_getNumOfTopo( imesh, prtn, part, root, iMesh_QUADRILATERAL, &count, &ierr );CHKERR;
    ASSERT( count == (int)part_quads.size() );

    if( test_type )
        iMeshP_getNumOfType( imesh, prtn, part, root, iBase_ALL_TYPES, &count, &ierr );
    else
        iMeshP_getNumOfTopo( imesh, prtn, part, root, iMesh_ALL_TOPOLOGIES, &count, &ierr );CHKERR;
    ASSERT( count == (int)part_all.size() );

    // compare local contents (using root set)

    iBase_EntityHandle* ptr = 0;
    int num_ent, junk1 = 0;
    iMeshP_getEntities( imesh, prtn, part, root, test_type ? iBase_FACE : iBase_ALL_TYPES,
                        test_type ? iMesh_ALL_TOPOLOGIES : iMesh_QUADRILATERAL, &ptr, &junk1, &num_ent, &ierr );CHKERR;
    std::vector< iBase_EntityHandle > act_quads( ptr, ptr + num_ent );
    free( ptr );
    junk1 = num_ent = 0;
    ptr             = 0;
    iMeshP_getEntities( imesh, prtn, part, root, iBase_ALL_TYPES, iMesh_ALL_TOPOLOGIES, &ptr, &junk1, &num_ent, &ierr );CHKERR;
    std::vector< iBase_EntityHandle > act_all( ptr, ptr + num_ent );
    free( ptr );
    std::sort( part_quads.begin(), part_quads.end() );
    std::sort( part_all.begin(), part_all.end() );
    std::sort( act_quads.begin(), act_quads.end() );
    std::sort( act_all.begin(), act_all.end() );
    ASSERT( part_quads == act_quads );
    ASSERT( part_all == act_all );

    // create an entity set containing half of the quads from the part
    std::vector< iBase_EntityHandle > half_quads( part_quads.size() / 2 );
    for( size_t i = 0; i < half_quads.size(); ++i )
        half_quads[i] = part_quads[2 * i];
    iBase_EntitySetHandle set;
    iMesh_createEntSet( imesh, 1, &set, &ierr );CHKERR;
    iMesh_addEntArrToSet( imesh, &half_quads[0], half_quads.size(), set, &ierr );CHKERR;

    // check if there exists any quads not in the part that we
    // can add to the set
    std::vector< iBase_EntityHandle > all_quads, other_quads;
    ierr = get_entities( imesh, root, iBase_FACE, iMesh_QUADRILATERAL, all_quads );CHKERR;
    std::sort( all_quads.begin(), all_quads.end() );
    std::sort( part_quads.begin(), part_quads.end() );
    std::set_difference( all_quads.begin(), all_quads.end(), part_quads.begin(), part_quads.end(),
                         std::back_inserter( other_quads ) );
    iMesh_addEntArrToSet( imesh, &other_quads[0], other_quads.size(), set, &ierr );CHKERR;

    // compare local counts (using non-root set)

    if( test_type )
        iMeshP_getNumOfType( imesh, prtn, part, set, iBase_FACE, &count, &ierr );
    else
        iMeshP_getNumOfTopo( imesh, prtn, part, set, iMesh_QUADRILATERAL, &count, &ierr );CHKERR;
    ASSERT( count == (int)half_quads.size() );

    if( test_type )
        iMeshP_getNumOfType( imesh, prtn, part, set, iBase_VERTEX, &count, &ierr );
    else
        iMeshP_getNumOfTopo( imesh, prtn, part, set, iMesh_POINT, &count, &ierr );CHKERR;
    ASSERT( count == 0 );

    // compare local contents (using non-root set)

    junk1   = 0;
    num_ent = 0;
    ptr     = 0;
    iMeshP_getEntities( imesh, prtn, part, set, test_type ? iBase_FACE : iBase_ALL_TYPES,
                        test_type ? iMesh_ALL_TOPOLOGIES : iMesh_QUADRILATERAL, &ptr, &junk1, &num_ent, &ierr );CHKERR;
    act_quads.resize( num_ent );
    std::copy( ptr, ptr + num_ent, act_quads.begin() );
    free( ptr );
    std::sort( half_quads.begin(), half_quads.end() );
    std::sort( act_quads.begin(), act_quads.end() );
    ASSERT( act_quads == half_quads );

    return iBase_SUCCESS;
}

/**\brief Test query by entity type
 *
 * Test:
 * - iMeshP_getNumOfTypeAll
 * - iMeshP_getNumOfType
 * - iMeshP_getEntities
 * -
 */
int test_get_by_type( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr;
    ierr = test_get_by_type_topo_all( imesh, prtn, true, map.num_parts() );
    PCHECK;
    ierr = test_get_by_type_topo_local( imesh, prtn, true );
    PCHECK;
    return 0;
}

/**\brief Test query by entity topology
 *
 * Test:
 * - iMeshP_getNumOfTopoAll
 * - iMeshP_getNumOfTopo
 * - iMeshP_getEntities
 * -
 */
int test_get_by_topo( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr;
    ierr = test_get_by_type_topo_all( imesh, prtn, false, map.num_parts() );
    PCHECK;
    ierr = test_get_by_type_topo_local( imesh, prtn, false );
    PCHECK;
    return 0;
}

/**\brief Test mapping from part id to part handle
 *
 * Test:
 * - iMeshP_getPartIdFromPartHandle
 * - iMeshP_getPartIdsFromPartHandlesArr
 * - iMeshP_getPartHandleFromPartId
 * - iMeshP_getPartHandlesFromPartsIdsArr
 */
int test_part_id_handle( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    // get local part ids
    int rank, ierr;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    std::vector< iMeshP_Part > ids;
    map.part_id_from_rank( rank, ids );

    // check single-part functions and build list of part handles
    std::vector< iMeshP_PartHandle > handles( ids.size() );
    size_t i;
    for( i = 0; i < ids.size(); ++i )
    {
        iMeshP_getPartHandleFromPartId( imesh, prtn, ids[i], &handles[i], &ierr );CHKERR;
        iMeshP_Part id;
        iMeshP_getPartIdFromPartHandle( imesh, prtn, handles[i], &id, &ierr );CHKERR;
        if( id != ids[i] ) break;
    }
    ASSERT( i == ids.size() );

    // test iMeshP_getPartIdsFromPartHandlesArr
    std::vector< iMeshP_Part > ids2( ids.size() );
    int junk1 = ids.size(), junk2 = 0;
    iMeshP_Part* ptr = &ids2[0];
    iMeshP_getPartIdsFromPartHandlesArr( imesh, prtn, &handles[0], handles.size(), &ptr, &junk1, &junk2, &ierr );
    PCHECK;
    ASSERT( ptr == &ids2[0] );
    ASSERT( junk2 == (int)ids2.size() );
    ASSERT( ids == ids2 );

    // test iMeshP_getPartHandlesFromPartsIdsArr
    std::vector< iMeshP_PartHandle > handles2( handles.size() );
    junk1                   = handles.size();
    junk2                   = 0;
    iMeshP_PartHandle* ptr2 = &handles2[0];
    iMeshP_getPartHandlesFromPartsIdsArr( imesh, prtn, &ids[0], ids.size(), &ptr2, &junk1, &junk2, &ierr );
    PCHECK;
    ASSERT( ptr2 == &handles2[0] );
    ASSERT( junk2 == (int)handles2.size() );
    ASSERT( handles == handles2 );

    return 0;
}

/**\brief Test get part rank
 *
 * Tests:
 * - iMeshP_getRankOfPart
 * - iMeshP_getRankOfPartArr
 */
int test_part_rank( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr = 0, rank;
    std::vector< iMeshP_Part > invalid, failed;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    // test iMeshP_getRankOfPart
    for( size_t i = 0; i < map.get_parts().size(); ++i )
    {
        int pr;
        iMeshP_getRankOfPart( imesh, prtn, map.get_parts()[i], &pr, &ierr );
        if( iBase_SUCCESS != ierr )
            failed.push_back( map.get_parts()[i] );
        else if( pr != map.get_ranks()[i] )
            invalid.push_back( map.get_parts()[i] );
    }
    if( !failed.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getRankOfPart failed for " << failed.size() << " parts."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getRankOfPart was incorrect for " << invalid.size() << " parts."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    PCHECK;

    // test iMeshP_getRankOfPartArr
    std::vector< int > ranks( map.get_parts().size() );
    int junk1 = ranks.size(), junk2, *ptr = &ranks[0];
    iMeshP_getRankOfPartArr( imesh, prtn, &map.get_parts()[0], map.get_parts().size(), &ptr, &junk1, &junk2, &ierr );
    PCHECK;
    assert( ptr == &ranks[0] );
    assert( junk1 == (int)ranks.size() );
    ASSERT( junk2 == (int)ranks.size() );
    for( size_t i = 0; i < map.get_parts().size(); ++i )
    {
        if( ranks[i] != map.get_ranks()[i] ) invalid.push_back( map.get_parts()[i] );
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getRankOfPartArr was incorrect for " << invalid.size()
                  << " parts." << std::endl;
        ierr = iBase_FAILURE;
    }
    PCHECK;

    return 0;
}

// see create_mesh(..)
static void get_part_neighbors( int logical_part_id, int num_parts, int neighbors[5], int& num_neighbors )
{
    num_neighbors = 0;
    if( logical_part_id + 1 < num_parts ) neighbors[num_neighbors++] = logical_part_id + 1;
    if( logical_part_id + 2 < num_parts ) neighbors[num_neighbors++] = logical_part_id + 2;
    if( logical_part_id % 2 )
    {
        neighbors[num_neighbors++] = logical_part_id - 1;
        if( logical_part_id > 2 )
        {
            neighbors[num_neighbors++] = logical_part_id - 3;
            neighbors[num_neighbors++] = logical_part_id - 2;
        }
    }
    else
    {
        if( logical_part_id + 3 < num_parts ) neighbors[num_neighbors++] = logical_part_id + 3;
        if( logical_part_id > 1 )
        {
            neighbors[num_neighbors++] = logical_part_id - 1;
            neighbors[num_neighbors++] = logical_part_id - 2;
        }
    }
}

/**\brief Test querying of part neighbors
 *
 * Test:
 * - iMeshP_getNumPartNbors
 * - iMeshP_getNumPartNborsArr
 * - iMeshP_getPartNbors
 * - iMeshP_getPartNborsArr
 */
int test_get_neighbors( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    std::vector< iMeshP_Part > local_parts;
    map.part_id_from_rank( rank, local_parts );

    // get handles for local parts
    std::vector< iMeshP_PartHandle > handles( local_parts.size() );
    iMeshP_PartHandle* ptr = &handles[0];
    int junk1 = handles.size(), junk2 = 0;
    iMeshP_getPartHandlesFromPartsIdsArr( imesh, prtn, &local_parts[0], local_parts.size(), &ptr, &junk1, &junk2,
                                          &ierr );
    PCHECK;
    assert( ptr == &handles[0] );
    assert( junk2 == (int)handles.size() );

    // get logical ids for local parts
    std::vector< int > logical_ids;
    map.local_id_from_rank( rank, logical_ids );

    // get neighbors for each local part
    std::vector< std::vector< iMeshP_Part > > neighbors( logical_ids.size() );
    for( size_t i = 0; i < logical_ids.size(); ++i )
    {
        int logical_neighbors[5], num_neighbors;
        get_part_neighbors( logical_ids[i], map.num_parts(), logical_neighbors, num_neighbors );
        neighbors[i].resize( num_neighbors );
        for( int j = 0; j < num_neighbors; ++j )
            neighbors[i][j] = map.part_id_from_local_id( logical_neighbors[j] );
        std::sort( neighbors[i].begin(), neighbors[i].end() );
    }

    // test iMeshP_getNumPartNbors
    std::vector< iMeshP_Part > invalid, failed;
    for( size_t i = 0; i < local_parts.size(); ++i )
    {
        int count;
        iMeshP_getNumPartNbors( imesh, prtn, handles[i], iBase_VERTEX, &count, &ierr );
        if( ierr )
            failed.push_back( local_parts[i] );
        else if( count != (int)neighbors[i].size() )
            invalid.push_back( local_parts[i] );
    }
    if( !failed.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getNumPartNbors failed for " << failed.size() << " parts."
                  << std::endl;
        ierr = iBase_FAILURE;
        PCHECK;
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getNumPartNbors was incorrect for " << invalid.size()
                  << " parts." << std::endl;
        ierr = iBase_FAILURE;
        PCHECK;
    }

    // test iMeshP_getPartNbors
    ierr = 0;
    for( size_t i = 0; i < local_parts.size(); ++i )
    {
        int count, junk = 0, another_count;
        iMeshP_Part* list = 0;
        iMeshP_getPartNbors( imesh, prtn, handles[i], iBase_VERTEX, &another_count, &list, &junk, &count, &ierr );
        assert( count == another_count );
        if( ierr )
            failed.push_back( local_parts[i] );
        else
        {
            std::sort( list, list + count );
            std::vector< iMeshP_Part > cpy( list, list + count );
            if( cpy != neighbors[i] ) invalid.push_back( local_parts[i] );
            free( list );
        }
    }
    if( !failed.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getPartNbors failed for " << failed.size() << " parts."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getPartNbors was incorrect for " << invalid.size() << " parts."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    PCHECK;

    // test iMeshP_getNumPartNborsArr
    std::vector< int > count_vect( handles.size() );
    int* count_arr = &count_vect[0];
    junk1          = handles.size();
    iMeshP_getNumPartNborsArr( imesh, prtn, &handles[0], handles.size(), iBase_VERTEX, &count_arr, &junk1, &junk2,
                               &ierr );
    PCHECK;
    assert( count_arr == &count_vect[0] );
    assert( junk2 == (int)handles.size() );
    for( size_t i = 0; i < local_parts.size(); ++i )
    {
        if( count_arr[i] != (int)neighbors[i].size() ) invalid.push_back( local_parts[i] );
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getNumPartNborsArr was incorrect for " << invalid.size()
                  << " parts." << std::endl;
        ierr = iBase_FAILURE;
    }
    PCHECK;

    // test iMeshP_getPartNborsArr
    iMeshP_Part* nbor_arr = 0;
    junk1 = handles.size(), junk2 = 0;
    int junk3 = 0, nbor_size;
    iMeshP_getPartNborsArr( imesh, prtn, &handles[0], handles.size(), iBase_VERTEX, &count_arr, &junk1, &junk2,
                            &nbor_arr, &junk3, &nbor_size, &ierr );
    PCHECK;
    assert( count_arr == &count_vect[0] );
    assert( junk2 == (int)handles.size() );
    std::vector< iMeshP_Part > all_nbors( nbor_arr, nbor_arr + nbor_size );
    free( nbor_arr );
    std::vector< iMeshP_Part >::iterator j = all_nbors.begin();
    bool bad_length                        = false;
    for( size_t i = 0; i < local_parts.size(); ++i )
    {
        if( all_nbors.end() - j > count_arr[i] )
        {
            bad_length = true;
            break;
        }
        if( count_arr[i] != (int)neighbors[i].size() )
        {
            invalid.push_back( local_parts[i] );
        }
        else
        {
            std::vector< iMeshP_Part >::iterator e = j + count_arr[i];
            std::sort( j, e );
            if( !std::equal( j, e, neighbors[i].begin() ) ) invalid.push_back( local_parts[i] );
        }
    }
    if( bad_length )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getPartNborsArr had inconsistent result array lengths."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !invalid.empty() )
    {
        std::cerr << "Processor " << rank << ": iMeshP_getPartNborsArr was incorrect for " << invalid.size()
                  << " parts." << std::endl;
        ierr = iBase_FAILURE;
    }
    PCHECK;

    return 0;
}

// Determine the expected vertices on the interface between two parts.
// Returns no vertices for non-adjacient parts and fails if both parts
// are the same.
// See create_mesh(..) for the assumed mesh.
static int interface_verts( iMesh_Instance imesh,
                            iMeshP_PartitionHandle prtn,
                            iMeshP_PartHandle local_part,
                            iMeshP_Part other_part,
                            const PartMap& map,
                            std::vector< iBase_EntityHandle >& vtx_handles )
{
    int ierr, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    iMeshP_Part local_id;
    iMeshP_getPartIdFromPartHandle( imesh, prtn, local_part, &local_id, &ierr );CHKERR;

    const int local_logical = map.local_id_from_part_id( local_id );
    const int other_logical = map.local_id_from_part_id( other_part );

    // get grid of local vertices

    iBase_EntityHandle verts[3][3];
    const double xbase = ( local_id / 2 ) * 2;
    const double ybase = ( local_id % 2 ) * 2;

    // get quads in partition
    iBase_EntityHandle quads[4], *ptr = quads;
    int junk1 = 4, junk2;
    iMesh_getEntities( imesh, local_part, iBase_FACE, iMesh_QUADRILATERAL, &ptr, &junk1, &junk2, &ierr );CHKERR;
    assert( ptr == quads );
    assert( junk1 == 4 );
    assert( junk2 == 4 );

    // get vertices in quads
    iBase_EntityHandle conn[16];
    int offsets[5], *off_ptr = offsets, junk3 = 5, junk4;
    ptr   = conn;
    junk1 = 16;
    iMesh_getEntArrAdj( imesh, quads, 4, iBase_VERTEX, &ptr, &junk1, &junk2, &off_ptr, &junk3, &junk4, &ierr );CHKERR;
    assert( ptr == conn );
    assert( junk1 == 16 );
    assert( junk2 == 16 );
    assert( off_ptr == offsets );
    assert( junk3 == 5 );
    assert( junk4 == 5 );

    // make unique vertex list
    std::sort( conn, conn + 16 );
    const int num_vtx = std::unique( conn, conn + 16 ) - conn;
    assert( 9 == num_vtx );

    // get vertex coords
    std::vector< double > coords( 27 );
    ierr = get_coords( imesh, conn, 9, &coords[0] );CHKERR;

    // use vertex coords to determine logical position
    for( int i = 0; i < num_vtx; ++i )
    {
        int x = (int)round( coords[3 * i] - xbase );
        int y = (int)round( coords[3 * i + 1] - ybase );
        if( x < 0 || x > 2 || y < 0 || y > 2 )
        {
            std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                      << "  Invalid vertex coordinate: (" << coords[3 * i] << ", " << coords[3 * i + 1] << ", "
                      << coords[3 * i + 2] << ")" << std::endl
                      << "  For logical partition " << local_id << std::endl;
            return iBase_FAILURE;
        }
        verts[x][y] = conn[i];
    }

    if( local_logical % 2 )
    {
        switch( other_logical - local_logical )
        {
            case 0:
                return iBase_FAILURE;
            case 1:  // upper right
                vtx_handles.resize( 1 );
                vtx_handles[0] = verts[2][0];
                break;
            case 2:  // right
                vtx_handles.resize( 3 );
                std::copy( verts[2], verts[2] + 3, vtx_handles.begin() );
                break;
            case -1:  // above
                vtx_handles.resize( 3 );
                vtx_handles[0] = verts[0][0];
                vtx_handles[1] = verts[1][0];
                vtx_handles[2] = verts[2][0];
                break;
            case -2:  // left
                vtx_handles.resize( 3 );
                std::copy( verts[0], verts[0] + 3, vtx_handles.begin() );
                break;
            case -3:  // upper left
                vtx_handles.resize( 1 );
                vtx_handles[0] = verts[0][0];
                break;
            default:
                vtx_handles.clear();
                break;
        }
    }
    else
    {
        switch( other_logical - local_logical )
        {
            case 0:
                return iBase_FAILURE;
            case 1:  // below
                vtx_handles.resize( 3 );
                vtx_handles[0] = verts[0][2];
                vtx_handles[1] = verts[1][2];
                vtx_handles[2] = verts[2][2];
                break;
            case 2:  // right
                vtx_handles.resize( 3 );
                std::copy( verts[2], verts[2] + 3, vtx_handles.begin() );
                break;
            case 3:  // lower right
                vtx_handles.resize( 1 );
                vtx_handles[0] = verts[2][2];
                break;
            case -1:  // lower left
                vtx_handles.resize( 1 );
                vtx_handles[0] = verts[0][2];
                break;
            case -2:  // left
                vtx_handles.resize( 3 );
                std::copy( verts[0], verts[0] + 3, vtx_handles.begin() );
                break;
            default:
                vtx_handles.clear();
                break;
        }
    }

    return iBase_SUCCESS;
}

/**\brief Test querying of part boundary entities
 *
 * Test:
 * - iMeshP_getNumPartBdryEnts
 * - iMeshP_getPartBdryEnts
 */
int test_get_part_boundary( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    // get local part handles and part ids, and global part id list
    std::vector< iMeshP_PartHandle > local_handles;
    std::vector< iMeshP_Part > local_ids;
    std::vector< iMeshP_Part > all_parts = map.get_parts();
    std::map< iMeshP_PartHandle, std::vector< iBase_EntityHandle > > part_bdry;
    ierr = get_local_parts( imesh, prtn, local_handles, &local_ids );CHKERR;

    // for each combination of local part with any other part,
    // check for valid function values.
    std::vector< std::pair< iMeshP_Part, iMeshP_Part > > num_failed, num_error, list_failed, list_error, error;
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        iMeshP_PartHandle local_handle = local_handles[i];
        iMeshP_Part local_id           = local_ids[i];
        for( std::vector< iMeshP_Part >::iterator j = all_parts.begin(); j != all_parts.end(); ++j )
        {
            iMeshP_Part other_id = *j;
            if( other_id == local_id ) continue;

            std::pair< iMeshP_Part, iMeshP_Part > part_pair;
            part_pair.first  = local_id;
            part_pair.second = other_id;

            // get expected values
            std::vector< iBase_EntityHandle > shared_verts;
            ierr = interface_verts( imesh, prtn, local_handle, other_id, map, shared_verts );
            if( ierr != iBase_SUCCESS )
            {
                error.push_back( part_pair );
                continue;
            }
            std::sort( shared_verts.begin(), shared_verts.end() );

            // test iMeshP_getNumPartBdryEnts
            int count;
            iMeshP_getNumPartBdryEnts( imesh, prtn, local_handle, iBase_VERTEX, iMesh_POINT, other_id, &count, &ierr );
            if( iBase_SUCCESS != ierr )
                num_error.push_back( part_pair );
            else if( count != (int)shared_verts.size() )
                num_failed.push_back( part_pair );

            // test iMeshP_getPartBdryEnts
            iBase_EntityHandle* ptr = 0;
            int junk                = 0;
            iMeshP_getPartBdryEnts( imesh, prtn, local_handle, iBase_VERTEX, iMesh_POINT, other_id, &ptr, &junk, &count,
                                    &ierr );
            if( iBase_SUCCESS != ierr )
                list_error.push_back( part_pair );
            else
            {
                std::copy( ptr, ptr + count, std::back_inserter( part_bdry[local_handles[i]] ) );
                std::sort( ptr, ptr + count );
                if( (int)shared_verts.size() != count || !std::equal( shared_verts.begin(), shared_verts.end(), ptr ) )
                    list_failed.push_back( part_pair );
                free( ptr );
            }
        }
    }

    if( !error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  Internal error for " << error.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !num_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getNumPartBdryEnts return error for " << num_error.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !list_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getPartBdryEnts return error for " << list_error.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !num_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getNumPartBdryEnts return incorrect results for " << num_failed.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !list_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getPartBdryEnts return incorrect results for " << list_failed.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }

    if( iBase_SUCCESS != ierr ) return ierr;

    // test with iMeshP_ALL_PARTS
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        std::vector< iBase_EntityHandle >& exp_bdry = part_bdry[local_handles[i]];
        std::sort( exp_bdry.begin(), exp_bdry.end() );
        exp_bdry.erase( std::unique( exp_bdry.begin(), exp_bdry.end() ), exp_bdry.end() );
        std::pair< iMeshP_Part, iMeshP_Part > part_pair;
        part_pair.first  = local_ids[i];
        part_pair.second = iMeshP_ALL_PARTS;

        int num = 0;
        iMeshP_getNumPartBdryEnts( imesh, prtn, local_handles[i], iBase_VERTEX, iMesh_POINT, iMeshP_ALL_PARTS, &num,
                                   &ierr );
        if( ierr )
            num_error.push_back( part_pair );
        else if( num != (int)exp_bdry.size() )
            num_failed.push_back( part_pair );

        iBase_EntityHandle* bdry = 0;
        int junk = num = 0;
        iMeshP_getPartBdryEnts( imesh, prtn, local_handles[i], iBase_VERTEX, iMesh_POINT, iMeshP_ALL_PARTS, &bdry,
                                &junk, &num, &ierr );
        if( ierr )
            list_error.push_back( part_pair );
        else
        {
            std::sort( bdry, bdry + num );
            if( num != (int)exp_bdry.size() || !std::equal( bdry, bdry + num, exp_bdry.begin() ) )
                list_failed.push_back( part_pair );
            free( bdry );
        }
    }
    if( !num_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getNumPartBdryEnts return error for " << num_error.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !list_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getPartBdryEnts return error for " << list_error.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !num_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getNumPartBdryEnts return incorrect results for " << num_failed.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !list_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_getPartBdryEnts return incorrect results for " << list_failed.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }

    return ierr;
}

/**\brief Test querying of part boundary entities
 *
 * Test:
 * - iMeshP_initPartBdryEntIter
 * - iMeshP_initPartBdryEntArrIter
 */
int test_part_boundary_iter( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank, has_data;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    // get local part handles and part ids, and global part id list
    std::vector< iMeshP_PartHandle > local_handles;
    std::vector< iMeshP_Part > local_ids;
    std::vector< iMeshP_Part > all_parts = map.get_parts();
    ierr                                 = get_local_parts( imesh, prtn, local_handles, &local_ids );CHKERR;

    std::vector< std::pair< iMeshP_Part, iMeshP_Part > > single_failed, single_error, single_step_error, array_failed,
        array_error, array_step_error;
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        iMeshP_PartHandle local_handle = local_handles[i];
        iMeshP_Part local_id           = local_ids[i];
        for( std::vector< iMeshP_Part >::iterator j = all_parts.begin(); j != all_parts.end(); ++j )
        {
            iMeshP_Part other_id = *j;
            if( other_id == local_id ) continue;

            std::pair< iMeshP_Part, iMeshP_Part > part_pair;
            part_pair.first  = local_id;
            part_pair.second = other_id;

            // get expected values
            std::vector< iBase_EntityHandle > shared_verts;
            ierr = interface_verts( imesh, prtn, local_handle, other_id, map, shared_verts );
            if( ierr != iBase_SUCCESS || 0 == shared_verts.size() ) continue;
            std::sort( shared_verts.begin(), shared_verts.end() );

            // test single entity iterator
            iBase_EntityIterator siter;
            iMeshP_initPartBdryEntIter( imesh, prtn, local_handle, iBase_VERTEX, iMesh_POINT, other_id, &siter, &ierr );
            if( ierr != iBase_SUCCESS )
            {
                single_error.push_back( part_pair );
            }
            else
            {
                std::vector< iBase_EntityHandle > results;
                for( ;; )
                {
                    iBase_EntityHandle handle;
                    iMesh_getNextEntIter( imesh, siter, &handle, &has_data, &ierr );
                    if( ierr != iBase_SUCCESS )
                    {
                        single_step_error.push_back( part_pair );
                        break;
                    }
                    if( !has_data ) break;
                    results.push_back( handle );
                }

                std::sort( results.begin(), results.end() );
                if( results.size() != shared_verts.size() ||
                    !std::equal( results.begin(), results.end(), shared_verts.begin() ) )
                    single_failed.push_back( part_pair );
            }
            iMesh_endEntIter( imesh, siter, &ierr );

            // test array iterator
            iBase_EntityArrIterator aiter;
            iMeshP_initPartBdryEntArrIter( imesh, prtn, local_handle, iBase_VERTEX, iMesh_POINT, shared_verts.size(),
                                           other_id, &aiter, &ierr );
            if( ierr != iBase_SUCCESS )
            {
                array_error.push_back( part_pair );
                iMesh_endEntArrIter( imesh, aiter, &ierr );
                continue;
            }
            iBase_EntityHandle results[5], *ptr = results;
            int junk = 5, count;
            iMesh_getNextEntArrIter( imesh, aiter, &ptr, &junk, &count, &has_data, &ierr );
            if( ierr != iBase_SUCCESS || !has_data )
            {
                array_step_error.push_back( part_pair );
                iMesh_endEntArrIter( imesh, aiter, &ierr );
                continue;
            }
            iMesh_endEntArrIter( imesh, aiter, &ierr );
            assert( count <= 5 );
            assert( ptr == results );
            std::sort( ptr, ptr + count );
            if( count != (int)shared_verts.size() || !std::equal( shared_verts.begin(), shared_verts.end(), results ) )
                array_failed.push_back( part_pair );
        }
    }

    if( !single_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_initPartBdryEntIter return error for " << single_error.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !single_step_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMesh_getNextEntIter return error for " << single_step_error.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !single_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_initPartBdryEntIter iterator iterated over invalid entities for " << single_failed.size()
                  << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }

    if( !array_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_initPartBdryEntArrIter return error for " << array_error.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !array_step_error.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMesh_getNextEntArrIter return error for " << array_step_error.size() << " part pairs."
                  << std::endl;
        ierr = iBase_FAILURE;
    }
    if( !array_failed.empty() )
    {
        std::cerr << "Processor " << rank << ": Error at " __FILE__ ":" << __LINE__ << std::endl
                  << "  iMeshP_initPartBdryEntArrIter iterator iterated over invalid entities for "
                  << array_failed.size() << " part pairs." << std::endl;
        ierr = iBase_FAILURE;
    }

    return ierr;
}

/**\brief Test adjacent entity query
 *
 * Test:
 * - iMeshP_getAdjEntities
 */
int test_get_adjacencies( iMesh_Instance /* imesh */, iMeshP_PartitionHandle /* prtn */, const PartMap& )
{
    return iBase_SUCCESS;
}

/**\brief Test entity iterators
 *
 * Test:
 * - iMeshP_initEntIter
 * - iMeshP_initEntArrIter
 */
int test_entity_iterator( iMesh_Instance /*imesh */, iMeshP_PartitionHandle /*prtn*/, const PartMap& )
{
    return iBase_SUCCESS;
}

/**\brief Test entity owner queries
 *
 * Test:
 * - iMeshP_getEntOwnerPart
 * - iMeshP_getEntOwnerPartArr
 * - iMeshP_isEntOwner
 * - iMeshP_isEntOwnerArr
 */
int test_entity_owner( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& /* map */ )
{
    int ierr, rank, size;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    // get local part handles and part ids
    std::vector< iMeshP_PartHandle > local_handles;
    std::vector< iMeshP_Part > local_ids;
    ierr = get_local_parts( imesh, prtn, local_handles, &local_ids );
    PCHECK;

    // test iMeshP_getEntOwnerPart for quads in each part
    std::vector< iBase_EntityHandle > all_quads;
    std::vector< iMeshP_Part > quad_owners;
    int invalid_count = 0;
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        std::vector< iBase_EntityHandle > quads;
        ierr = get_entities( imesh, local_handles[0], iBase_FACE, iMesh_QUADRILATERAL, quads );
        if( ierr ) break;

        for( size_t j = 0; j < quads.size(); ++j )
        {
            all_quads.push_back( quads[j] );
            quad_owners.push_back( local_ids[i] );
            iMeshP_Part owner;
            iMeshP_getEntOwnerPart( imesh, prtn, quads[j], &owner, &ierr );
            if( iBase_SUCCESS != ierr ) break;

            if( owner != local_ids[i] ) ++invalid_count;
        }
        if( iBase_SUCCESS != ierr ) break;
    }
    PCHECK;
    ASSERT( 0 == invalid_count );

    // test iMeshP_getEntOwnerPartArr for quads in each part
    invalid_count = 0;
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        std::vector< iBase_EntityHandle > quads;
        ierr = get_entities( imesh, local_handles[0], iBase_FACE, iMesh_QUADRILATERAL, quads );
        if( ierr ) break;

        std::vector< iMeshP_Part > owners( quads.size() ), expected( quads.size(), local_ids[i] );
        int junk         = owners.size(), count;
        iMeshP_Part* ptr = &owners[0];
        iMeshP_getEntOwnerPartArr( imesh, prtn, &quads[0], quads.size(), &ptr, &junk, &count, &ierr );
        if( ierr ) break;
        assert( ptr == &owners[0] );
        assert( junk == (int)owners.size() );
        assert( count == (int)quads.size() );
        if( owners != expected ) ++invalid_count;
    }
    PCHECK;
    ASSERT( 0 == invalid_count );

    // get all vertices
    iBase_EntityHandle* vtx_arr = 0;
    int junk1                   = 0, num_vtx;
    int *junk2 = 0, junk3 = 0, junk4;
    iMesh_getEntArrAdj( imesh, &all_quads[0], all_quads.size(), iBase_VERTEX, &vtx_arr, &junk1, &num_vtx, &junk2,
                        &junk3, &junk4, &ierr );
    PCHECK;
    free( junk2 );
    std::sort( vtx_arr, vtx_arr + num_vtx );
    num_vtx = std::unique( vtx_arr, vtx_arr + num_vtx ) - vtx_arr;
    std::vector< iBase_EntityHandle > all_verts( vtx_arr, vtx_arr + num_vtx );
    free( vtx_arr );

    // check consistency between iMeshP_getEntOwnerPart and iMeshP_getEntOwnerPartArr
    // for all vertices
    std::vector< iMeshP_Part > vert_owners( all_verts.size() );
    junk1              = vert_owners.size();
    iMeshP_Part* junk5 = &vert_owners[0];
    iMeshP_getEntOwnerPartArr( imesh, prtn, &all_verts[0], all_verts.size(), &junk5, &junk1, &junk3, &ierr );
    PCHECK;
    assert( junk5 == &vert_owners[0] );
    assert( junk1 == (int)vert_owners.size() );
    assert( junk3 == (int)all_verts.size() );

    invalid_count = 0;
    for( size_t i = 0; i < all_verts.size(); ++i )
    {
        iMeshP_Part owner;
        iMeshP_getEntOwnerPart( imesh, prtn, all_verts[i], &owner, &ierr );
        if( iBase_SUCCESS != ierr || owner != vert_owners[i] ) ++invalid_count;
    }
    ASSERT( 0 == invalid_count );

    // get lists for all entities
    std::vector< iBase_EntityHandle > all_entities( all_verts );
    std::copy( all_quads.begin(), all_quads.end(), std::back_inserter( all_entities ) );
    std::vector< iMeshP_Part > all_owners( vert_owners );
    std::copy( quad_owners.begin(), quad_owners.end(), std::back_inserter( all_owners ) );

    // check consistency of iMeshP_isEntOwner for all entities
    invalid_count = 0;
    ierr          = iBase_SUCCESS;
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        for( size_t j = 0; ierr == iBase_SUCCESS && j < all_entities.size(); ++j )
        {
            int is_owner;
            iMeshP_isEntOwner( imesh, prtn, local_handles[i], all_entities[j], &is_owner, &ierr );
            if( ierr != iBase_SUCCESS ) break;
            if( !is_owner == ( local_ids[i] == all_owners[j] ) ) ++invalid_count;
        }
    }
    PCHECK;
    ASSERT( 0 == invalid_count );

    // check consistency of iMeshP_isEntOwnerArr for all entities
    for( size_t i = 0; i < local_handles.size(); ++i )
    {
        std::vector< int > is_owner_list( all_entities.size() );
        junk1      = is_owner_list.size();
        int* junk6 = &is_owner_list[0];
        iMeshP_isEntOwnerArr( imesh, prtn, local_handles[i], &all_entities[0], all_entities.size(), &junk6, &junk1,
                              &junk3, &ierr );
        if( iBase_SUCCESS != ierr ) break;
        assert( junk6 == &is_owner_list[0] );
        assert( junk1 == (int)is_owner_list.size() );
        assert( junk3 == (int)all_entities.size() );
        invalid_count = 0;
        for( size_t j = 0; j < all_entities.size(); ++j )
        {
            if( !( is_owner_list[j] ) == ( local_ids[0] == all_owners[j] ) ) ++invalid_count;
        }
    }
    PCHECK;
    ASSERT( 0 == invalid_count );

    // check globally consistent owners for all vertices

    // first communicate total number of vertex entries to be sent to root proc
    int local_count = all_verts.size(), global_count = 0;
    ierr = MPI_Reduce( &local_count, &global_count, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD );CHKERR;

    // for each vertex, store { (x << 2) | y, owning part id }
    std::vector< int > vtxdata( 2 * all_verts.size() );
    std::vector< double > coords( 3 * all_verts.size() );
    ierr = get_coords( imesh, &all_verts[0], all_verts.size(), &coords[0] );CHKERR;
    for( size_t i = 0; i < all_verts.size(); ++i )
    {
        int x              = (int)round( coords[3 * i] );
        int y              = (int)round( coords[3 * i + 1] );
        vtxdata[2 * i]     = ( x << 3 ) | y;
        vtxdata[2 * i + 1] = vert_owners[i];
    }

    // collect all data on root procesor
    std::vector< int > all_data( 2 * global_count );
    std::vector< int > displ( size ), counts( size );
    for( int i = 0; i < size; i++ )
    {
        counts[i] = vtxdata.size();
        displ[i]  = i * vtxdata.size();
    }

    // we could have used a simple gather, because all sequences are the same
    ierr = MPI_Gatherv( &vtxdata[0], vtxdata.size(), MPI_INT, &all_data[0], &counts[0], &displ[0], MPI_INT, 0,
                        MPI_COMM_WORLD );CHKERR;

    if( rank == 0 )
    {
        // map from vertex tag to indices into data
        std::multimap< int, int > data_map;
        for( int i = 0; i < global_count; ++i )
        {
            std::pair< int, int > p;
            p.first  = all_data[2 * i];
            p.second = i;
            data_map.insert( p );
        }

        // check consistent data for each vtx
        std::multimap< int, int >::const_iterator a, b;
        for( a = data_map.begin(); a != data_map.end(); a = b )
        {
            for( b = a; b != data_map.end() && a->first == b->first; ++b )
            {
                int idx1 = a->second;
                int idx2 = b->second;
                if( all_data[2 * idx1 + 1] == all_data[2 * idx2 + 1] ) continue;

                ierr = iBase_FAILURE;

                int proc1 = std::lower_bound( displ.begin(), displ.end(), 2 * idx1 ) - displ.begin();
                if( displ[proc1] != 2 * idx1 ) ++proc1;
                int proc2 = std::lower_bound( displ.begin(), displ.end(), 2 * idx2 ) - displ.begin();
                if( displ[proc2] != 2 * idx2 ) ++proc2;

                std::cerr << "Error at " __FILE__ ":" << __LINE__ << " : " << std::endl
                          << "  For vertex at (" << ( a->first >> 2 ) << ", " << ( a->first & 3 ) << ") :" << std::endl
                          << "  Processor " << proc1 << " has " << all_data[2 * idx1 + 1] << " as the owning part"
                          << std::endl
                          << "  Processor " << proc2 << " has " << all_data[2 * idx2 + 1] << " as the owning part"
                          << std::endl;
            }
        }
    }

    return ierr;
}

static int get_part_boundary_verts( iMesh_Instance imesh,
                                    iMeshP_PartitionHandle prtn,
                                    const PartMap& map,
                                    iMeshP_PartHandle part,
                                    std::vector< iBase_EntityHandle >& boundary )
{
    int ierr, logical_id;
    ierr = map.part_from_coords( imesh, part, logical_id );CHKERR;

    int neighbors[5], num_neighbors;
    get_part_neighbors( logical_id, map.get_parts().size(), neighbors, num_neighbors );

    for( int j = 0; j < num_neighbors; ++j )
    {
        std::vector< iBase_EntityHandle > iface;
        ierr = interface_verts( imesh, prtn, part, neighbors[j], map, iface );CHKERR;
        std::copy( iface.begin(), iface.end(), std::back_inserter( boundary ) );
    }

    std::sort( boundary.begin(), boundary.end() );
    boundary.erase( std::unique( boundary.begin(), boundary.end() ), boundary.end() );
    return iBase_SUCCESS;
}

/**\brief Test entity status
 *
 * Test:
 * - iMeshP_getEntStatus
 * - iMeshP_getEntStatusArr
 */
int test_entity_status( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank, size;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    // get local part handles
    std::vector< iMeshP_PartHandle > parts;
    ierr = get_local_parts( imesh, prtn, parts );
    PCHECK;

    // for each part
    int num_quad_ent_incorrect = 0, num_quad_ent_error = 0;
    int num_quad_arr_incorrect = 0, num_quad_arr_error = 0;
    int num_vert_ent_incorrect = 0, num_vert_ent_error = 0;
    int num_vert_arr_incorrect = 0, num_vert_arr_error = 0;
    for( size_t i = 0; i < parts.size(); ++i )
    {
        const iMeshP_PartHandle part = parts[i];

        // get quads and vertices
        std::vector< iBase_EntityHandle > quads, verts;
        ierr = get_part_quads_and_verts( imesh, part, quads, verts );
        if( ierr ) break;

        // check quad status (no ghosting yet)
        for( size_t j = 0; j < quads.size(); ++j )
        {
            int status;
            iMeshP_getEntStatus( imesh, prtn, part, quads[j], &status, &ierr );
            if( ierr != iBase_SUCCESS )
            {
                ++num_quad_ent_error;
                ierr = iBase_SUCCESS;
                continue;
            }

            if( status != iMeshP_INTERNAL ) ++num_quad_ent_incorrect;
        }

        // check quad status using iMeshP_getEntStatusArr
        std::vector< int > stat_list( quads.size() );
        int* junk1 = &stat_list[0];
        int junk2  = stat_list.size(), count;
        iMeshP_getEntStatusArr( imesh, prtn, part, &quads[0], quads.size(), &junk1, &junk2, &count, &ierr );
        if( ierr != iBase_SUCCESS )
        {
            ++num_quad_arr_error;
            ierr = iBase_SUCCESS;
            continue;
        }
        assert( junk1 == &stat_list[0] );
        assert( junk2 == (int)stat_list.size() );
        assert( count == (int)quads.size() );
        for( size_t j = 0; j < quads.size(); ++j )
            if( stat_list[j] != iMeshP_INTERNAL ) ++num_quad_arr_incorrect;

        // figure out which vertices are on the boundary
        std::vector< iBase_EntityHandle > boundary;
        ierr = get_part_boundary_verts( imesh, prtn, map, part, boundary );
        if( ierr ) break;
        std::sort( boundary.begin(), boundary.end() );

        // check vertex status (no ghosting yet)
        for( size_t j = 0; j < verts.size(); ++j )
        {
            int status;
            iMeshP_getEntStatus( imesh, prtn, part, verts[j], &status, &ierr );
            if( ierr != iBase_SUCCESS )
            {
                ++num_vert_ent_error;
                ierr = iBase_SUCCESS;
                continue;
            }
            bool on_boundary = std::binary_search( boundary.begin(), boundary.end(), verts[j] );
            if( status != ( on_boundary ? iMeshP_BOUNDARY : iMeshP_INTERNAL ) ) ++num_vert_ent_incorrect;
        }

        // check vert status using iMeshP_getEntStatusArr
        stat_list.resize( verts.size() );
        junk1 = &stat_list[0];
        junk2 = stat_list.size();
        iMeshP_getEntStatusArr( imesh, prtn, part, &verts[0], verts.size(), &junk1, &junk2, &count, &ierr );
        if( ierr != iBase_SUCCESS )
        {
            ++num_vert_arr_error;
            ierr = iBase_SUCCESS;
            continue;
        }
        assert( junk1 == &stat_list[0] );
        assert( junk2 == (int)stat_list.size() );
        assert( count == (int)verts.size() );
        for( size_t j = 0; j < verts.size(); ++j )
        {
            bool on_boundary = std::binary_search( boundary.begin(), boundary.end(), verts[j] );
            if( stat_list[j] != ( on_boundary ? iMeshP_BOUNDARY : iMeshP_INTERNAL ) ) ++num_vert_arr_incorrect;
        }
    }
    PCHECK;  // check if loop interrupted by any internal errors

    ASSERT( 0 == num_quad_ent_error );
    ASSERT( 0 == num_quad_arr_error );
    ASSERT( 0 == num_vert_ent_error );
    ASSERT( 0 == num_vert_arr_error );
    ASSERT( 0 == num_quad_ent_incorrect );
    ASSERT( 0 == num_quad_arr_incorrect );
    ASSERT( 0 == num_vert_ent_incorrect );
    ASSERT( 0 == num_vert_arr_incorrect );

    return iBase_SUCCESS;
}

/**\brief Test information about entity copies for interface entities
 *
 * Test:
 * - iMeshP_getNumCopies
 * - iMeshP_getCopyParts
 */
int test_entity_copy_parts( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank, size;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    // get local part handles
    std::vector< iMeshP_PartHandle > parts;
    ierr = get_local_parts( imesh, prtn, parts );
    PCHECK;
    ASSERT( !parts.empty() );

    // select a singe part to test
    const iMeshP_PartHandle part = parts[0];
    int logical_id;
    ierr = map.part_from_coords( imesh, part, logical_id );CHKERR;
    const iMeshP_Part part_id = map.part_id_from_local_id( logical_id );

    // get vertices in part
    std::vector< iBase_EntityHandle > quads, verts;
    ierr = get_part_quads_and_verts( imesh, part, quads, verts );
    PCHECK;

    // get neighbors
    int neighbors[5], num_neighbors;
    get_part_neighbors( logical_id, map.get_parts().size(), neighbors, num_neighbors );

    // build map of sharing data for each vertex
    std::map< iBase_EntityHandle, std::vector< iMeshP_Part > > vert_sharing;
    for( int j = 0; j < num_neighbors; ++j )
    {
        std::vector< iBase_EntityHandle > iface;
        ierr = interface_verts( imesh, prtn, part, neighbors[j], map, iface );CHKERR;
        for( size_t k = 0; k < iface.size(); ++k )
            vert_sharing[iface[k]].push_back( map.part_id_from_local_id( neighbors[j] ) );
    }

    // test getNumCopies for each vertex
    std::map< iBase_EntityHandle, std::vector< iMeshP_Part > >::iterator i;
    int num_failed = 0, num_incorrect = 0;
    for( i = vert_sharing.begin(); i != vert_sharing.end(); ++i )
    {
        int count;
        iBase_EntityHandle vtx = i->first;
        iMeshP_getNumCopies( imesh, prtn, vtx, &count, &ierr );
        if( ierr )
            ++num_failed;
        else if( (unsigned)count != i->second.size() + 1 )  // add one for the part we queried from
            ++num_incorrect;
    }
    ASSERT( 0 == num_failed );
    ASSERT( 0 == num_incorrect );

    // get getCopyParts for each vertex
    num_failed = num_incorrect = 0;
    for( i = vert_sharing.begin(); i != vert_sharing.end(); ++i )
    {
        iMeshP_Part* list = 0;
        int junk          = 0, count;
        iMeshP_getCopyParts( imesh, prtn, i->first, &list, &junk, &count, &ierr );
        if( iBase_SUCCESS != ierr )
        {
            ++num_failed;
            continue;
        }
        if( (unsigned)count != i->second.size() + 1 )
        {  // add one for the part we queried from
            ++num_incorrect;
            free( list );
            continue;
        }

        std::vector< iMeshP_Part > expected( i->second );
        expected.push_back( part_id );
        std::sort( list, list + count );
        std::sort( expected.begin(), expected.end() );
        bool eq = std::equal( list, list + count, expected.begin() );
        free( list );
        if( !eq ) ++num_incorrect;
    }
    ASSERT( 0 == num_failed );
    ASSERT( 0 == num_incorrect );

    return iBase_SUCCESS;
}

// store remote handle data for a vertex
struct VtxCopyData
{
    std::vector< iMeshP_Part > parts;
    std::vector< iBase_EntityHandle > handles;
};

/**\brief Test information about entity copies for interface entities
 *
 * Test:
 * - iMeshP_getCopies
 * - iMeshP_getCopyOnPart
 * - iMeshP_getOwnerCopy
 */
int test_entity_copies( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& /* map */ )
{
    int ierr, rank, size;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    // generate a unique ID for each vertex using the coordinates.
    // see create_mesh(..): each vertex has integer coordinates (x,y,0)
    //                      with x in [0,inf] and y in [0,4]
    // then to an Allgatherv to exchange handles for each processor

    // cast everything to iBase_EntityHandle so we can pack it all in one communication
    MPI_Datatype tmp_type;
    if( sizeof( iBase_EntityHandle ) == sizeof( unsigned ) )
        tmp_type = MPI_UNSIGNED;
    else if( sizeof( iBase_EntityHandle ) == sizeof( unsigned long ) )
        tmp_type = MPI_UNSIGNED_LONG;
    else if( sizeof( iBase_EntityHandle ) == sizeof( unsigned long long ) )
        tmp_type = MPI_UNSIGNED_LONG_LONG;
    else
        return iBase_FAILURE;
    const MPI_Datatype type = tmp_type;  // make it const

    // get local part handles
    std::vector< iMeshP_PartHandle > parts;
    ierr = get_local_parts( imesh, prtn, parts );
    PCHECK;
    std::vector< iMeshP_Part > part_ids( parts.size() );
    iMeshP_Part* junk1 = &part_ids[0];
    int junk2          = part_ids.size(), junk3;
    iMeshP_getPartIdsFromPartHandlesArr( imesh, prtn, &parts[0], parts.size(), &junk1, &junk2, &junk3, &ierr );
    PCHECK;
    assert( junk1 == &part_ids[0] );
    assert( junk2 == (int)part_ids.size() );
    assert( junk3 == (int)parts.size() );

    // build list of {vtx_id, part_id, handle} tuples to send
    // also build list of local vertex handles
    std::vector< iBase_EntityHandle > local_data, local_vertices;
    for( size_t i = 0; i < parts.size(); ++i )
    {
        // get vertices
        std::vector< iBase_EntityHandle > quads, verts;
        ierr = get_part_quads_and_verts( imesh, parts[i], quads, verts );
        if( ierr ) break;

        // add all vertices to local_data
        for( size_t j = 0; j < verts.size(); ++j )
        {
            int tag = 0;
            ierr    = vertex_tag( imesh, verts[j], tag );
            if( ierr ) break;
            long tmp_h = tag;
            local_data.push_back( (iBase_EntityHandle)tmp_h );
            tmp_h = part_ids[i];
            local_data.push_back( (iBase_EntityHandle)tmp_h );
            local_data.push_back( verts[j] );
        }
        if( ierr ) break;

        std::copy( verts.begin(), verts.end(), std::back_inserter( local_vertices ) );
    }

    // build list of local vertices
    std::sort( local_vertices.begin(), local_vertices.end() );
    local_vertices.erase( std::unique( local_vertices.begin(), local_vertices.end() ), local_vertices.end() );
    std::vector< int > local_vtx_tags( local_vertices.size() );CHKERR;
    for( size_t i = 0; i < local_vertices.size(); ++i )
    {
        ierr = vertex_tag( imesh, local_vertices[i], local_vtx_tags[i] );
        if( ierr ) break;
    }
    CHKERR;

    // communicate data
    std::vector< int > gcounts( size ), gdisp( size );
    int local_data_size = local_data.size();
    ierr                = MPI_Allgather( &local_data_size, 1, MPI_INT, &gcounts[0], 1, MPI_INT, MPI_COMM_WORLD );CHKERR;
    gdisp[0] = 0;
    for( int i = 1; i < size; ++i )
        gdisp[i] = gdisp[i - 1] + gcounts[i - 1];
    std::vector< iBase_EntityHandle > global_data( gdisp[size - 1] + gcounts[size - 1] );
    ierr = MPI_Allgatherv( &local_data[0], local_data_size, type, &global_data[0], &gcounts[0], &gdisp[0], type,
                           MPI_COMM_WORLD );CHKERR;

    // arrange global data in a more useful way
    std::map< int, VtxCopyData > vtx_sharing;
    assert( global_data.size() % 3 == 0 );
    for( size_t i = 0; i < global_data.size(); i += 3 )
    {
        int tag                   = (int)(size_t)global_data[i];
        iMeshP_Part part          = (iMeshP_Part)(size_t)global_data[i + 1];
        iBase_EntityHandle handle = global_data[i + 2];
        vtx_sharing[tag].parts.push_back( part );
        vtx_sharing[tag].handles.push_back( handle );
    }

    // test iMeshP_getCopies for each local vertex
    int num_error = 0, num_incorrect = 0, junk4;
    for( size_t i = 0; i < local_vertices.size(); ++i )
    {
        int num_copies = -1;
        // iMeshP_Part* part_ids = 0;
        iMeshP_Part* ptr_part_ids  = 0;  // Use ptr_part_ids to avoid shadowing std::vector<iMeshP_Part> part_ids
        iBase_EntityHandle* copies = 0;
        junk2 = junk3 = junk4 = 0;
        iMeshP_getCopies( imesh, prtn, local_vertices[i], &ptr_part_ids, &junk2, &num_copies, &copies, &junk3, &junk4,
                          &ierr );
        if( iBase_SUCCESS != ierr )
        {
            ++num_error;
            continue;
        }
        assert( junk4 == num_copies );

        VtxCopyData& expected = vtx_sharing[local_vtx_tags[i]];
        if( num_copies != (int)expected.parts.size() )
            ++num_incorrect;
        else
            for( size_t j = 0; j < expected.parts.size(); ++j )
            {
                int idx = std::find( ptr_part_ids, ptr_part_ids + num_copies, expected.parts[j] ) - ptr_part_ids;
                if( idx == num_copies || copies[idx] != expected.handles[j] )
                {
                    ++num_incorrect;
                    break;
                }
            }
        free( ptr_part_ids );
        free( copies );
    }
    ASSERT( 0 == num_error );
    ASSERT( 0 == num_incorrect );

    // test iMeshP_getCopyOnPart for each local vertex
    num_error = num_incorrect = 0;
    for( size_t i = 0; i < local_vertices.size(); ++i )
    {
        VtxCopyData& expected = vtx_sharing[local_vtx_tags[i]];
        for( size_t j = 0; j < expected.parts.size(); ++j )
        {
            iBase_EntityHandle copy;
            iMeshP_getCopyOnPart( imesh, prtn, local_vertices[i], expected.parts[j], &copy, &ierr );
            if( iBase_SUCCESS != ierr )
                ++num_error;
            else if( expected.handles[j] != copy )
                ++num_incorrect;
        }
    }
    ASSERT( 0 == num_error );
    ASSERT( 0 == num_incorrect );

    // test iMeshP_getOwnerCopy for each local vertex
    num_error = num_incorrect = 0;
    for( size_t i = 0; i < local_vertices.size(); ++i )
    {
        VtxCopyData& expected = vtx_sharing[local_vtx_tags[i]];
        iMeshP_Part owner_id  = 0;
        iMeshP_getEntOwnerPart( imesh, prtn, local_vertices[i], &owner_id, &ierr );
        if( iBase_SUCCESS != ierr ) continue;  // not testing getEntOwnerPart here

        size_t idx = std::find( expected.parts.begin(), expected.parts.end(), owner_id ) - expected.parts.begin();
        if( idx == expected.parts.size() ) continue;  // not testing getEntOwnerPart here

        iMeshP_Part owner_id_2  = 0;
        iBase_EntityHandle copy = 0;
        iMeshP_getOwnerCopy( imesh, prtn, local_vertices[i], &owner_id_2, &copy, &ierr );
        if( iBase_SUCCESS != ierr )
            ++num_error;
        else if( owner_id_2 != owner_id && copy != expected.handles[idx] )
            ++num_incorrect;
    }
    ASSERT( 0 == num_error );
    ASSERT( 0 == num_incorrect );

    return iBase_SUCCESS;
}

int get_num_adj_quads( iMesh_Instance imesh, iBase_EntityHandle vtx, int& num )
{
    iBase_EntityHandle* list = 0;
    int ierr, junk = 0;
    iMesh_getEntAdj( imesh, vtx, iBase_FACE, &list, &junk, &num, &ierr );
    if( iBase_SUCCESS == ierr ) free( list );
    return ierr;
}

int get_adj( iMesh_Instance imesh, iBase_EntityHandle ent, int type, std::vector< iBase_EntityHandle >& adj )
{
    iBase_EntityHandle* list = 0;
    int ierr, num, junk = 0;
    iMesh_getEntAdj( imesh, ent, type, &list, &junk, &num, &ierr );
    if( iBase_SUCCESS == ierr )
    {
        std::copy( list, list + num, std::back_inserter( adj ) );
        free( list );
    }
    return ierr;
}

// assume regular quad mesh
int get_boundary_vertices( iMesh_Instance imesh, std::vector< iBase_EntityHandle >& bdry )
{
    int ierr, n;
    iBase_EntitySetHandle root;
    iMesh_getRootSet( imesh, &root, &ierr );CHKERR;
    std::vector< iBase_EntityHandle > all_verts;
    ierr = get_entities( imesh, root, iBase_VERTEX, iMesh_POINT, all_verts );CHKERR;
    bdry.clear();
    for( size_t i = 0; i < all_verts.size(); ++i )
    {
        ierr = get_num_adj_quads( imesh, all_verts[i], n );CHKERR;
        if( n != 4 ) bdry.push_back( all_verts[i] );
    }
    return iBase_SUCCESS;
}

int check_one_layer( iMesh_Instance imesh,
                     iBase_EntityHandle vtx,
                     const std::vector< iBase_EntityHandle >& sorted_vertices )
{
    int ierr;
    if( std::binary_search( sorted_vertices.begin(), sorted_vertices.end(), vtx ) ) return iBase_SUCCESS;
    std::vector< iBase_EntityHandle > quads, verts;
    ierr = get_adj( imesh, vtx, iBase_FACE, quads );CHKERR;
    for( size_t i = 0; i < quads.size(); ++i )
    {
        verts.clear();
        ierr = get_adj( imesh, quads[i], iBase_VERTEX, verts );CHKERR;
        for( size_t j = 0; j < verts.size(); ++j )
        {
            if( std::binary_search( sorted_vertices.begin(), sorted_vertices.end(), verts[j] ) ) return iBase_SUCCESS;
        }
    }

    return iBase_FAILURE;
}

// get number of adjacent quads to each vertex, both on the local
// processor and in the entire mesh
int get_num_adj_all( iMesh_Instance imesh,
                     const std::vector< iBase_EntityHandle >& verts,
                     std::vector< int >& num_local_adj,
                     std::vector< int >& num_all_adj )
{
    int ierr, size;
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    std::vector< int > vtx_tags( verts.size() );
    num_local_adj.resize( verts.size() );
    for( size_t i = 0; i < verts.size(); ++i )
    {
        ierr = get_num_adj_quads( imesh, verts[i], num_local_adj[i] );CHKERR;
        ierr = vertex_tag( imesh, verts[i], vtx_tags[i] );CHKERR;
    }

    std::vector< int > counts( size ), displ( size );
    int num_vtx = verts.size();
    ierr        = MPI_Allgather( &num_vtx, 1, MPI_INT, &counts[0], 1, MPI_INT, MPI_COMM_WORLD );CHKERR;
    displ[0] = 0;
    for( int i = 1; i < size; ++i )
        displ[i] = displ[i - 1] + counts[i - 1];
    int total = displ[size - 1] + counts[size - 1];
    std::vector< int > all_tags( total ), all_adj_counts( total );
    ierr = MPI_Allgatherv( &vtx_tags[0], vtx_tags.size(), MPI_INT, &all_tags[0], &counts[0], &displ[0], MPI_INT,
                           MPI_COMM_WORLD );CHKERR;
    ierr = MPI_Allgatherv( &num_local_adj[0], num_local_adj.size(), MPI_INT, &all_adj_counts[0], &counts[0], &displ[0],
                           MPI_INT, MPI_COMM_WORLD );CHKERR;

    num_all_adj.clear();
    num_all_adj.resize( total, 0 );
    for( int i = 0; i < total; ++i )
    {
        std::vector< int >::iterator it = std::find( vtx_tags.begin(), vtx_tags.end(), all_tags[i] );
        if( it == vtx_tags.end() ) continue;
        int idx = it - vtx_tags.begin();
        num_all_adj[idx] += all_adj_counts[i];
    }

    return iBase_SUCCESS;
}

/**\brief Test creation of ghost entities
 *
 * Test:
 * - iMeshP_createGhostEntsAll
 */
int test_create_ghost_ents( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& /* map */ )
{
    int ierr;

    // get boundary vertices
    std::vector< iBase_EntityHandle > bdry;
    ierr = get_boundary_vertices( imesh, bdry );
    PCHECK;
    // get counts of adjacent entities
    std::vector< int > num_local_adj, num_global_adj;
    ierr = get_num_adj_all( imesh, bdry, num_local_adj, num_global_adj );
    PCHECK;
    // create one layer of ghost entities
    iMeshP_createGhostEntsAll( imesh, prtn, iBase_FACE, iBase_VERTEX, 1, 0, &ierr );
    PCHECK;
    // check that each vertex has the correct number of adjacent entities
    int num_incorrect = 0;
    for( size_t i = 0; i < bdry.size(); ++i )
    {
        int n;
        ierr = get_num_adj_quads( imesh, bdry[i], n );
        if( iBase_SUCCESS != ierr || num_global_adj[i] != n ) ++num_incorrect;
    }
    ASSERT( 0 == num_incorrect );
    // get new the new boundary
    std::vector< iBase_EntityHandle > new_bdry;
    ierr = get_boundary_vertices( imesh, new_bdry );
    PCHECK;
    // check that each vertex on the new boundary is separated by
    // at most one layer from the old boundary
    std::sort( bdry.begin(), bdry.end() );
    num_incorrect = 0;
    for( size_t i = 0; i < new_bdry.size(); ++i )
    {
        ierr = check_one_layer( imesh, new_bdry[i], bdry );
        if( ierr ) ++num_incorrect;
    }
    ASSERT( 0 == num_incorrect );
    // make another layer of ghost entiites
    bdry.swap( new_bdry );
    new_bdry.clear();
    ierr = get_num_adj_all( imesh, bdry, num_local_adj, num_global_adj );
    PCHECK;
    iMeshP_createGhostEntsAll( imesh, prtn, iBase_FACE, iBase_VERTEX, 2, 0, &ierr );
    PCHECK;
    // check that each vertex has the correct number of adjacent entities
    num_incorrect = 0;
    for( size_t i = 0; i < bdry.size(); ++i )
    {
        int n;
        ierr = get_num_adj_quads( imesh, bdry[i], n );
        if( iBase_SUCCESS != ierr || num_global_adj[i] != n ) ++num_incorrect;
    }
    // check that each vertex on the new boundary is separated by
    // at most one layer from the old boundary
    std::sort( bdry.begin(), bdry.end() );
    num_incorrect = 0;
    for( size_t i = 0; i < new_bdry.size(); ++i )
    {
        ierr = check_one_layer( imesh, new_bdry[i], bdry );
        if( ierr ) ++num_incorrect;
    }
    ASSERT( 0 == num_incorrect );

    return iBase_SUCCESS;
}

/**\brief Test exchange entities
 *
 * Test:
 * - iMeshP_exchEntArrToPartsAll
 */
int test_exchange_ents( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& map )
{
    int ierr, rank, size;
    int num_err = 0;
    iMeshP_RequestHandle request;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    std::vector< iBase_EntityHandle > all_elems;
    std::vector< iMeshP_Part > all_ids;
    std::vector< iBase_EntityHandle > quads;

    // get local part handles and part ids
    std::vector< iMeshP_PartHandle > local_handles;
    std::vector< iMeshP_Part > local_ids;
    ierr = get_local_parts( imesh, prtn, local_handles, &local_ids );
    PCHECK;

    // get loacal quads before exchange
    quads.clear();
    ierr = get_entities( imesh, local_handles[0], iBase_FACE, iMesh_QUADRILATERAL, quads );CHKERR;
    int n_quads = quads.size();

    // send all elements in local processor to all other processors
    for( size_t i = 0; i < map.get_parts().size(); ++i )
    {
        if( map.get_parts()[i] == (unsigned int)rank ) continue;  // skip own rank

        for( int j = 0; j < n_quads; j++ )
        {
            all_elems.push_back( quads[j] );
            all_ids.push_back( map.get_parts()[i] );
        }
    }

    // exchange entities
    iMeshP_exchEntArrToPartsAll( imesh, prtn, &all_elems[0], all_elems.size(), &all_ids[0], 0, 0, &request, &ierr );
    if( iBase_SUCCESS != ierr ) ++num_err;

    // get local quads after exchange
    quads.clear();
    ierr = get_entities( imesh, local_handles[0], iBase_FACE, iMesh_QUADRILATERAL, quads );CHKERR;

    // # of elements should be # of quads * # of processors
    ASSERT( quads.size() == (unsigned int)n_quads * size );

    ASSERT( 0 == num_err );

    return iBase_SUCCESS;
}

/**\brief Test commuinication of tag data
 *
 * Test:
 * - iMeshP_pushTags
 * - iMeshP_pushTagsEnt
 */
int test_push_tag_data_common( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, int num_ghost_layers )
{
    const char* src_name = "test_src";
    const char* dst_name = "test_dst";
    int ierr, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    if( num_ghost_layers )
    {
        iMeshP_createGhostEntsAll( imesh, prtn, iBase_FACE, iBase_VERTEX, num_ghost_layers, 0, &ierr );
        PCHECK;
    }

    iBase_TagHandle src_tag, dst_tag;
    iMesh_createTag( imesh, src_name, 1, iBase_INTEGER, &src_tag, &ierr, strlen( src_name ) );CHKERR;
    iMesh_createTag( imesh, dst_name, 1, iBase_INTEGER, &dst_tag, &ierr, strlen( dst_name ) );CHKERR;

    iBase_EntitySetHandle root;
    iMesh_getRootSet( imesh, &root, &ierr );CHKERR;

    std::vector< iBase_EntityHandle > verts;
    ierr = get_entities( imesh, root, iBase_VERTEX, iMesh_POINT, verts );CHKERR;

    // test iMeshP_pushTags
    // each processor writes its rank on all vertices
    // after push, each vertex should be tagged with the rank of its owner

    std::vector< int > tag_vals( verts.size(), rank );
    iMesh_setIntArrData( imesh, &verts[0], verts.size(), src_tag, &tag_vals[0], tag_vals.size(), &ierr );CHKERR;

    iMeshP_pushTags( imesh, prtn, src_tag, dst_tag, iBase_VERTEX, iMesh_POINT, &ierr );
    PCHECK;

    tag_vals.clear();
    tag_vals.resize( verts.size(), -1 );
    iBase_TagHandle id_tag;
    iMesh_getTagHandle( imesh, "GLOBAL_ID", &id_tag, &ierr, strlen( "GLOBAL_ID" ) );
    std::vector< int > ids( verts.size() );
    int *junk1 = &ids[0], junk2 = ids.size(), junk3;
    iMesh_getIntArrData( imesh, &verts[0], verts.size(), id_tag, &junk1, &junk2, &junk3, &ierr );
    PCHECK;
    int errcount = 0;
    for( size_t i = 0; i < verts.size(); ++i )
    {
        iMesh_getIntData( imesh, verts[i], dst_tag, &tag_vals[i], &ierr );
        if( ierr != iBase_SUCCESS )
        {
            std::cerr << "Rank " << rank << " : getIntData failed for vertex " << ids[i] << std::endl;
            std::cerr.flush();
            ++errcount;
        }
    }
    ASSERT( 0 == errcount );

    //  int *junk1 = &tag_vals[0], junk2 = tag_vals.size(), junk3;
    //  iMesh_getIntArrData( imesh, &verts[0], verts.size(), dst_tag, &junk1, &junk2, &junk3, &ierr
    //  ); PCHECK; assert( junk1 == &tag_vals[0] ); assert( junk2 == (int)tag_vals.size() ); assert(
    //  junk3 == (int)verts.size() );

    std::vector< int > expected( verts.size() );
    std::vector< iMeshP_Part > parts( verts.size() );
    iMeshP_Part* junk4 = &parts[0];
    junk2              = parts.size();
    iMeshP_getEntOwnerPartArr( imesh, prtn, &verts[0], verts.size(), &junk4, &junk2, &junk3, &ierr );
    PCHECK;
    assert( junk4 == &parts[0] );
    assert( junk2 == (int)parts.size() );
    assert( junk3 == (int)verts.size() );
    junk1 = &expected[0];
    junk2 = expected.size();
    iMeshP_getRankOfPartArr( imesh, prtn, &parts[0], parts.size(), &junk1, &junk2, &junk3, &ierr );
    PCHECK;
    assert( junk1 == &expected[0] );
    assert( junk2 == (int)expected.size() );
    assert( junk3 == (int)parts.size() );

    ASSERT( tag_vals == expected );

    // test iMeshP_pushTagsEnt
    // write -1 on all vertices
    // For each vertex owned by this processor and shared with more than
    // two others, write the rank of the owning processor.

    tag_vals.clear();
    tag_vals.resize( verts.size(), -1 );
    iMesh_setIntArrData( imesh, &verts[0], verts.size(), src_tag, &tag_vals[0], tag_vals.size(), &ierr );
    PCHECK;
    tag_vals.resize( verts.size(), -1 );
    iMesh_setIntArrData( imesh, &verts[0], verts.size(), dst_tag, &tag_vals[0], tag_vals.size(), &ierr );
    PCHECK;

    std::vector< iBase_EntityHandle > some;
    for( size_t i = 0; i < verts.size(); ++i )
    {
        int num;
        iMeshP_getNumCopies( imesh, prtn, verts[i], &num, &ierr );
        if( iBase_SUCCESS != ierr ) break;
        if( num > 2 )
            some.push_back( verts[i] );
        else
            expected[i] = -1;
    }

    tag_vals.clear();
    tag_vals.resize( some.size(), rank );
    iMesh_setIntArrData( imesh, &some[0], some.size(), src_tag, &tag_vals[0], tag_vals.size(), &ierr );
    PCHECK;

    iMeshP_pushTagsEnt( imesh, prtn, src_tag, dst_tag, &some[0], some.size(), &ierr );
    PCHECK;

    tag_vals.clear();
    tag_vals.resize( verts.size(), -1 );
    junk1 = &tag_vals[0];
    junk2 = tag_vals.size();
    iMesh_getIntArrData( imesh, &verts[0], verts.size(), dst_tag, &junk1, &junk2, &junk3, &ierr );CHKERR;
    assert( junk1 == &tag_vals[0] );
    assert( junk2 == (int)tag_vals.size() );
    assert( junk3 == (int)verts.size() );

    ASSERT( tag_vals == expected );
    return iBase_SUCCESS;
}

/**\brief Test commuinication of tag data
 *
 * Test:
 * - iMeshP_pushTags
 * - iMeshP_pushTagsEnt
 */
int test_push_tag_data_iface( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& )
{
    return test_push_tag_data_common( imesh, prtn, 0 );
}

/**\brief Test commuinication of tag data
 *
 * Test:
 * - iMeshP_pushTags
 * - iMeshP_pushTagsEnt
 */
int test_push_tag_data_ghost( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, const PartMap& )
{
    return test_push_tag_data_common( imesh, prtn, 1 );
}

/**************************************************************************
                          PartMap class
 **************************************************************************/

int PartMap::build_map( iMesh_Instance imesh, iMeshP_PartitionHandle prtn, int num_expected_parts )
{
    int ierr, rank, size;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    // get local parts
    std::vector< iMeshP_PartHandle > local_parts;
    std::vector< iMeshP_Part > imesh_ids;
    ierr = get_local_parts( imesh, prtn, local_parts, &imesh_ids );CHKERR;

    // get logical ids for local parts
    std::vector< int > local_ids( local_parts.size() );
    for( size_t i = 0; i < local_parts.size(); ++i )
    {
        ierr = part_from_coords( imesh, local_parts[i], local_ids[i] );CHKERR;
    }

    // get total number of parts
    int num_global = 0, num_local = local_parts.size();
    ierr = MPI_Allreduce( &num_local, &num_global, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );CHKERR;
    if( num_global != num_expected_parts )
    {
        std::cerr << "Invalid/unexpected global part count at " __FILE__ ":" << __LINE__ << " (proc " << rank
                  << "): " << std::endl
                  << "  Expected: " << num_expected_parts << std::endl
                  << "  Actual:   " << num_global << std::endl;
        return 1;
    }

    // get counts and displacements for Allgatherv calls
    std::vector< int > dspls( size ), counts( size );
    ierr = MPI_Allgather( &num_local, 1, MPI_INT, &counts[0], 1, MPI_INT, MPI_COMM_WORLD );CHKERR;
    dspls[0] = 0;
    for( int i = 1; i < size; ++i )
        dspls[i] = dspls[i - 1] + counts[i - 1];

    // gather iMeshP_Part list from each processor
    std::vector< unsigned > global_part_ids( num_expected_parts );
    assert( sizeof( iMeshP_Part ) == sizeof( int ) );
    ierr = MPI_Allgatherv( &imesh_ids[0], num_local, MPI_UNSIGNED, &global_part_ids[0], &counts[0], &dspls[0],
                           MPI_UNSIGNED, MPI_COMM_WORLD );CHKERR;

    // gather local ids from each processor
    std::vector< int > global_id_list( num_expected_parts );
    ierr = MPI_Allgatherv( &local_ids[0], num_local, MPI_INT, &global_id_list[0], &counts[0], &dspls[0], MPI_INT,
                           MPI_COMM_WORLD );CHKERR;

    // build owner list
    std::vector< int > global_owners( num_expected_parts );
    for( int i = 0; i < size; ++i )
        for( int j = 0; j < counts[i]; ++j )
            global_owners[dspls[i] + j] = i;

    // populate member lists
    sortedPartList = global_part_ids;
    std::sort( sortedPartList.begin(), sortedPartList.end() );
    partLocalIds.resize( num_expected_parts );
    partRanks.resize( num_expected_parts );
    for( int i = 0; i < num_expected_parts; ++i )
    {
        int idx = std::lower_bound( sortedPartList.begin(), sortedPartList.end(), global_part_ids[i] ) -
                  sortedPartList.begin();
        partLocalIds[idx] = global_id_list[i];
        partRanks[idx]    = global_owners[i];
    }

    // do some consistency checking
    if( std::unique( sortedPartList.begin(), sortedPartList.end() ) != sortedPartList.end() )
    {
        if( rank == 0 )
        {
            std::cerr << "ERROR: Duplicate iMeshP_Part values detected at " __FILE__ ":" << __LINE__ << std::endl;
        }
        return 1;
    }

    // build revesre local id map and check for duplicates
    localIdReverseMap.clear();
    localIdReverseMap.resize( num_expected_parts, -1 );
    for( int i = 0; i < num_expected_parts; ++i )
    {
        int idx = partLocalIds[i];
        if( localIdReverseMap[idx] != -1 )
        {
            if( rank == 0 )
            {
                std::cerr << "ERROR: Part mesh has been duplicated in multiple parts." << std::endl
                          << "  Detected at " __FILE__ ":" << __LINE__ << std::endl
                          << "  See PartMap::part_from_coords" << std::endl;
            }
            return 1;
        }
        if( idx >= num_expected_parts )
        {
            if( rank == 0 )
            {
                std::cerr << "ERROR: Part mesh invalid/incorrect mesh." << std::endl
                          << "  Detected at " __FILE__ ":" << __LINE__ << std::endl
                          << "  See PartMap::part_from_coords" << std::endl;
            }
            return 1;
        }

        localIdReverseMap[idx] = i;
    }

    return 0;
}

void PartMap::part_id_from_rank( int rank, std::vector< iMeshP_Part >& parts ) const
{
    for( size_t i = 0; i < sortedPartList.size(); ++i )
        if( partRanks[i] == rank ) parts.push_back( sortedPartList[i] );
}

void PartMap::local_id_from_rank( int rank, std::vector< int >& ids ) const
{
    for( size_t i = 0; i < sortedPartList.size(); ++i )
        if( partRanks[i] == rank ) ids.push_back( partLocalIds[i] );
}

int PartMap::part_from_coords( iMesh_Instance imesh, iMeshP_PartHandle part, int& id )
{
    int ierr, rank;
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );

    // get elements
    const int num_elem = 4;
    iBase_EntityHandle array[num_elem];
    iBase_EntityHandle* ptr = array;
    int junk1 = num_elem, n = -1;
    iMesh_getEntities( imesh, part, iBase_FACE, iMesh_QUADRILATERAL, &ptr, &junk1, &n, &ierr );CHKERR;
    assert( ptr == array );
    assert( junk1 == num_elem );
    if( n != num_elem )
    {
        std::cerr << "Internal error at " __FILE__ ":" << __LINE__ << " (proc " << rank
                  << "): Expected all parts to have " << num_elem << " elements.  Found one with " << n << std::endl;
        return 1;
    }

    // get vertices
    iBase_EntityHandle adj_array[4 * num_elem];
    int junk2, junk3, offset_array[5];
    ptr       = adj_array;
    junk1     = sizeof( adj_array ) / sizeof( adj_array[0] );
    junk2     = sizeof( offset_array ) / sizeof( offset_array[0] );
    int* ptr2 = offset_array;
    iMesh_getEntArrAdj( imesh, array, num_elem, iBase_VERTEX, &ptr, &junk1, &n, &ptr2, &junk2, &junk3, &ierr );CHKERR;
    assert( ptr == adj_array );
    assert( ptr2 == offset_array );
    assert( junk1 == sizeof( adj_array ) / sizeof( adj_array[0] ) );
    assert( junk2 == sizeof( offset_array ) / sizeof( offset_array[0] ) );
    assert( n == 4 * num_elem );
    assert( offset_array[0] == 0 );
    for( int i = 1; i < junk3; ++i )
        assert( offset_array[i] - offset_array[i - 1] == 4 );

    // find center vertex
    iBase_EntityHandle vtx;
    bool all_match;
    for( int i = 0; i < 4; ++i )
    {
        vtx       = adj_array[i];
        all_match = true;
        for( int j = 1; j < 4; ++j )
        {
            iBase_EntityHandle* mvtx = adj_array + 4 * j;
            int k;
            for( k = 0; k < 4; ++k )
                if( mvtx[k] == vtx ) break;
            if( k == 4 ) all_match = false;
        }
        if( all_match ) break;
    }
    assert( all_match );

    // get center vertex coordinates
    double x, y, z;
    iMesh_getVtxCoord( imesh, vtx, &x, &y, &z, &ierr );CHKERR;
    assert( 0.0 == z );
    const int xi = ( (int)round( x ) - 1 ) / 2;
    const int yi = ( (int)round( y ) - 1 ) / 2;
    assert( xi >= 0 );
    assert( yi >= 0 );
    assert( fabs( x - 2 * xi - 1 ) < 1e-12 );
    assert( fabs( y - 2 * yi - 1 ) < 1e-12 );

    id = 2 * xi + yi;
    return 0;
}