MOAB: Mesh Oriented datABase  (version 5.4.1)
ParallelComm.cpp
Go to the documentation of this file.
00001 #include "moab/Interface.hpp"
00002 #include "moab/ParallelComm.hpp"
00003 #include "moab/WriteUtilIface.hpp"
00004 #include "moab/ReadUtilIface.hpp"
00005 #include "SequenceManager.hpp"
00006 #include "moab/Error.hpp"
00007 #include "EntitySequence.hpp"
00008 #include "MBTagConventions.hpp"
00009 #include "moab/Skinner.hpp"
00010 #include "MBParallelConventions.h"
00011 #include "moab/Core.hpp"
00012 #include "ElementSequence.hpp"
00013 #include "moab/CN.hpp"
00014 #include "moab/RangeMap.hpp"
00015 #include "moab/MeshTopoUtil.hpp"
00016 #include "TagInfo.hpp"
00017 #include "DebugOutput.hpp"
00018 #include "SharedSetData.hpp"
00019 #include "moab/ScdInterface.hpp"
00020 #include "moab/TupleList.hpp"
00021 #include "moab/gs.hpp"
00022 
00023 #include <iostream>
00024 #include <sstream>
00025 #include <algorithm>
00026 #include <functional>
00027 #include <numeric>
00028 
00029 #include <cmath>
00030 #include <cstdlib>
00031 #include <cassert>
00032 
00033 #ifdef MOAB_HAVE_MPI
00034 #include "moab_mpi.h"
00035 #endif
00036 #ifdef MOAB_HAVE_MPE
00037 #include "mpe.h"
00038 int IFACE_START, IFACE_END;
00039 int GHOST_START, GHOST_END;
00040 int SHAREDV_START, SHAREDV_END;
00041 int RESOLVE_START, RESOLVE_END;
00042 int ENTITIES_START, ENTITIES_END;
00043 int RHANDLES_START, RHANDLES_END;
00044 int OWNED_START, OWNED_END;
00045 #endif
00046 
00047 namespace moab
00048 {
00049 
00050 const unsigned int ParallelComm::INITIAL_BUFF_SIZE = 1024;
00051 
00052 const int MAX_BCAST_SIZE = ( 1 << 28 );
00053 
00054 std::vector< ParallelComm::Buffer* > msgs;
00055 unsigned int __PACK_num = 0, __UNPACK_num = 0, __PACK_count = 0, __UNPACK_count = 0;
00056 std::string __PACK_string, __UNPACK_string;
00057 
00058 #ifdef DEBUG_PACKING_TIMES
00059 #define PC( n, m )                                                            \
00060     {                                                                         \
00061         if( __PACK_num == (unsigned int)n && __PACK_string == m )             \
00062             __PACK_count++;                                                   \
00063         else                                                                  \
00064         {                                                                     \
00065             if( __PACK_count > 1 ) std::cerr << " (" << __PACK_count << "x)"; \
00066             __PACK_count  = 1;                                                \
00067             __PACK_string = m;                                                \
00068             __PACK_num    = n;                                                \
00069             std::cerr << std::endl << "PACK: " << n << m;                     \
00070         }                                                                     \
00071     }
00072 #define UPC( n, m )                                                              \
00073     {                                                                            \
00074         if( __UNPACK_num == (unsigned int)n && __UNPACK_string == m )            \
00075             __UNPACK_count++;                                                    \
00076         else                                                                     \
00077         {                                                                        \
00078             if( __UNPACK_count > 1 ) std::cerr << "(" << __UNPACK_count << "x)"; \
00079             __UNPACK_count  = 1;                                                 \
00080             __UNPACK_string = m;                                                 \
00081             __UNPACK_num    = n;                                                 \
00082             std::cerr << std::endl << "UNPACK: " << n << m;                      \
00083         }                                                                        \
00084     }
00085 #else
00086 #define PC( n, m )
00087 #define UPC( n, m )
00088 #endif
00089 
00090 template < typename T >
00091 static inline void UNPACK( unsigned char*& buff, T* val, size_t count )
00092 {
00093     memcpy( val, buff, count * sizeof( T ) );
00094     buff += count * sizeof( T );
00095 }
00096 
00097 template < typename T >
00098 static inline void PACK( unsigned char*& buff, const T* val, size_t count )
00099 {
00100     memcpy( buff, val, count * sizeof( T ) );
00101     buff += count * sizeof( T );
00102 }
00103 
00104 static inline void PACK_INTS( unsigned char*& buff, const int* int_val, size_t num )
00105 {
00106     PACK( buff, int_val, num );
00107     PC( num, " ints" );
00108 }
00109 
00110 static inline void PACK_INT( unsigned char*& buff, int int_val )
00111 {
00112     PACK_INTS( buff, &int_val, 1 );
00113 }
00114 
00115 static inline void PACK_DBLS( unsigned char*& buff, const double* dbl_val, size_t num )
00116 {
00117     PACK( buff, dbl_val, num );
00118     PC( num, " doubles" );
00119 }
00120 
00121 // static inline
00122 // void PACK_DBL(unsigned char*& buff, const double dbl_val)
00123 //{ PACK_DBLS(buff, &dbl_val, 1); }
00124 
00125 static inline void PACK_EH( unsigned char*& buff, const EntityHandle* eh_val, size_t num )
00126 {
00127     PACK( buff, eh_val, num );
00128     PC( num, " handles" );
00129 }
00130 
00131 // static inline
00132 // void PACK_CHAR_64(unsigned char*& buff, const char* str)
00133 //{
00134 //  memcpy(buff, str, 64);
00135 //  buff += 64;
00136 //  PC(64, " chars");
00137 //}
00138 
00139 static inline void PACK_VOID( unsigned char*& buff, const void* val, size_t num )
00140 {
00141     PACK( buff, reinterpret_cast< const unsigned char* >( val ), num );
00142     PC( num, " void" );
00143 }
00144 
00145 static inline void PACK_BYTES( unsigned char*& buff, const void* val, int num )
00146 {
00147     PACK_INT( buff, num );
00148     PACK_VOID( buff, val, num );
00149 }
00150 
00151 static inline void PACK_RANGE( unsigned char*& buff, const Range& rng )
00152 {
00153     PACK_INT( buff, rng.psize() );
00154     Range::const_pair_iterator cit;
00155     for( cit = rng.const_pair_begin(); cit != rng.const_pair_end(); ++cit )
00156     {
00157         EntityHandle eh[2] = { cit->first, cit->second };
00158         PACK_EH( buff, eh, 2 );
00159     }
00160     PC( rng.psize(), "-subranged range" );
00161 }
00162 
00163 static inline void UNPACK_INTS( unsigned char*& buff, int* int_val, size_t num )
00164 {
00165     UNPACK( buff, int_val, num );
00166     UPC( num, " ints" );
00167 }
00168 
00169 static inline void UNPACK_INT( unsigned char*& buff, int& int_val )
00170 {
00171     UNPACK_INTS( buff, &int_val, 1 );
00172 }
00173 
00174 static inline void UNPACK_DBLS( unsigned char*& buff, double* dbl_val, size_t num )
00175 {
00176     UNPACK( buff, dbl_val, num );
00177     UPC( num, " doubles" );
00178 }
00179 
00180 static inline void UNPACK_DBL( unsigned char*& buff, double& dbl_val )
00181 {
00182     UNPACK_DBLS( buff, &dbl_val, 1 );
00183 }
00184 
00185 static inline void UNPACK_EH( unsigned char*& buff, EntityHandle* eh_val, size_t num )
00186 {
00187     UNPACK( buff, eh_val, num );
00188     UPC( num, " handles" );
00189 }
00190 
00191 // static inline
00192 // void UNPACK_CHAR_64(unsigned char*& buff, char* char_val)
00193 //{
00194 //  memcpy(buff, char_val, 64);
00195 //  buff += 64;
00196 //  UPC(64, " chars");
00197 //}
00198 
00199 static inline void UNPACK_VOID( unsigned char*& buff, void* val, size_t num )
00200 {
00201     UNPACK( buff, reinterpret_cast< unsigned char* >( val ), num );
00202     UPC( num, " void" );
00203 }
00204 
00205 static inline void UNPACK_TYPE( unsigned char*& buff, EntityType& type )
00206 {
00207     int int_type = MBMAXTYPE;
00208     UNPACK_INT( buff, int_type );
00209     type = static_cast< EntityType >( int_type );
00210     assert( type >= MBVERTEX && type <= MBMAXTYPE );
00211 }
00212 
00213 static inline void UNPACK_RANGE( unsigned char*& buff, Range& rng )
00214 {
00215     int num_subs;
00216     EntityHandle eh[2];
00217     UNPACK_INT( buff, num_subs );
00218     for( int i = 0; i < num_subs; i++ )
00219     {
00220         UPC( num_subs, "-subranged range" );
00221         UNPACK_EH( buff, eh, 2 );
00222         rng.insert( eh[0], eh[1] );
00223     }
00224 }
00225 
00226 enum MBMessageTag
00227 {
00228     MB_MESG_ANY = MPI_ANY_TAG,
00229     MB_MESG_ENTS_ACK,
00230     MB_MESG_ENTS_SIZE,
00231     MB_MESG_ENTS_LARGE,
00232     MB_MESG_REMOTEH_ACK,
00233     MB_MESG_REMOTEH_SIZE,
00234     MB_MESG_REMOTEH_LARGE,
00235     MB_MESG_TAGS_ACK,
00236     MB_MESG_TAGS_SIZE,
00237     MB_MESG_TAGS_LARGE
00238 };
00239 
00240 static inline size_t RANGE_SIZE( const Range& rng )
00241 {
00242     return 2 * sizeof( EntityHandle ) * rng.psize() + sizeof( int );
00243 }
00244 
00245 #define PRINT_DEBUG_ISEND( A, B, C, D, E )    print_debug_isend( ( A ), ( B ), ( C ), ( D ), ( E ) )
00246 #define PRINT_DEBUG_IRECV( A, B, C, D, E, F ) print_debug_irecv( ( A ), ( B ), ( C ), ( D ), ( E ), ( F ) )
00247 #define PRINT_DEBUG_RECD( A )                 print_debug_recd( ( A ) )
00248 #define PRINT_DEBUG_WAITANY( A, B, C )        print_debug_waitany( ( A ), ( B ), ( C ) )
00249 
00250 void ParallelComm::print_debug_isend( int from, int to, unsigned char* buff, int tag, int sz )
00251 {
00252     myDebug->tprintf( 3, "Isend, %d->%d, buffer ptr = %p, tag=%d, size=%d\n", from, to, (void*)buff, tag, sz );
00253 }
00254 
00255 void ParallelComm::print_debug_irecv( int to, int from, unsigned char* buff, int sz, int tag, int incoming )
00256 {
00257     myDebug->tprintf( 3, "Irecv, %d<-%d, buffer ptr = %p, tag=%d, size=%d", to, from, (void*)buff, tag, sz );
00258     if( tag < MB_MESG_REMOTEH_ACK )
00259         myDebug->printf( 3, ", incoming1=%d\n", incoming );
00260     else if( tag < MB_MESG_TAGS_ACK )
00261         myDebug->printf( 3, ", incoming2=%d\n", incoming );
00262     else
00263         myDebug->printf( 3, ", incoming=%d\n", incoming );
00264 }
00265 
00266 void ParallelComm::print_debug_recd( MPI_Status status )
00267 {
00268     if( myDebug->get_verbosity() == 3 )
00269     {
00270         int this_count;
00271         int success = MPI_Get_count( &status, MPI_UNSIGNED_CHAR, &this_count );
00272         if( MPI_SUCCESS != success ) this_count = -1;
00273         myDebug->tprintf( 3, "Received from %d, count = %d, tag = %d\n", status.MPI_SOURCE, this_count,
00274                           status.MPI_TAG );
00275     }
00276 }
00277 
00278 void ParallelComm::print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc )
00279 {
00280     if( myDebug->get_verbosity() == 3 )
00281     {
00282         myDebug->tprintf( 3, "Waitany, p=%d, ", proc );
00283         if( tag < MB_MESG_REMOTEH_ACK )
00284             myDebug->print( 3, ", recv_ent_reqs=" );
00285         else if( tag < MB_MESG_TAGS_ACK )
00286             myDebug->print( 3, ", recv_remoteh_reqs=" );
00287         else
00288             myDebug->print( 3, ", recv_tag_reqs=" );
00289         for( unsigned int i = 0; i < reqs.size(); i++ )
00290             myDebug->printf( 3, " %p", (void*)(intptr_t)reqs[i] );
00291         myDebug->print( 3, "\n" );
00292     }
00293 }
00294 
00295 /** Name of tag used to store ParallelComm Index on mesh paritioning sets */
00296 const char* PARTITIONING_PCOMM_TAG_NAME = "__PRTN_PCOMM";
00297 
00298 /** \brief Tag storing parallel communication objects
00299  *
00300  * This tag stores pointers to ParallelComm communication
00301  * objects; one of these is allocated for each different
00302  * communicator used to read mesh. ParallelComm stores
00303  * partition and interface sets corresponding to its parallel mesh.
00304  * By default, a parallel read uses the first ParallelComm object
00305  * on the interface instance; if instantiated with one, ReadParallel
00306  * adds this object to the interface instance too.
00307  *
00308  * Tag type: opaque
00309  * Tag size: MAX_SHARING_PROCS*sizeof(ParallelComm*)
00310  */
00311 #define PARALLEL_COMM_TAG_NAME "__PARALLEL_COMM"
00312 
00313 ParallelComm::ParallelComm( Interface* impl, MPI_Comm cm, int* id )
00314     : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
00315       pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
00316       myDebug( NULL )
00317 {
00318     initialize();
00319     sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
00320     if( id ) *id = pcommID;
00321 }
00322 
00323 ParallelComm::ParallelComm( Interface* impl, std::vector< unsigned char >& /*tmp_buff*/, MPI_Comm cm, int* id )
00324     : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
00325       pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
00326       myDebug( NULL )
00327 {
00328     initialize();
00329     sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
00330     if( id ) *id = pcommID;
00331 }
00332 
00333 ParallelComm::~ParallelComm()
00334 {
00335     remove_pcomm( this );
00336     delete_all_buffers();
00337     delete myDebug;
00338     delete sharedSetData;
00339 }
00340 
00341 void ParallelComm::initialize()
00342 {
00343     Core* core      = dynamic_cast< Core* >( mbImpl );
00344     sequenceManager = core->sequence_manager();
00345     mbImpl->query_interface( errorHandler );
00346 
00347     // Initialize MPI, if necessary
00348     int flag   = 1;
00349     int retval = MPI_Initialized( &flag );
00350     if( MPI_SUCCESS != retval || !flag )
00351     {
00352         int argc    = 0;
00353         char** argv = NULL;
00354 
00355         // mpi not initialized yet - initialize here
00356         retval = MPI_Init( &argc, &argv );
00357         assert( MPI_SUCCESS == retval );
00358     }
00359 
00360     // Reserve space for vectors
00361     buffProcs.reserve( MAX_SHARING_PROCS );
00362     localOwnedBuffs.reserve( MAX_SHARING_PROCS );
00363     remoteOwnedBuffs.reserve( MAX_SHARING_PROCS );
00364 
00365     pcommID = add_pcomm( this );
00366 
00367     if( !myDebug )
00368     {
00369         myDebug = new DebugOutput( "ParallelComm", std::cerr );
00370         myDebug->set_rank( procConfig.proc_rank() );
00371     }
00372 }
00373 
00374 int ParallelComm::add_pcomm( ParallelComm* pc )
00375 {
00376     // Add this pcomm to instance tag
00377     std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS, (ParallelComm*)NULL );
00378     Tag pc_tag = pcomm_tag( mbImpl, true );
00379     assert( 0 != pc_tag );
00380 
00381     const EntityHandle root = 0;
00382     ErrorCode result        = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
00383     if( MB_SUCCESS != result && MB_TAG_NOT_FOUND != result ) return -1;
00384     int index = 0;
00385     while( index < MAX_SHARING_PROCS && pc_array[index] )
00386         index++;
00387     if( index == MAX_SHARING_PROCS )
00388     {
00389         index = -1;
00390         assert( false );
00391     }
00392     else
00393     {
00394         pc_array[index] = pc;
00395         mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
00396     }
00397     return index;
00398 }
00399 
00400 void ParallelComm::remove_pcomm( ParallelComm* pc )
00401 {
00402     // Remove this pcomm from instance tag
00403     std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS );
00404     Tag pc_tag = pcomm_tag( mbImpl, true );
00405 
00406     const EntityHandle root                      = 0;
00407     ErrorCode result                             = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
00408     std::vector< ParallelComm* >::iterator pc_it = std::find( pc_array.begin(), pc_array.end(), pc );
00409     assert( MB_SUCCESS == result && pc_it != pc_array.end() );
00410     // Empty if test to get around compiler warning about unused var
00411     if( MB_SUCCESS == result )
00412     {
00413     }
00414 
00415     *pc_it = NULL;
00416     mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
00417 }
00418 
00419 //! Assign a global id space, for largest-dimension or all entities (and
00420 //! in either case for vertices too)
00421 ErrorCode ParallelComm::assign_global_ids( EntityHandle this_set,
00422                                            const int dimension,
00423                                            const int start_id,
00424                                            const bool largest_dim_only,
00425                                            const bool parallel,
00426                                            const bool owned_only )
00427 {
00428     Range entities[4];
00429     ErrorCode result;
00430     std::vector< unsigned char > pstatus;
00431     for( int dim = 0; dim <= dimension; dim++ )
00432     {
00433         if( dim == 0 || !largest_dim_only || dim == dimension )
00434         {
00435             result = mbImpl->get_entities_by_dimension( this_set, dim, entities[dim] );MB_CHK_SET_ERR( result, "Failed to get vertices in assign_global_ids" );
00436         }
00437 
00438         // Need to filter out non-locally-owned entities!!!
00439         pstatus.resize( entities[dim].size() );
00440         result = mbImpl->tag_get_data( pstatus_tag(), entities[dim], &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus in assign_global_ids" );
00441 
00442         Range dum_range;
00443         Range::iterator rit;
00444         unsigned int i;
00445         for( rit = entities[dim].begin(), i = 0; rit != entities[dim].end(); ++rit, i++ )
00446             if( pstatus[i] & PSTATUS_NOT_OWNED ) dum_range.insert( *rit );
00447         entities[dim] = subtract( entities[dim], dum_range );
00448     }
00449 
00450     return assign_global_ids( entities, dimension, start_id, parallel, owned_only );
00451 }
00452 
00453 //! Assign a global id space, for largest-dimension or all entities (and
00454 //! in either case for vertices too)
00455 ErrorCode ParallelComm::assign_global_ids( Range entities[],
00456                                            const int dimension,
00457                                            const int start_id,
00458                                            const bool parallel,
00459                                            const bool owned_only )
00460 {
00461     int local_num_elements[4];
00462     ErrorCode result;
00463     for( int dim = 0; dim <= dimension; dim++ )
00464     {
00465         local_num_elements[dim] = entities[dim].size();
00466     }
00467 
00468     // Communicate numbers
00469     std::vector< int > num_elements( procConfig.proc_size() * 4 );
00470 #ifdef MOAB_HAVE_MPI
00471     if( procConfig.proc_size() > 1 && parallel )
00472     {
00473         int retval =
00474             MPI_Allgather( local_num_elements, 4, MPI_INT, &num_elements[0], 4, MPI_INT, procConfig.proc_comm() );
00475         if( 0 != retval ) return MB_FAILURE;
00476     }
00477     else
00478 #endif
00479         for( int dim = 0; dim < 4; dim++ )
00480             num_elements[dim] = local_num_elements[dim];
00481 
00482     // My entities start at one greater than total_elems[d]
00483     int total_elems[4] = { start_id, start_id, start_id, start_id };
00484 
00485     for( unsigned int proc = 0; proc < procConfig.proc_rank(); proc++ )
00486     {
00487         for( int dim = 0; dim < 4; dim++ )
00488             total_elems[dim] += num_elements[4 * proc + dim];
00489     }
00490 
00491     // Assign global ids now
00492     Tag gid_tag = mbImpl->globalId_tag();
00493 
00494     for( int dim = 0; dim < 4; dim++ )
00495     {
00496         if( entities[dim].empty() ) continue;
00497         num_elements.resize( entities[dim].size() );
00498         int i = 0;
00499         for( Range::iterator rit = entities[dim].begin(); rit != entities[dim].end(); ++rit )
00500             num_elements[i++] = total_elems[dim]++;
00501 
00502         result = mbImpl->tag_set_data( gid_tag, entities[dim], &num_elements[0] );MB_CHK_SET_ERR( result, "Failed to set global id tag in assign_global_ids" );
00503     }
00504 
00505     if( owned_only ) return MB_SUCCESS;
00506 
00507     // Exchange tags
00508     for( int dim = 1; dim < 4; dim++ )
00509         entities[0].merge( entities[dim] );
00510 
00511     return exchange_tags( gid_tag, entities[0] );
00512 }
00513 
00514 int ParallelComm::get_buffers( int to_proc, bool* is_new )
00515 {
00516     int ind                                   = -1;
00517     std::vector< unsigned int >::iterator vit = std::find( buffProcs.begin(), buffProcs.end(), to_proc );
00518     if( vit == buffProcs.end() )
00519     {
00520         assert( "shouldn't need buffer to myself" && to_proc != (int)procConfig.proc_rank() );
00521         ind = buffProcs.size();
00522         buffProcs.push_back( (unsigned int)to_proc );
00523         localOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
00524         remoteOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
00525         if( is_new ) *is_new = true;
00526     }
00527     else
00528     {
00529         ind = vit - buffProcs.begin();
00530         if( is_new ) *is_new = false;
00531     }
00532     assert( ind < MAX_SHARING_PROCS );
00533     return ind;
00534 }
00535 
00536 ErrorCode ParallelComm::broadcast_entities( const int from_proc,
00537                                             Range& entities,
00538                                             const bool adjacencies,
00539                                             const bool tags )
00540 {
00541 #ifndef MOAB_HAVE_MPI
00542     return MB_FAILURE;
00543 #else
00544 
00545     ErrorCode result = MB_SUCCESS;
00546     int success;
00547     int buff_size;
00548 
00549     Buffer buff( INITIAL_BUFF_SIZE );
00550     buff.reset_ptr( sizeof( int ) );
00551     if( (int)procConfig.proc_rank() == from_proc )
00552     {
00553         result = add_verts( entities );MB_CHK_SET_ERR( result, "Failed to add adj vertices" );
00554 
00555         buff.reset_ptr( sizeof( int ) );
00556         result = pack_buffer( entities, adjacencies, tags, false, -1, &buff );MB_CHK_SET_ERR( result, "Failed to compute buffer size in broadcast_entities" );
00557         buff.set_stored_size();
00558         buff_size = buff.buff_ptr - buff.mem_ptr;
00559     }
00560 
00561     success = MPI_Bcast( &buff_size, 1, MPI_INT, from_proc, procConfig.proc_comm() );
00562     if( MPI_SUCCESS != success )
00563     {
00564         MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
00565     }
00566 
00567     if( !buff_size )  // No data
00568         return MB_SUCCESS;
00569 
00570     if( (int)procConfig.proc_rank() != from_proc ) buff.reserve( buff_size );
00571 
00572     size_t offset = 0;
00573     while( buff_size )
00574     {
00575         int sz  = std::min( buff_size, MAX_BCAST_SIZE );
00576         success = MPI_Bcast( buff.mem_ptr + offset, sz, MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
00577         if( MPI_SUCCESS != success )
00578         {
00579             MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer failed" );
00580         }
00581 
00582         offset += sz;
00583         buff_size -= sz;
00584     }
00585 
00586     if( (int)procConfig.proc_rank() != from_proc )
00587     {
00588         std::vector< std::vector< EntityHandle > > dum1a, dum1b;
00589         std::vector< std::vector< int > > dum1p;
00590         std::vector< EntityHandle > dum2, dum4;
00591         std::vector< unsigned int > dum3;
00592         buff.reset_ptr( sizeof( int ) );
00593         result = unpack_buffer( buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );MB_CHK_SET_ERR( result, "Failed to unpack buffer in broadcast_entities" );
00594         std::copy( dum4.begin(), dum4.end(), range_inserter( entities ) );
00595     }
00596 
00597     return MB_SUCCESS;
00598 #endif
00599 }
00600 
00601 ErrorCode ParallelComm::scatter_entities( const int from_proc,
00602                                           std::vector< Range >& entities,
00603                                           const bool adjacencies,
00604                                           const bool tags )
00605 {
00606 #ifndef MOAB_HAVE_MPI
00607     return MB_FAILURE;
00608 #else
00609     ErrorCode result = MB_SUCCESS;
00610     int i, success, buff_size, prev_size;
00611     int nProcs         = (int)procConfig.proc_size();
00612     int* sendCounts    = new int[nProcs];
00613     int* displacements = new int[nProcs];
00614     sendCounts[0]      = sizeof( int );
00615     displacements[0]   = 0;
00616     Buffer buff( INITIAL_BUFF_SIZE );
00617     buff.reset_ptr( sizeof( int ) );
00618     buff.set_stored_size();
00619     unsigned int my_proc = procConfig.proc_rank();
00620 
00621     // Get buffer size array for each remote processor
00622     if( my_proc == (unsigned int)from_proc )
00623     {
00624         for( i = 1; i < nProcs; i++ )
00625         {
00626             prev_size = buff.buff_ptr - buff.mem_ptr;
00627             buff.reset_ptr( prev_size + sizeof( int ) );
00628             result = add_verts( entities[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
00629 
00630             result = pack_buffer( entities[i], adjacencies, tags, false, -1, &buff );
00631             if( MB_SUCCESS != result )
00632             {
00633                 delete[] sendCounts;
00634                 delete[] displacements;
00635                 MB_SET_ERR( result, "Failed to pack buffer in scatter_entities" );
00636             }
00637 
00638             buff_size                               = buff.buff_ptr - buff.mem_ptr - prev_size;
00639             *( (int*)( buff.mem_ptr + prev_size ) ) = buff_size;
00640             sendCounts[i]                           = buff_size;
00641         }
00642     }
00643 
00644     // Broadcast buffer size array
00645     success = MPI_Bcast( sendCounts, nProcs, MPI_INT, from_proc, procConfig.proc_comm() );
00646     if( MPI_SUCCESS != success )
00647     {
00648         delete[] sendCounts;
00649         delete[] displacements;
00650         MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
00651     }
00652 
00653     for( i = 1; i < nProcs; i++ )
00654     {
00655         displacements[i] = displacements[i - 1] + sendCounts[i - 1];
00656     }
00657 
00658     Buffer rec_buff;
00659     rec_buff.reserve( sendCounts[my_proc] );
00660 
00661     // Scatter actual geometry
00662     success = MPI_Scatterv( buff.mem_ptr, sendCounts, displacements, MPI_UNSIGNED_CHAR, rec_buff.mem_ptr,
00663                             sendCounts[my_proc], MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
00664 
00665     if( MPI_SUCCESS != success )
00666     {
00667         delete[] sendCounts;
00668         delete[] displacements;
00669         MB_SET_ERR( MB_FAILURE, "MPI_Scatterv of buffer failed" );
00670     }
00671 
00672     // Unpack in remote processors
00673     if( my_proc != (unsigned int)from_proc )
00674     {
00675         std::vector< std::vector< EntityHandle > > dum1a, dum1b;
00676         std::vector< std::vector< int > > dum1p;
00677         std::vector< EntityHandle > dum2, dum4;
00678         std::vector< unsigned int > dum3;
00679         rec_buff.reset_ptr( sizeof( int ) );
00680         result = unpack_buffer( rec_buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );
00681         if( MB_SUCCESS != result )
00682         {
00683             delete[] sendCounts;
00684             delete[] displacements;
00685             MB_SET_ERR( result, "Failed to unpack buffer in scatter_entities" );
00686         }
00687 
00688         std::copy( dum4.begin(), dum4.end(), range_inserter( entities[my_proc] ) );
00689     }
00690 
00691     delete[] sendCounts;
00692     delete[] displacements;
00693 
00694     return MB_SUCCESS;
00695 #endif
00696 }
00697 
00698 ErrorCode ParallelComm::send_entities( const int to_proc,
00699                                        Range& orig_ents,
00700                                        const bool adjs,
00701                                        const bool tags,
00702                                        const bool store_remote_handles,
00703                                        const bool is_iface,
00704                                        Range& /*final_ents*/,
00705                                        int& incoming1,
00706                                        int& incoming2,
00707                                        TupleList& entprocs,
00708                                        std::vector< MPI_Request >& recv_remoteh_reqs,
00709                                        bool /*wait_all*/ )
00710 {
00711 #ifndef MOAB_HAVE_MPI
00712     return MB_FAILURE;
00713 #else
00714     // Pack entities to local buffer
00715     int ind = get_buffers( to_proc );
00716     localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
00717 
00718     // Add vertices
00719     ErrorCode result = add_verts( orig_ents );MB_CHK_SET_ERR( result, "Failed to add verts in send_entities" );
00720 
00721     // Filter out entities already shared with destination
00722     Range tmp_range;
00723     result = filter_pstatus( orig_ents, PSTATUS_SHARED, PSTATUS_AND, to_proc, &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
00724     if( !tmp_range.empty() )
00725     {
00726         orig_ents = subtract( orig_ents, tmp_range );
00727     }
00728 
00729     result = pack_buffer( orig_ents, adjs, tags, store_remote_handles, to_proc, localOwnedBuffs[ind], &entprocs );MB_CHK_SET_ERR( result, "Failed to pack buffer in send_entities" );
00730 
00731     // Send buffer
00732     result = send_buffer( to_proc, localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], recvReqs[2 * ind + 1],
00733                           (int*)( remoteOwnedBuffs[ind]->mem_ptr ),
00734                           //&ackbuff,
00735                           incoming1, MB_MESG_REMOTEH_SIZE,
00736                           ( !is_iface && store_remote_handles ? localOwnedBuffs[ind] : NULL ),
00737                           &recv_remoteh_reqs[2 * ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to send buffer" );
00738 
00739     return MB_SUCCESS;
00740 #endif
00741 }
00742 
00743 ErrorCode ParallelComm::send_entities( std::vector< unsigned int >& send_procs,
00744                                        std::vector< Range* >& send_ents,
00745                                        int& incoming1,
00746                                        int& incoming2,
00747                                        const bool store_remote_handles )
00748 {
00749 #ifdef MOAB_HAVE_MPE
00750     if( myDebug->get_verbosity() == 2 )
00751     {
00752         MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_entities." );
00753     }
00754 #endif
00755     myDebug->tprintf( 1, "Entering send_entities\n" );
00756     if( myDebug->get_verbosity() == 4 )
00757     {
00758         msgs.clear();
00759         msgs.reserve( MAX_SHARING_PROCS );
00760     }
00761 
00762     unsigned int i;
00763     int ind;
00764     ErrorCode result = MB_SUCCESS;
00765 
00766     // Set buffProcs with communicating procs
00767     unsigned int n_proc = send_procs.size();
00768     for( i = 0; i < n_proc; i++ )
00769     {
00770         ind    = get_buffers( send_procs[i] );
00771         result = add_verts( *send_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
00772 
00773         // Filter out entities already shared with destination
00774         Range tmp_range;
00775         result = filter_pstatus( *send_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
00776         if( !tmp_range.empty() )
00777         {
00778             *send_ents[i] = subtract( *send_ents[i], tmp_range );
00779         }
00780     }
00781 
00782     //===========================================
00783     // Get entities to be sent to neighbors
00784     // Need to get procs each entity is sent to
00785     //===========================================
00786     Range allsent, tmp_range;
00787     int npairs = 0;
00788     TupleList entprocs;
00789     for( i = 0; i < n_proc; i++ )
00790     {
00791         int n_ents = send_ents[i]->size();
00792         if( n_ents > 0 )
00793         {
00794             npairs += n_ents;  // Get the total # of proc/handle pairs
00795             allsent.merge( *send_ents[i] );
00796         }
00797     }
00798 
00799     // Allocate a TupleList of that size
00800     entprocs.initialize( 1, 0, 1, 0, npairs );
00801     entprocs.enableWriteAccess();
00802 
00803     // Put the proc/handle pairs in the list
00804     for( i = 0; i < n_proc; i++ )
00805     {
00806         for( Range::iterator rit = send_ents[i]->begin(); rit != send_ents[i]->end(); ++rit )
00807         {
00808             entprocs.vi_wr[entprocs.get_n()]  = send_procs[i];
00809             entprocs.vul_wr[entprocs.get_n()] = *rit;
00810             entprocs.inc_n();
00811         }
00812     }
00813 
00814     // Sort by handle
00815     moab::TupleList::buffer sort_buffer;
00816     sort_buffer.buffer_init( npairs );
00817     entprocs.sort( 1, &sort_buffer );
00818     entprocs.disableWriteAccess();
00819     sort_buffer.reset();
00820 
00821     myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
00822                       (unsigned long)allsent.size() );
00823 
00824     //===========================================
00825     // Pack and send ents from this proc to others
00826     //===========================================
00827     for( i = 0; i < n_proc; i++ )
00828     {
00829         if( send_ents[i]->size() > 0 )
00830         {
00831             ind = get_buffers( send_procs[i] );
00832             myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", send_ents[i]->compactness(),
00833                               (unsigned long)send_ents[i]->size() );
00834             // Reserve space on front for size and for initial buff size
00835             localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
00836             result = pack_buffer( *send_ents[i], false, true, store_remote_handles, buffProcs[ind],
00837                                   localOwnedBuffs[ind], &entprocs, &allsent );
00838 
00839             if( myDebug->get_verbosity() == 4 )
00840             {
00841                 msgs.resize( msgs.size() + 1 );
00842                 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
00843             }
00844 
00845             // Send the buffer (size stored in front in send_buffer)
00846             result = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind],
00847                                   recvReqs[2 * ind + 1], &ackbuff, incoming1, MB_MESG_REMOTEH_SIZE,
00848                                   ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recvRemotehReqs[2 * ind],
00849                                   &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost send" );
00850         }
00851     }
00852     entprocs.reset();
00853 
00854 #ifdef MOAB_HAVE_MPE
00855     if( myDebug->get_verbosity() == 2 )
00856     {
00857         MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_entities." );
00858     }
00859 #endif
00860 
00861     return MB_SUCCESS;
00862 }
00863 
00864 /////////////////////////////////////////////////////////////////////////////////
00865 // Send and Receive routines for a sequence of entities: use case UMR
00866 /////////////////////////////////////////////////////////////////////////////////
00867 void print_buff( unsigned char* ch, int size )
00868 {
00869     for( int i = 0; i < size; i++ )
00870         std::cout << ch[i];
00871     std::cout << "\n";
00872 }
00873 ErrorCode ParallelComm::send_recv_entities( std::vector< int >& send_procs,
00874                                             std::vector< std::vector< int > >& msgsizes,
00875                                             std::vector< std::vector< EntityHandle > >& senddata,
00876                                             std::vector< std::vector< EntityHandle > >& recvdata )
00877 {
00878 #ifdef USE_MPE
00879     if( myDebug->get_verbosity() == 2 )
00880     {
00881         MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_recv_entities." );
00882     }
00883 #endif
00884     myDebug->tprintf( 1, "Entering send_recv_entities\n" );
00885     if( myDebug->get_verbosity() == 4 )
00886     {
00887         msgs.clear();
00888         msgs.reserve( MAX_SHARING_PROCS );
00889     }
00890 
00891     // unsigned int i;
00892     int i, ind, success;
00893     ErrorCode error = MB_SUCCESS;
00894 
00895     //===========================================
00896     // Pack and send ents from this proc to others
00897     //===========================================
00898 
00899     // std::cout<<"resetting all buffers"<<std::endl;
00900 
00901     reset_all_buffers();
00902     sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
00903     std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
00904     int ack_buff;
00905     int incoming = 0;
00906 
00907     std::vector< unsigned int >::iterator sit;
00908 
00909     for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
00910     {
00911         incoming++;
00912         PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
00913                            MB_MESG_ENTS_SIZE, incoming );
00914 
00915         success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
00916                              MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
00917         if( success != MPI_SUCCESS )
00918         {
00919             MB_SET_ERR( MB_FAILURE, "Failed to post irecv in send_recv_entities" );
00920         }
00921     }
00922 
00923     //  std::set<unsigned int>::iterator it;
00924     for( i = 0; i < (int)send_procs.size(); i++ )
00925     {
00926         // Get index of the shared processor in the local buffer
00927         ind = get_buffers( send_procs[i] );
00928         localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
00929 
00930         int buff_size = msgsizes[i].size() * sizeof( int ) + senddata[i].size() * sizeof( EntityHandle );
00931         localOwnedBuffs[ind]->check_space( buff_size );
00932 
00933         // Pack entities
00934         std::vector< int > msg;
00935         msg.insert( msg.end(), msgsizes[i].begin(), msgsizes[i].end() );
00936         PACK_INTS( localOwnedBuffs[ind]->buff_ptr, &msg[0], msg.size() );
00937 
00938         std::vector< EntityHandle > entities;
00939         entities.insert( entities.end(), senddata[i].begin(), senddata[i].end() );
00940         PACK_EH( localOwnedBuffs[ind]->buff_ptr, &entities[0], entities.size() );
00941         localOwnedBuffs[ind]->set_stored_size();
00942 
00943         if( myDebug->get_verbosity() == 4 )
00944         {
00945             msgs.resize( msgs.size() + 1 );
00946             msgs.back() = new Buffer( *localOwnedBuffs[ind] );
00947         }
00948 
00949         // Send the buffer (size stored in front in send_buffer)
00950         error = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind],
00951                              recv_ent_reqs[3 * ind + 2], &ack_buff, incoming );MB_CHK_SET_ERR( error, "Failed to Isend in send_recv_entities" );
00952     }
00953 
00954     //===========================================
00955     // Receive and unpack ents from received data
00956     //===========================================
00957 
00958     while( incoming )
00959     {
00960 
00961         MPI_Status status;
00962         int index_in_recv_requests;
00963 
00964         PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
00965         success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &index_in_recv_requests, &status );
00966         if( MPI_SUCCESS != success )
00967         {
00968             MB_SET_ERR( MB_FAILURE, "Failed in waitany in send_recv_entities" );
00969         }
00970 
00971         // Processor index in the list is divided by 3
00972         ind = index_in_recv_requests / 3;
00973 
00974         PRINT_DEBUG_RECD( status );
00975 
00976         // OK, received something; decrement incoming counter
00977         incoming--;
00978 
00979         bool done = false;
00980 
00981         error = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind],
00982                              recv_ent_reqs[3 * ind + 1],  // This is for receiving the second message
00983                              recv_ent_reqs[3 * ind + 2],  // This would be for ack, but it is not
00984                                                           // used; consider removing it
00985                              incoming, localOwnedBuffs[ind],
00986                              sendReqs[3 * ind + 1],  // Send request for sending the second message
00987                              sendReqs[3 * ind + 2],  // This is for sending the ack
00988                              done );MB_CHK_SET_ERR( error, "Failed to resize recv buffer" );
00989 
00990         if( done )
00991         {
00992             remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
00993 
00994             int from_proc = status.MPI_SOURCE;
00995             int idx       = std::find( send_procs.begin(), send_procs.end(), from_proc ) - send_procs.begin();
00996 
00997             int msg = msgsizes[idx].size();
00998             std::vector< int > recvmsg( msg );
00999             int ndata = senddata[idx].size();
01000             std::vector< EntityHandle > dum_vec( ndata );
01001 
01002             UNPACK_INTS( remoteOwnedBuffs[ind]->buff_ptr, &recvmsg[0], msg );
01003             UNPACK_EH( remoteOwnedBuffs[ind]->buff_ptr, &dum_vec[0], ndata );
01004 
01005             recvdata[idx].insert( recvdata[idx].end(), dum_vec.begin(), dum_vec.end() );
01006         }
01007     }
01008 
01009 #ifdef USE_MPE
01010     if( myDebug->get_verbosity() == 2 )
01011     {
01012         MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_recv_entities." );
01013     }
01014 #endif
01015 
01016     return MB_SUCCESS;
01017 }
01018 
01019 ErrorCode ParallelComm::update_remote_data( EntityHandle entity,
01020                                             std::vector< int >& procs,
01021                                             std::vector< EntityHandle >& handles )
01022 {
01023     ErrorCode error;
01024     unsigned char pstatus = PSTATUS_INTERFACE;
01025 
01026     int procmin = *std::min_element( procs.begin(), procs.end() );
01027 
01028     if( (int)rank() > procmin )
01029         pstatus |= PSTATUS_NOT_OWNED;
01030     else
01031         procmin = rank();
01032 
01033     // DBG
01034     // std::cout<<"entity = "<<entity<<std::endl;
01035     // for (int j=0; j<procs.size(); j++)
01036     // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl;
01037     // DBG
01038 
01039     if( (int)procs.size() > 1 )
01040     {
01041         procs.push_back( rank() );
01042         handles.push_back( entity );
01043 
01044         int idx = std::find( procs.begin(), procs.end(), procmin ) - procs.begin();
01045 
01046         std::iter_swap( procs.begin(), procs.begin() + idx );
01047         std::iter_swap( handles.begin(), handles.begin() + idx );
01048 
01049         // DBG
01050         //  std::cout<<"entity = "<<entity<<std::endl;
01051         // for (int j=0; j<procs.size(); j++)
01052         // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl;
01053         // DBG
01054     }
01055 
01056     // if ((entity == 10388) && (rank()==1))
01057     //    std::cout<<"Here"<<std::endl;
01058 
01059     error = update_remote_data( entity, &procs[0], &handles[0], procs.size(), pstatus );MB_CHK_ERR( error );
01060 
01061     return MB_SUCCESS;
01062 }
01063 
01064 ErrorCode ParallelComm::get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc )
01065 {
01066     ErrorCode error;
01067     std::vector< EntityHandle > newents;
01068     error = get_remote_handles( true, local_vec, rem_vec, num_ents, to_proc, newents );MB_CHK_ERR( error );
01069 
01070     return MB_SUCCESS;
01071 }
01072 
01073 //////////////////////////////////////////////////////////////////
01074 
01075 ErrorCode ParallelComm::recv_entities( const int from_proc,
01076                                        const bool store_remote_handles,
01077                                        const bool is_iface,
01078                                        Range& final_ents,
01079                                        int& incoming1,
01080                                        int& incoming2,
01081                                        std::vector< std::vector< EntityHandle > >& L1hloc,
01082                                        std::vector< std::vector< EntityHandle > >& L1hrem,
01083                                        std::vector< std::vector< int > >& L1p,
01084                                        std::vector< EntityHandle >& L2hloc,
01085                                        std::vector< EntityHandle >& L2hrem,
01086                                        std::vector< unsigned int >& L2p,
01087                                        std::vector< MPI_Request >& recv_remoteh_reqs,
01088                                        bool /*wait_all*/ )
01089 {
01090 #ifndef MOAB_HAVE_MPI
01091     return MB_FAILURE;
01092 #else
01093     // Non-blocking receive for the first message (having size info)
01094     int ind1 = get_buffers( from_proc );
01095     incoming1++;
01096     PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE,
01097                        MB_MESG_ENTS_SIZE, incoming1 );
01098     int success = MPI_Irecv( remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc,
01099                              MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind1] );
01100     if( success != MPI_SUCCESS )
01101     {
01102         MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
01103     }
01104 
01105     // Receive messages in while loop
01106     return recv_messages( from_proc, store_remote_handles, is_iface, final_ents, incoming1, incoming2, L1hloc, L1hrem,
01107                           L1p, L2hloc, L2hrem, L2p, recv_remoteh_reqs );
01108 #endif
01109 }
01110 
01111 ErrorCode ParallelComm::recv_entities( std::set< unsigned int >& recv_procs,
01112                                        int incoming1,
01113                                        int incoming2,
01114                                        const bool store_remote_handles,
01115                                        const bool migrate )
01116 {
01117     //===========================================
01118     // Receive/unpack new entities
01119     //===========================================
01120     // Number of incoming messages is the number of procs we communicate with
01121     int success, ind, i;
01122     ErrorCode result;
01123     MPI_Status status;
01124     std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
01125     std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
01126     std::vector< std::vector< int > > L1p( buffProcs.size() );
01127     std::vector< EntityHandle > L2hloc, L2hrem;
01128     std::vector< unsigned int > L2p;
01129     std::vector< EntityHandle > new_ents;
01130 
01131     while( incoming1 )
01132     {
01133         // Wait for all recvs of ents before proceeding to sending remote handles,
01134         // b/c some procs may have sent to a 3rd proc ents owned by me;
01135         PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
01136 
01137         success = MPI_Waitany( 2 * buffProcs.size(), &recvReqs[0], &ind, &status );
01138         if( MPI_SUCCESS != success )
01139         {
01140             MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
01141         }
01142 
01143         PRINT_DEBUG_RECD( status );
01144 
01145         // OK, received something; decrement incoming counter
01146         incoming1--;
01147         bool done = false;
01148 
01149         // In case ind is for ack, we need index of one before it
01150         unsigned int base_ind = 2 * ( ind / 2 );
01151         result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 2], recvReqs[ind], recvReqs[ind + 1],
01152                               incoming1, localOwnedBuffs[ind / 2], sendReqs[base_ind], sendReqs[base_ind + 1], done,
01153                               ( store_remote_handles ? localOwnedBuffs[ind / 2] : NULL ), MB_MESG_REMOTEH_SIZE,
01154                               &recvRemotehReqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
01155 
01156         if( done )
01157         {
01158             if( myDebug->get_verbosity() == 4 )
01159             {
01160                 msgs.resize( msgs.size() + 1 );
01161                 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 2] );
01162             }
01163 
01164             // Message completely received - process buffer that was sent
01165             remoteOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
01166             result = unpack_buffer( remoteOwnedBuffs[ind / 2]->buff_ptr, store_remote_handles, buffProcs[ind / 2],
01167                                     ind / 2, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true );
01168             if( MB_SUCCESS != result )
01169             {
01170                 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
01171                 print_buffer( remoteOwnedBuffs[ind / 2]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 2], false );
01172                 return result;
01173             }
01174 
01175             if( recvReqs.size() != 2 * buffProcs.size() )
01176             {
01177                 // Post irecv's for remote handles from new proc
01178                 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
01179                 for( i = recvReqs.size(); i < (int)( 2 * buffProcs.size() ); i += 2 )
01180                 {
01181                     localOwnedBuffs[i / 2]->reset_buffer();
01182                     incoming2++;
01183                     PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 2], localOwnedBuffs[i / 2]->mem_ptr,
01184                                        INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
01185                     success = MPI_Irecv( localOwnedBuffs[i / 2]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
01186                                          buffProcs[i / 2], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
01187                                          &recvRemotehReqs[i] );
01188                     if( success != MPI_SUCCESS )
01189                     {
01190                         MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
01191                     }
01192                 }
01193                 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
01194                 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
01195             }
01196         }
01197     }
01198 
01199     // Assign and remove newly created elements from/to receive processor
01200     result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" );
01201     if( migrate )
01202     {
01203         // result = remove_entities_part(allsent, procConfig.proc_rank());MB_CHK_SET_ERR(ressult,
01204         // "Failed to remove entities to part");
01205     }
01206 
01207     // Add requests for any new addl procs
01208     if( recvReqs.size() != 2 * buffProcs.size() )
01209     {
01210         // Shouldn't get here...
01211         MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" );
01212     }
01213 
01214 #ifdef MOAB_HAVE_MPE
01215     if( myDebug->get_verbosity() == 2 )
01216     {
01217         MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending recv entities." );
01218     }
01219 #endif
01220 
01221     //===========================================
01222     // Send local handles for new entity to owner
01223     //===========================================
01224     std::set< unsigned int >::iterator it  = recv_procs.begin();
01225     std::set< unsigned int >::iterator eit = recv_procs.end();
01226     for( ; it != eit; ++it )
01227     {
01228         ind = get_buffers( *it );
01229         // Reserve space on front for size and for initial buff size
01230         remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
01231 
01232         result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
01233         remoteOwnedBuffs[ind]->set_stored_size();
01234 
01235         if( myDebug->get_verbosity() == 4 )
01236         {
01237             msgs.resize( msgs.size() + 1 );
01238             msgs.back() = new Buffer( *remoteOwnedBuffs[ind] );
01239         }
01240         result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[2 * ind],
01241                               recvRemotehReqs[2 * ind + 1], &ackbuff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
01242     }
01243 
01244     //===========================================
01245     // Process remote handles of my ghosteds
01246     //===========================================
01247     while( incoming2 )
01248     {
01249         PRINT_DEBUG_WAITANY( recvRemotehReqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
01250         success = MPI_Waitany( 2 * buffProcs.size(), &recvRemotehReqs[0], &ind, &status );
01251         if( MPI_SUCCESS != success )
01252         {
01253             MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
01254         }
01255 
01256         // OK, received something; decrement incoming counter
01257         incoming2--;
01258 
01259         PRINT_DEBUG_RECD( status );
01260         bool done             = false;
01261         unsigned int base_ind = 2 * ( ind / 2 );
01262         result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 2], recvRemotehReqs[ind],
01263                               recvRemotehReqs[ind + 1], incoming2, remoteOwnedBuffs[ind / 2], sendReqs[base_ind],
01264                               sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
01265         if( done )
01266         {
01267             // Incoming remote handles
01268             if( myDebug->get_verbosity() == 4 )
01269             {
01270                 msgs.resize( msgs.size() + 1 );
01271                 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
01272             }
01273 
01274             localOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
01275             result =
01276                 unpack_remote_handles( buffProcs[ind / 2], localOwnedBuffs[ind / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
01277         }
01278     }
01279 
01280 #ifdef MOAB_HAVE_MPE
01281     if( myDebug->get_verbosity() == 2 )
01282     {
01283         MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
01284         MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending recv entities (still doing checks)." );
01285     }
01286 #endif
01287     myDebug->tprintf( 1, "Exiting recv_entities.\n" );
01288 
01289     return MB_SUCCESS;
01290 }
01291 
01292 ErrorCode ParallelComm::recv_messages( const int from_proc,
01293                                        const bool store_remote_handles,
01294                                        const bool is_iface,
01295                                        Range& final_ents,
01296                                        int& incoming1,
01297                                        int& incoming2,
01298                                        std::vector< std::vector< EntityHandle > >& L1hloc,
01299                                        std::vector< std::vector< EntityHandle > >& L1hrem,
01300                                        std::vector< std::vector< int > >& L1p,
01301                                        std::vector< EntityHandle >& L2hloc,
01302                                        std::vector< EntityHandle >& L2hrem,
01303                                        std::vector< unsigned int >& L2p,
01304                                        std::vector< MPI_Request >& recv_remoteh_reqs )
01305 {
01306 #ifndef MOAB_HAVE_MPI
01307     return MB_FAILURE;
01308 #else
01309     MPI_Status status;
01310     ErrorCode result;
01311     int ind1 = get_buffers( from_proc );
01312     int success, ind2;
01313     std::vector< EntityHandle > new_ents;
01314 
01315     // Wait and receive messages
01316     while( incoming1 )
01317     {
01318         PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
01319         success = MPI_Waitany( 2, &recvReqs[2 * ind1], &ind2, &status );
01320         if( MPI_SUCCESS != success )
01321         {
01322             MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_messages" );
01323         }
01324 
01325         PRINT_DEBUG_RECD( status );
01326 
01327         // OK, received something; decrement incoming counter
01328         incoming1--;
01329         bool done = false;
01330 
01331         // In case ind is for ack, we need index of one before it
01332         ind2 += 2 * ind1;
01333         unsigned int base_ind = 2 * ( ind2 / 2 );
01334 
01335         result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind2 / 2],
01336                               // recvbuff,
01337                               recvReqs[ind2], recvReqs[ind2 + 1], incoming1, localOwnedBuffs[ind2 / 2],
01338                               sendReqs[base_ind], sendReqs[base_ind + 1], done,
01339                               ( !is_iface && store_remote_handles ? localOwnedBuffs[ind2 / 2] : NULL ),
01340                               MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
01341 
01342         if( done )
01343         {
01344             // If it is done, unpack buffer
01345             remoteOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
01346             result = unpack_buffer( remoteOwnedBuffs[ind2 / 2]->buff_ptr, store_remote_handles, from_proc, ind2 / 2,
01347                                     L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );MB_CHK_SET_ERR( result, "Failed to unpack buffer in recev_messages" );
01348 
01349             std::copy( new_ents.begin(), new_ents.end(), range_inserter( final_ents ) );
01350 
01351             // Send local handles for new elements to owner
01352             // Reserve space on front for size and for initial buff size
01353             remoteOwnedBuffs[ind2 / 2]->reset_buffer( sizeof( int ) );
01354 
01355             result = pack_remote_handles( L1hloc[ind2 / 2], L1hrem[ind2 / 2], L1p[ind2 / 2], from_proc,
01356                                           remoteOwnedBuffs[ind2 / 2] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
01357             remoteOwnedBuffs[ind2 / 2]->set_stored_size();
01358 
01359             result = send_buffer( buffProcs[ind2 / 2], remoteOwnedBuffs[ind2 / 2], MB_MESG_REMOTEH_SIZE, sendReqs[ind2],
01360                                   recv_remoteh_reqs[ind2 + 1], (int*)( localOwnedBuffs[ind2 / 2]->mem_ptr ),
01361                                   //&ackbuff,
01362                                   incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
01363         }
01364     }
01365 
01366     return MB_SUCCESS;
01367 #endif
01368 }
01369 
01370 ErrorCode ParallelComm::recv_remote_handle_messages( const int from_proc,
01371                                                      int& incoming2,
01372                                                      std::vector< EntityHandle >& L2hloc,
01373                                                      std::vector< EntityHandle >& L2hrem,
01374                                                      std::vector< unsigned int >& L2p,
01375                                                      std::vector< MPI_Request >& recv_remoteh_reqs )
01376 {
01377 #ifndef MOAB_HAVE_MPI
01378     return MB_FAILURE;
01379 #else
01380     MPI_Status status;
01381     ErrorCode result;
01382     int ind1 = get_buffers( from_proc );
01383     int success, ind2;
01384 
01385     while( incoming2 )
01386     {
01387         PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
01388         success = MPI_Waitany( 2, &recv_remoteh_reqs[2 * ind1], &ind2, &status );
01389         if( MPI_SUCCESS != success )
01390         {
01391             MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_remote_handle_messages" );
01392         }
01393 
01394         // OK, received something; decrement incoming counter
01395         incoming2--;
01396 
01397         PRINT_DEBUG_RECD( status );
01398 
01399         bool done = false;
01400         ind2 += 2 * ind1;
01401         unsigned int base_ind = 2 * ( ind2 / 2 );
01402         result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind2 / 2], recv_remoteh_reqs[ind2],
01403                               recv_remoteh_reqs[ind2 + 1], incoming2, remoteOwnedBuffs[ind2 / 2], sendReqs[base_ind],
01404                               sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
01405         if( done )
01406         {
01407             // Incoming remote handles
01408             localOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
01409             result =
01410                 unpack_remote_handles( buffProcs[ind2 / 2], localOwnedBuffs[ind2 / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
01411         }
01412     }
01413 
01414     return MB_SUCCESS;
01415 #endif
01416 }
01417 
01418 ErrorCode ParallelComm::pack_buffer( Range& orig_ents,
01419                                      const bool /*adjacencies*/,
01420                                      const bool tags,
01421                                      const bool store_remote_handles,
01422                                      const int to_proc,
01423                                      Buffer* buff,
01424                                      TupleList* entprocs,
01425                                      Range* allsent )
01426 {
01427     // Pack the buffer with the entity ranges, adjacencies, and tags sections
01428     //
01429     // Note: new entities used in subsequent connectivity lists, sets, or tags,
01430     // are referred to as (MBMAXTYPE + index), where index is into vector
01431     // of new entities, 0-based
01432     ErrorCode result;
01433 
01434     Range set_range;
01435     std::vector< Tag > all_tags;
01436     std::vector< Range > tag_ranges;
01437 
01438     Range::const_iterator rit;
01439 
01440     // Entities
01441     result = pack_entities( orig_ents, buff, store_remote_handles, to_proc, false, entprocs, allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
01442 
01443     // Sets
01444     result = pack_sets( orig_ents, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing sets (count) failed" );
01445 
01446     // Tags
01447     Range final_ents;
01448     if( tags )
01449     {
01450         result = get_tag_send_list( orig_ents, all_tags, tag_ranges );MB_CHK_SET_ERR( result, "Failed to get tagged entities" );
01451         result = pack_tags( orig_ents, all_tags, all_tags, tag_ranges, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing tags (count) failed" );
01452     }
01453     else
01454     {  // Set tag size to 0
01455         buff->check_space( sizeof( int ) );
01456         PACK_INT( buff->buff_ptr, 0 );
01457         buff->set_stored_size();
01458     }
01459 
01460     return result;
01461 }
01462 
01463 ErrorCode ParallelComm::unpack_buffer( unsigned char* buff_ptr,
01464                                        const bool store_remote_handles,
01465                                        const int from_proc,
01466                                        const int ind,
01467                                        std::vector< std::vector< EntityHandle > >& L1hloc,
01468                                        std::vector< std::vector< EntityHandle > >& L1hrem,
01469                                        std::vector< std::vector< int > >& L1p,
01470                                        std::vector< EntityHandle >& L2hloc,
01471                                        std::vector< EntityHandle >& L2hrem,
01472                                        std::vector< unsigned int >& L2p,
01473                                        std::vector< EntityHandle >& new_ents,
01474                                        const bool created_iface )
01475 {
01476     unsigned char* tmp_buff = buff_ptr;
01477     ErrorCode result;
01478     result = unpack_entities( buff_ptr, store_remote_handles, ind, false, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p,
01479                               new_ents, created_iface );MB_CHK_SET_ERR( result, "Unpacking entities failed" );
01480     if( myDebug->get_verbosity() == 3 )
01481     {
01482         myDebug->tprintf( 4, "unpack_entities buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
01483         tmp_buff = buff_ptr;
01484     }
01485     result = unpack_sets( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking sets failed" );
01486     if( myDebug->get_verbosity() == 3 )
01487     {
01488         myDebug->tprintf( 4, "unpack_sets buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
01489         tmp_buff = buff_ptr;
01490     }
01491     result = unpack_tags( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking tags failed" );
01492     if( myDebug->get_verbosity() == 3 )
01493     {
01494         myDebug->tprintf( 4, "unpack_tags buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
01495         // tmp_buff = buff_ptr;
01496     }
01497 
01498     if( myDebug->get_verbosity() == 3 ) myDebug->print( 4, "\n" );
01499 
01500     return MB_SUCCESS;
01501 }
01502 
01503 int ParallelComm::estimate_ents_buffer_size( Range& entities, const bool store_remote_handles )
01504 {
01505     int buff_size = 0;
01506     std::vector< EntityHandle > dum_connect_vec;
01507     const EntityHandle* connect;
01508     int num_connect;
01509 
01510     int num_verts = entities.num_of_type( MBVERTEX );
01511     // # verts + coords + handles
01512     buff_size += 2 * sizeof( int ) + 3 * sizeof( double ) * num_verts;
01513     if( store_remote_handles ) buff_size += sizeof( EntityHandle ) * num_verts;
01514 
01515     // Do a rough count by looking at first entity of each type
01516     for( EntityType t = MBEDGE; t < MBENTITYSET; t++ )
01517     {
01518         const Range::iterator rit = entities.lower_bound( t );
01519         if( TYPE_FROM_HANDLE( *rit ) != t ) continue;
01520 
01521         ErrorCode result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect_vec );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get connectivity to estimate buffer size", -1 );
01522 
01523         // Number, type, nodes per entity
01524         buff_size += 3 * sizeof( int );
01525         int num_ents = entities.num_of_type( t );
01526         // Connectivity, handle for each ent
01527         buff_size += ( num_connect + 1 ) * sizeof( EntityHandle ) * num_ents;
01528     }
01529 
01530     // Extra entity type at end, passed as int
01531     buff_size += sizeof( int );
01532 
01533     return buff_size;
01534 }
01535 
01536 int ParallelComm::estimate_sets_buffer_size( Range& entities, const bool /*store_remote_handles*/ )
01537 {
01538     // Number of sets
01539     int buff_size = sizeof( int );
01540 
01541     // Do a rough count by looking at first entity of each type
01542     Range::iterator rit = entities.lower_bound( MBENTITYSET );
01543     ErrorCode result;
01544 
01545     for( ; rit != entities.end(); ++rit )
01546     {
01547         unsigned int options;
01548         result = mbImpl->get_meshset_options( *rit, options );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get meshset options", -1 );
01549 
01550         buff_size += sizeof( int );
01551 
01552         Range set_range;
01553         if( options & MESHSET_SET )
01554         {
01555             // Range-based set; count the subranges
01556             result = mbImpl->get_entities_by_handle( *rit, set_range );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get set entities", -1 );
01557 
01558             // Set range
01559             buff_size += RANGE_SIZE( set_range );
01560         }
01561         else if( options & MESHSET_ORDERED )
01562         {
01563             // Just get the number of entities in the set
01564             int num_ents;
01565             result = mbImpl->get_number_entities_by_handle( *rit, num_ents );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get number entities in ordered set", -1 );
01566 
01567             // Set vec
01568             buff_size += sizeof( EntityHandle ) * num_ents + sizeof( int );
01569         }
01570 
01571         // Get numbers of parents/children
01572         int num_par, num_ch;
01573         result = mbImpl->num_child_meshsets( *rit, &num_ch );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num children", -1 );
01574         result = mbImpl->num_parent_meshsets( *rit, &num_par );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num parents", -1 );
01575 
01576         buff_size += ( num_ch + num_par ) * sizeof( EntityHandle ) + 2 * sizeof( int );
01577     }
01578 
01579     return buff_size;
01580 }
01581 
01582 ErrorCode ParallelComm::pack_entities( Range& entities,
01583                                        Buffer* buff,
01584                                        const bool store_remote_handles,
01585                                        const int to_proc,
01586                                        const bool /*is_iface*/,
01587                                        TupleList* entprocs,
01588                                        Range* /*allsent*/ )
01589 {
01590     // Packed information:
01591     // 1. # entities = E
01592     // 2. for e in E
01593     //   a. # procs sharing e, incl. sender and receiver = P
01594     //   b. for p in P (procs sharing e)
01595     //   c. for p in P (handle for e on p) (Note1)
01596     // 3. vertex/entity info
01597 
01598     // Get an estimate of the buffer size & pre-allocate buffer size
01599     int buff_size = estimate_ents_buffer_size( entities, store_remote_handles );
01600     if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate ents buffer size" );
01601     buff->check_space( buff_size );
01602     myDebug->tprintf( 3, "estimate buffer size for %d entities: %d \n", (int)entities.size(), buff_size );
01603 
01604     unsigned int num_ents;
01605     ErrorCode result;
01606 
01607     std::vector< EntityHandle > entities_vec( entities.size() );
01608     std::copy( entities.begin(), entities.end(), entities_vec.begin() );
01609 
01610     // First pack procs/handles sharing this ent, not including this dest but including
01611     // others (with zero handles)
01612     if( store_remote_handles )
01613     {
01614         // Buff space is at least proc + handle for each entity; use avg of 4 other procs
01615         // to estimate buff size, but check later
01616         buff->check_space( sizeof( int ) + ( 5 * sizeof( int ) + sizeof( EntityHandle ) ) * entities.size() );
01617 
01618         // 1. # entities = E
01619         PACK_INT( buff->buff_ptr, entities.size() );
01620 
01621         Range::iterator rit;
01622 
01623         // Pre-fetch sharedp and pstatus
01624         std::vector< int > sharedp_vals( entities.size() );
01625         result = mbImpl->tag_get_data( sharedp_tag(), entities, &sharedp_vals[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
01626         std::vector< char > pstatus_vals( entities.size() );
01627         result = mbImpl->tag_get_data( pstatus_tag(), entities, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
01628 
01629         unsigned int i;
01630         int tmp_procs[MAX_SHARING_PROCS];
01631         EntityHandle tmp_handles[MAX_SHARING_PROCS];
01632         std::set< unsigned int > dumprocs;
01633 
01634         // 2. for e in E
01635         for( rit = entities.begin(), i = 0; rit != entities.end(); ++rit, i++ )
01636         {
01637             unsigned int ind =
01638                 std::lower_bound( entprocs->vul_rd, entprocs->vul_rd + entprocs->get_n(), *rit ) - entprocs->vul_rd;
01639             assert( ind < entprocs->get_n() );
01640 
01641             while( ind < entprocs->get_n() && entprocs->vul_rd[ind] == *rit )
01642                 dumprocs.insert( entprocs->vi_rd[ind++] );
01643 
01644             result = build_sharedhps_list( *rit, pstatus_vals[i], sharedp_vals[i], dumprocs, num_ents, tmp_procs,
01645                                            tmp_handles );MB_CHK_SET_ERR( result, "Failed to build sharedhps" );
01646 
01647             dumprocs.clear();
01648 
01649             // Now pack them
01650             buff->check_space( ( num_ents + 1 ) * sizeof( int ) + num_ents * sizeof( EntityHandle ) );
01651             PACK_INT( buff->buff_ptr, num_ents );
01652             PACK_INTS( buff->buff_ptr, tmp_procs, num_ents );
01653             PACK_EH( buff->buff_ptr, tmp_handles, num_ents );
01654 
01655 #ifndef NDEBUG
01656             // Check for duplicates in proc list
01657             unsigned int dp = 0;
01658             for( ; dp < MAX_SHARING_PROCS && -1 != tmp_procs[dp]; dp++ )
01659                 dumprocs.insert( tmp_procs[dp] );
01660             assert( dumprocs.size() == dp );
01661             dumprocs.clear();
01662 #endif
01663         }
01664     }
01665 
01666     // Pack vertices
01667     Range these_ents = entities.subset_by_type( MBVERTEX );
01668     num_ents         = these_ents.size();
01669 
01670     if( num_ents )
01671     {
01672         buff_size = 2 * sizeof( int ) + 3 * num_ents * sizeof( double );
01673         buff->check_space( buff_size );
01674 
01675         // Type, # ents
01676         PACK_INT( buff->buff_ptr, ( (int)MBVERTEX ) );
01677         PACK_INT( buff->buff_ptr, ( (int)num_ents ) );
01678 
01679         std::vector< double > tmp_coords( 3 * num_ents );
01680         result = mbImpl->get_coords( these_ents, &tmp_coords[0] );MB_CHK_SET_ERR( result, "Failed to get vertex coordinates" );
01681         PACK_DBLS( buff->buff_ptr, &tmp_coords[0], 3 * num_ents );
01682 
01683         myDebug->tprintf( 4, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
01684                           CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
01685     }
01686 
01687     // Now entities; go through range, packing by type and equal # verts per element
01688     Range::iterator start_rit = entities.find( *these_ents.rbegin() );
01689     ++start_rit;
01690     int last_nodes       = -1;
01691     EntityType last_type = MBMAXTYPE;
01692     these_ents.clear();
01693     Range::iterator end_rit = start_rit;
01694     EntitySequence* seq;
01695     ElementSequence* eseq;
01696 
01697     while( start_rit != entities.end() || !these_ents.empty() )
01698     {
01699         // Cases:
01700         // A: !end, last_type == MBMAXTYPE, seq: save contig sequence in these_ents
01701         // B: !end, last type & nodes same, seq: save contig sequence in these_ents
01702         // C: !end, last type & nodes different: pack these_ents, then save contig sequence in
01703         // these_ents D: end: pack these_ents
01704 
01705         // Find the sequence holding current start entity, if we're not at end
01706         eseq = NULL;
01707         if( start_rit != entities.end() )
01708         {
01709             result = sequenceManager->find( *start_rit, seq );MB_CHK_SET_ERR( result, "Failed to find entity sequence" );
01710             if( NULL == seq ) return MB_FAILURE;
01711             eseq = dynamic_cast< ElementSequence* >( seq );
01712         }
01713 
01714         // Pack the last batch if at end or next one is different
01715         if( !these_ents.empty() &&
01716             ( !eseq || eseq->type() != last_type || last_nodes != (int)eseq->nodes_per_element() ) )
01717         {
01718             result = pack_entity_seq( last_nodes, store_remote_handles, to_proc, these_ents, entities_vec, buff );MB_CHK_SET_ERR( result, "Failed to pack entities from a sequence" );
01719             these_ents.clear();
01720         }
01721 
01722         if( eseq )
01723         {
01724             // Continuation of current range, just save these entities
01725             // Get position in entities list one past end of this sequence
01726             end_rit = entities.lower_bound( start_rit, entities.end(), eseq->end_handle() + 1 );
01727 
01728             // Put these entities in the range
01729             std::copy( start_rit, end_rit, range_inserter( these_ents ) );
01730 
01731             last_type  = eseq->type();
01732             last_nodes = eseq->nodes_per_element();
01733         }
01734         else if( start_rit != entities.end() && TYPE_FROM_HANDLE( *start_rit ) == MBENTITYSET )
01735             break;
01736 
01737         start_rit = end_rit;
01738     }
01739 
01740     // Pack MBMAXTYPE to indicate end of ranges
01741     buff->check_space( sizeof( int ) );
01742     PACK_INT( buff->buff_ptr, ( (int)MBMAXTYPE ) );
01743 
01744     buff->set_stored_size();
01745     return MB_SUCCESS;
01746 }
01747 
01748 ErrorCode ParallelComm::build_sharedhps_list( const EntityHandle entity,
01749                                               const unsigned char pstatus,
01750                                               const int
01751 #ifndef NDEBUG
01752                                                   sharedp
01753 #endif
01754                                               ,
01755                                               const std::set< unsigned int >& procs,
01756                                               unsigned int& num_ents,
01757                                               int* tmp_procs,
01758                                               EntityHandle* tmp_handles )
01759 {
01760     num_ents = 0;
01761     unsigned char pstat;
01762     ErrorCode result = get_sharing_data( entity, tmp_procs, tmp_handles, pstat, num_ents );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
01763     assert( pstat == pstatus );
01764 
01765     // Build shared proc/handle lists
01766     // Start with multi-shared, since if it is the owner will be first
01767     if( pstatus & PSTATUS_MULTISHARED )
01768     {
01769     }
01770     else if( pstatus & PSTATUS_NOT_OWNED )
01771     {
01772         // If not multishared and not owned, other sharing proc is owner, put that
01773         // one first
01774         assert( "If not owned, I should be shared too" && pstatus & PSTATUS_SHARED && 1 == num_ents );
01775         tmp_procs[1]   = procConfig.proc_rank();
01776         tmp_handles[1] = entity;
01777         num_ents       = 2;
01778     }
01779     else if( pstatus & PSTATUS_SHARED )
01780     {
01781         // If not multishared and owned, I'm owner
01782         assert( "shared and owned, should be only 1 sharing proc" && 1 == num_ents );
01783         tmp_procs[1]   = tmp_procs[0];
01784         tmp_procs[0]   = procConfig.proc_rank();
01785         tmp_handles[1] = tmp_handles[0];
01786         tmp_handles[0] = entity;
01787         num_ents       = 2;
01788     }
01789     else
01790     {
01791         // Not shared yet, just add owner (me)
01792         tmp_procs[0]   = procConfig.proc_rank();
01793         tmp_handles[0] = entity;
01794         num_ents       = 1;
01795     }
01796 
01797 #ifndef NDEBUG
01798     int tmp_ps = num_ents;
01799 #endif
01800 
01801     // Now add others, with zero handle for now
01802     for( std::set< unsigned int >::iterator sit = procs.begin(); sit != procs.end(); ++sit )
01803     {
01804 #ifndef NDEBUG
01805         if( tmp_ps && std::find( tmp_procs, tmp_procs + tmp_ps, *sit ) != tmp_procs + tmp_ps )
01806         {
01807             std::cerr << "Trouble with something already in shared list on proc " << procConfig.proc_rank()
01808                       << ". Entity:" << std::endl;
01809             list_entities( &entity, 1 );
01810             std::cerr << "pstatus = " << (int)pstatus << ", sharedp = " << sharedp << std::endl;
01811             std::cerr << "tmp_ps = ";
01812             for( int i = 0; i < tmp_ps; i++ )
01813                 std::cerr << tmp_procs[i] << " ";
01814             std::cerr << std::endl;
01815             std::cerr << "procs = ";
01816             for( std::set< unsigned int >::iterator sit2 = procs.begin(); sit2 != procs.end(); ++sit2 )
01817                 std::cerr << *sit2 << " ";
01818             assert( false );
01819         }
01820 #endif
01821         tmp_procs[num_ents]   = *sit;
01822         tmp_handles[num_ents] = 0;
01823         num_ents++;
01824     }
01825 
01826     // Put -1 after procs and 0 after handles
01827     if( MAX_SHARING_PROCS > num_ents )
01828     {
01829         tmp_procs[num_ents]   = -1;
01830         tmp_handles[num_ents] = 0;
01831     }
01832 
01833     return MB_SUCCESS;
01834 }
01835 
01836 ErrorCode ParallelComm::pack_entity_seq( const int nodes_per_entity,
01837                                          const bool store_remote_handles,
01838                                          const int to_proc,
01839                                          Range& these_ents,
01840                                          std::vector< EntityHandle >& entities_vec,
01841                                          Buffer* buff )
01842 {
01843     int tmp_space = 3 * sizeof( int ) + nodes_per_entity * these_ents.size() * sizeof( EntityHandle );
01844     buff->check_space( tmp_space );
01845 
01846     // Pack the entity type
01847     PACK_INT( buff->buff_ptr, ( (int)TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
01848 
01849     // Pack # ents
01850     PACK_INT( buff->buff_ptr, these_ents.size() );
01851 
01852     // Pack the nodes per entity
01853     PACK_INT( buff->buff_ptr, nodes_per_entity );
01854     myDebug->tprintf( 3, "after some pack int  %d \n", buff->get_current_size() );
01855 
01856     // Pack the connectivity
01857     std::vector< EntityHandle > connect;
01858     ErrorCode result = MB_SUCCESS;
01859     for( Range::const_iterator rit = these_ents.begin(); rit != these_ents.end(); ++rit )
01860     {
01861         connect.clear();
01862         result = mbImpl->get_connectivity( &( *rit ), 1, connect, false );MB_CHK_SET_ERR( result, "Failed to get connectivity" );
01863         assert( (int)connect.size() == nodes_per_entity );
01864         result =
01865             get_remote_handles( store_remote_handles, &connect[0], &connect[0], connect.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
01866         PACK_EH( buff->buff_ptr, &connect[0], connect.size() );
01867     }
01868 
01869     myDebug->tprintf( 3, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
01870                       CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
01871 
01872     return result;
01873 }
01874 
01875 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
01876                                             EntityHandle* from_vec,
01877                                             EntityHandle* to_vec_tmp,
01878                                             int num_ents,
01879                                             int to_proc,
01880                                             const std::vector< EntityHandle >& new_ents )
01881 {
01882     // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE RANGE-BASED VERSION, NO REUSE
01883     // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE
01884     // OTHER VERSION TOO!!!
01885     if( 0 == num_ents ) return MB_SUCCESS;
01886 
01887     // Use a local destination ptr in case we're doing an in-place copy
01888     std::vector< EntityHandle > tmp_vector;
01889     EntityHandle* to_vec = to_vec_tmp;
01890     if( to_vec == from_vec )
01891     {
01892         tmp_vector.resize( num_ents );
01893         to_vec = &tmp_vector[0];
01894     }
01895 
01896     if( !store_remote_handles )
01897     {
01898         int err;
01899         // In this case, substitute position in new_ents list
01900         for( int i = 0; i < num_ents; i++ )
01901         {
01902             int ind = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
01903             assert( new_ents[ind] == from_vec[i] );
01904             to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
01905             assert( to_vec[i] != 0 && !err && -1 != ind );
01906         }
01907     }
01908     else
01909     {
01910         Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
01911         ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
01912 
01913         // Get single-proc destination handles and shared procs
01914         std::vector< int > sharing_procs( num_ents );
01915         result = mbImpl->tag_get_data( shh_tag, from_vec, num_ents, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
01916         result = mbImpl->tag_get_data( shp_tag, from_vec, num_ents, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
01917         for( int j = 0; j < num_ents; j++ )
01918         {
01919             if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
01920         }
01921 
01922         EntityHandle tmp_handles[MAX_SHARING_PROCS];
01923         int tmp_procs[MAX_SHARING_PROCS];
01924         int i;
01925         // Go through results, and for 0-valued ones, look for multiple shared proc
01926         for( i = 0; i < num_ents; i++ )
01927         {
01928             if( !to_vec[i] )
01929             {
01930                 result = mbImpl->tag_get_data( shps_tag, from_vec + i, 1, tmp_procs );
01931                 if( MB_SUCCESS == result )
01932                 {
01933                     for( int j = 0; j < MAX_SHARING_PROCS; j++ )
01934                     {
01935                         if( -1 == tmp_procs[j] )
01936                             break;
01937                         else if( tmp_procs[j] == to_proc )
01938                         {
01939                             result = mbImpl->tag_get_data( shhs_tag, from_vec + i, 1, tmp_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
01940                             to_vec[i] = tmp_handles[j];
01941                             assert( to_vec[i] );
01942                             break;
01943                         }
01944                     }
01945                 }
01946                 if( !to_vec[i] )
01947                 {
01948                     int j = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
01949                     if( (int)new_ents.size() == j )
01950                     {
01951                         std::cout << "Failed to find new entity in send list, proc " << procConfig.proc_rank()
01952                                   << std::endl;
01953                         for( int k = 0; k <= num_ents; k++ )
01954                             std::cout << k << ": " << from_vec[k] << " " << to_vec[k] << std::endl;
01955                         MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
01956                     }
01957                     int err;
01958                     to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
01959                     if( err )
01960                     {
01961                         MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
01962                     }
01963                 }
01964             }
01965         }
01966     }
01967 
01968     // memcpy over results if from_vec and to_vec are the same
01969     if( to_vec_tmp == from_vec ) memcpy( from_vec, to_vec, num_ents * sizeof( EntityHandle ) );
01970 
01971     return MB_SUCCESS;
01972 }
01973 
01974 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
01975                                             const Range& from_range,
01976                                             EntityHandle* to_vec,
01977                                             int to_proc,
01978                                             const std::vector< EntityHandle >& new_ents )
01979 {
01980     // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE VECTOR-BASED VERSION, NO REUSE
01981     // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE
01982     // OTHER VERSION TOO!!!
01983     if( from_range.empty() ) return MB_SUCCESS;
01984 
01985     if( !store_remote_handles )
01986     {
01987         int err;
01988         // In this case, substitute position in new_ents list
01989         Range::iterator rit;
01990         unsigned int i;
01991         for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
01992         {
01993             int ind = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
01994             assert( new_ents[ind] == *rit );
01995             to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
01996             assert( to_vec[i] != 0 && !err && -1 != ind );
01997         }
01998     }
01999     else
02000     {
02001         Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
02002         ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
02003 
02004         // Get single-proc destination handles and shared procs
02005         std::vector< int > sharing_procs( from_range.size() );
02006         result = mbImpl->tag_get_data( shh_tag, from_range, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
02007         result = mbImpl->tag_get_data( shp_tag, from_range, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
02008         for( unsigned int j = 0; j < from_range.size(); j++ )
02009         {
02010             if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
02011         }
02012 
02013         EntityHandle tmp_handles[MAX_SHARING_PROCS];
02014         int tmp_procs[MAX_SHARING_PROCS];
02015         // Go through results, and for 0-valued ones, look for multiple shared proc
02016         Range::iterator rit;
02017         unsigned int i;
02018         for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
02019         {
02020             if( !to_vec[i] )
02021             {
02022                 result = mbImpl->tag_get_data( shhs_tag, &( *rit ), 1, tmp_handles );
02023                 if( MB_SUCCESS == result )
02024                 {
02025                     result = mbImpl->tag_get_data( shps_tag, &( *rit ), 1, tmp_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
02026                     for( int j = 0; j < MAX_SHARING_PROCS; j++ )
02027                         if( tmp_procs[j] == to_proc )
02028                         {
02029                             to_vec[i] = tmp_handles[j];
02030                             break;
02031                         }
02032                 }
02033 
02034                 if( !to_vec[i] )
02035                 {
02036                     int j = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
02037                     if( (int)new_ents.size() == j )
02038                     {
02039                         MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
02040                     }
02041                     int err;
02042                     to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
02043                     if( err )
02044                     {
02045                         MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
02046                     }
02047                 }
02048             }
02049         }
02050     }
02051 
02052     return MB_SUCCESS;
02053 }
02054 
02055 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
02056                                             const Range& from_range,
02057                                             Range& to_range,
02058                                             int to_proc,
02059                                             const std::vector< EntityHandle >& new_ents )
02060 {
02061     std::vector< EntityHandle > to_vector( from_range.size() );
02062 
02063     ErrorCode result = get_remote_handles( store_remote_handles, from_range, &to_vector[0], to_proc, new_ents );MB_CHK_SET_ERR( result, "Failed to get remote handles" );
02064     std::copy( to_vector.begin(), to_vector.end(), range_inserter( to_range ) );
02065     return result;
02066 }
02067 
02068 ErrorCode ParallelComm::unpack_entities( unsigned char*& buff_ptr,
02069                                          const bool store_remote_handles,
02070                                          const int /*from_ind*/,
02071                                          const bool is_iface,
02072                                          std::vector< std::vector< EntityHandle > >& L1hloc,
02073                                          std::vector< std::vector< EntityHandle > >& L1hrem,
02074                                          std::vector< std::vector< int > >& L1p,
02075                                          std::vector< EntityHandle >& L2hloc,
02076                                          std::vector< EntityHandle >& L2hrem,
02077                                          std::vector< unsigned int >& L2p,
02078                                          std::vector< EntityHandle >& new_ents,
02079                                          const bool created_iface )
02080 {
02081     // General algorithm:
02082     // - unpack # entities
02083     // - save start of remote handle info, then scan forward to entity definition data
02084     // - for all vertices or entities w/ same # verts:
02085     //   . get entity type, num ents, and (if !vert) # verts
02086     //   . for each ent:
02087     //      o get # procs/handles in remote handle info
02088     //      o if # procs/handles > 2, check for already-created entity:
02089     //        x get index of owner proc (1st in proc list), resize L1 list if nec
02090     //        x look for already-arrived entity in L2 by owner handle
02091     //      o if no existing entity:
02092     //        x if iface, look for existing entity with same connect & type
02093     //        x if none found, create vertex or element
02094     //        x if !iface & multi-shared, save on L2
02095     //        x if !iface, put new entity on new_ents list
02096     //      o update proc/handle, pstatus tags, adjusting to put owner first if iface
02097     //      o if !iface, save new handle on L1 for all sharing procs
02098 
02099     // Lists of handles/procs to return to sending/other procs
02100     // L1hloc[p], L1hrem[p]: handle pairs [h, h'], where h is the local proc handle
02101     //         and h' is either the remote proc handle (if that is known) or
02102     //         the owner proc handle (otherwise);
02103     // L1p[p]: indicates whether h is remote handle (= -1) or owner (rank of owner)
02104     // L2hloc, L2hrem: local/remote handles for entities shared by > 2 procs;
02105     //         remote handles are on owning proc
02106     // L2p: owning procs for handles in L2hrem
02107 
02108     ErrorCode result;
02109     bool done         = false;
02110     ReadUtilIface* ru = NULL;
02111 
02112     result = mbImpl->query_interface( ru );MB_CHK_SET_ERR( result, "Failed to get ReadUtilIface" );
02113 
02114     // 1. # entities = E
02115     int num_ents             = 0;
02116     unsigned char* buff_save = buff_ptr;
02117     int i, j;
02118 
02119     if( store_remote_handles )
02120     {
02121         UNPACK_INT( buff_ptr, num_ents );
02122 
02123         buff_save = buff_ptr;
02124 
02125         // Save place where remote handle info starts, then scan forward to ents
02126         for( i = 0; i < num_ents; i++ )
02127         {
02128             UNPACK_INT( buff_ptr, j );
02129             if( j < 0 )
02130             {
02131                 std::cout << "Should be non-negative # proc/handles.";
02132                 return MB_FAILURE;
02133             }
02134 
02135             buff_ptr += j * ( sizeof( int ) + sizeof( EntityHandle ) );
02136         }
02137     }
02138 
02139     std::vector< EntityHandle > msg_ents;
02140 
02141     while( !done )
02142     {
02143         EntityType this_type = MBMAXTYPE;
02144         UNPACK_TYPE( buff_ptr, this_type );
02145         assert( this_type != MBENTITYSET );
02146 
02147         // MBMAXTYPE signifies end of entities data
02148         if( MBMAXTYPE == this_type ) break;
02149 
02150         // Get the number of ents
02151         int num_ents2, verts_per_entity = 0;
02152         UNPACK_INT( buff_ptr, num_ents2 );
02153 
02154         // Unpack the nodes per entity
02155         if( MBVERTEX != this_type && num_ents2 )
02156         {
02157             UNPACK_INT( buff_ptr, verts_per_entity );
02158         }
02159 
02160         std::vector< int > ps( MAX_SHARING_PROCS, -1 );
02161         std::vector< EntityHandle > hs( MAX_SHARING_PROCS, 0 );
02162         for( int e = 0; e < num_ents2; e++ )
02163         {
02164             // Check for existing entity, otherwise make new one
02165             EntityHandle new_h = 0;
02166             EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
02167             double coords[3];
02168             int num_ps = -1;
02169 
02170             //=======================================
02171             // Unpack all the data at once, to make sure the buffer pointers
02172             // are tracked correctly
02173             //=======================================
02174             if( store_remote_handles )
02175             {
02176                 // Pointers to other procs/handles
02177                 UNPACK_INT( buff_save, num_ps );
02178                 if( 0 >= num_ps )
02179                 {
02180                     std::cout << "Shouldn't ever be fewer than 1 procs here." << std::endl;
02181                     return MB_FAILURE;
02182                 }
02183 
02184                 UNPACK_INTS( buff_save, &ps[0], num_ps );
02185                 UNPACK_EH( buff_save, &hs[0], num_ps );
02186             }
02187 
02188             if( MBVERTEX == this_type )
02189             {
02190                 UNPACK_DBLS( buff_ptr, coords, 3 );
02191             }
02192             else
02193             {
02194                 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
02195                 UNPACK_EH( buff_ptr, connect, verts_per_entity );
02196 
02197                 // Update connectivity to local handles
02198                 result = get_local_handles( connect, verts_per_entity, msg_ents );MB_CHK_SET_ERR( result, "Failed to get local handles" );
02199             }
02200 
02201             //=======================================
02202             // Now, process that data; begin by finding an identical
02203             // entity, if there is one
02204             //=======================================
02205             if( store_remote_handles )
02206             {
02207                 result = find_existing_entity( is_iface, ps[0], hs[0], num_ps, connect, verts_per_entity, this_type,
02208                                                L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
02209             }
02210 
02211             //=======================================
02212             // If we didn't find one, we'll have to create one
02213             //=======================================
02214             bool created_here = false;
02215             if( !new_h && !is_iface )
02216             {
02217                 if( MBVERTEX == this_type )
02218                 {
02219                     // Create a vertex
02220                     result = mbImpl->create_vertex( coords, new_h );MB_CHK_SET_ERR( result, "Failed to make new vertex" );
02221                 }
02222                 else
02223                 {
02224                     // Create the element
02225                     result = mbImpl->create_element( this_type, connect, verts_per_entity, new_h );MB_CHK_SET_ERR( result, "Failed to make new element" );
02226 
02227                     // Update adjacencies
02228                     result = ru->update_adjacencies( new_h, 1, verts_per_entity, connect );MB_CHK_SET_ERR( result, "Failed to update adjacencies" );
02229                 }
02230 
02231                 // Should have a new handle now
02232                 assert( new_h );
02233 
02234                 created_here = true;
02235             }
02236 
02237             //=======================================
02238             // Take care of sharing data
02239             //=======================================
02240 
02241             // Need to save entities found in order, for interpretation of
02242             // later parts of this message
02243             if( !is_iface )
02244             {
02245                 assert( new_h );
02246                 msg_ents.push_back( new_h );
02247             }
02248 
02249             if( created_here ) new_ents.push_back( new_h );
02250 
02251             if( new_h && store_remote_handles )
02252             {
02253                 unsigned char new_pstat = 0x0;
02254                 if( is_iface )
02255                 {
02256                     new_pstat = PSTATUS_INTERFACE;
02257                     // Here, lowest rank proc should be first
02258                     int idx = std::min_element( &ps[0], &ps[0] + num_ps ) - &ps[0];
02259                     if( idx )
02260                     {
02261                         std::swap( ps[0], ps[idx] );
02262                         std::swap( hs[0], hs[idx] );
02263                     }
02264                     // Set ownership based on lowest rank; can't be in update_remote_data, because
02265                     // there we don't know whether it resulted from ghosting or not
02266                     if( ( num_ps > 1 && ps[0] != (int)rank() ) ) new_pstat |= PSTATUS_NOT_OWNED;
02267                 }
02268                 else if( created_here )
02269                 {
02270                     if( created_iface )
02271                         new_pstat = PSTATUS_NOT_OWNED;
02272                     else
02273                         new_pstat = PSTATUS_GHOST | PSTATUS_NOT_OWNED;
02274                 }
02275 
02276                 // Update sharing data and pstatus, adjusting order if iface
02277                 result = update_remote_data( new_h, &ps[0], &hs[0], num_ps, new_pstat );MB_CHK_SET_ERR( result, "unpack_entities" );
02278 
02279                 // If a new multi-shared entity, save owner for subsequent lookup in L2 lists
02280                 if( store_remote_handles && !is_iface && num_ps > 2 )
02281                 {
02282                     L2hrem.push_back( hs[0] );
02283                     L2hloc.push_back( new_h );
02284                     L2p.push_back( ps[0] );
02285                 }
02286 
02287                 // Need to send this new handle to all sharing procs
02288                 if( !is_iface )
02289                 {
02290                     for( j = 0; j < num_ps; j++ )
02291                     {
02292                         if( ps[j] == (int)procConfig.proc_rank() ) continue;
02293                         int idx = get_buffers( ps[j] );
02294                         if( idx == (int)L1hloc.size() )
02295                         {
02296                             L1hloc.resize( idx + 1 );
02297                             L1hrem.resize( idx + 1 );
02298                             L1p.resize( idx + 1 );
02299                         }
02300 
02301                         // Don't bother adding if it's already in the list
02302                         std::vector< EntityHandle >::iterator vit =
02303                             std::find( L1hloc[idx].begin(), L1hloc[idx].end(), new_h );
02304                         if( vit != L1hloc[idx].end() )
02305                         {
02306                             // If it's in the list but remote handle isn't known but we know
02307                             // it, replace in the list
02308                             if( L1p[idx][vit - L1hloc[idx].begin()] != -1 && hs[j] )
02309                             {
02310                                 L1hrem[idx][vit - L1hloc[idx].begin()] = hs[j];
02311                                 L1p[idx][vit - L1hloc[idx].begin()]    = -1;
02312                             }
02313                             else
02314                                 continue;
02315                         }
02316                         else
02317                         {
02318                             if( !hs[j] )
02319                             {
02320                                 assert( -1 != ps[0] && num_ps > 2 );
02321                                 L1p[idx].push_back( ps[0] );
02322                                 L1hrem[idx].push_back( hs[0] );
02323                             }
02324                             else
02325                             {
02326                                 assert(
02327                                     "either this remote handle isn't in the remote list, or "
02328                                     "it's for another proc" &&
02329                                     ( std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) == L1hrem[idx].end() ||
02330                                       L1p[idx][std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) -
02331                                                L1hrem[idx].begin()] != -1 ) );
02332                                 L1p[idx].push_back( -1 );
02333                                 L1hrem[idx].push_back( hs[j] );
02334                             }
02335                             L1hloc[idx].push_back( new_h );
02336                         }
02337                     }
02338                 }
02339 
02340                 assert( "Shouldn't be here for non-shared entities" && -1 != num_ps );
02341                 std::fill( &ps[0], &ps[num_ps], -1 );
02342                 std::fill( &hs[0], &hs[num_ps], 0 );
02343             }
02344         }
02345 
02346         myDebug->tprintf( 4, "Unpacked %d ents of type %s", num_ents2, CN::EntityTypeName( this_type ) );
02347     }
02348 
02349     myDebug->tprintf( 4, "Done unpacking entities.\n" );
02350 
02351     // Need to sort here, to enable searching
02352     std::sort( new_ents.begin(), new_ents.end() );
02353 
02354     return MB_SUCCESS;
02355 }
02356 
02357 ErrorCode ParallelComm::print_buffer( unsigned char* buff_ptr, int mesg_tag, int from_proc, bool sent )
02358 {
02359     std::cerr << procConfig.proc_rank();
02360     if( sent )
02361         std::cerr << " sent";
02362     else
02363         std::cerr << " received";
02364     std::cerr << " message type " << mesg_tag << " to/from proc " << from_proc << "; contents:" << std::endl;
02365 
02366     int msg_length, num_ents;
02367     unsigned char* orig_ptr = buff_ptr;
02368     UNPACK_INT( buff_ptr, msg_length );
02369     std::cerr << msg_length << " bytes..." << std::endl;
02370 
02371     if( MB_MESG_ENTS_SIZE == mesg_tag || MB_MESG_ENTS_LARGE == mesg_tag )
02372     {
02373         // 1. # entities = E
02374         int i, j, k;
02375         std::vector< int > ps;
02376         std::vector< EntityHandle > hs;
02377 
02378         UNPACK_INT( buff_ptr, num_ents );
02379         std::cerr << num_ents << " entities..." << std::endl;
02380 
02381         // Save place where remote handle info starts, then scan forward to ents
02382         for( i = 0; i < num_ents; i++ )
02383         {
02384             UNPACK_INT( buff_ptr, j );
02385             if( 0 > j ) return MB_FAILURE;
02386             ps.resize( j );
02387             hs.resize( j );
02388             std::cerr << "Entity " << i << ", # procs = " << j << std::endl;
02389             UNPACK_INTS( buff_ptr, &ps[0], j );
02390             UNPACK_EH( buff_ptr, &hs[0], j );
02391             std::cerr << "   Procs: ";
02392             for( k = 0; k < j; k++ )
02393                 std::cerr << ps[k] << " ";
02394             std::cerr << std::endl;
02395             std::cerr << "   Handles: ";
02396             for( k = 0; k < j; k++ )
02397                 std::cerr << hs[k] << " ";
02398             std::cerr << std::endl;
02399 
02400             if( buff_ptr - orig_ptr > msg_length )
02401             {
02402                 std::cerr << "End of buffer..." << std::endl;
02403                 std::cerr.flush();
02404                 return MB_FAILURE;
02405             }
02406         }
02407 
02408         while( true )
02409         {
02410             EntityType this_type = MBMAXTYPE;
02411             UNPACK_TYPE( buff_ptr, this_type );
02412             assert( this_type != MBENTITYSET );
02413 
02414             // MBMAXTYPE signifies end of entities data
02415             if( MBMAXTYPE == this_type ) break;
02416 
02417             // Get the number of ents
02418             int num_ents2, verts_per_entity = 0;
02419             UNPACK_INT( buff_ptr, num_ents2 );
02420 
02421             // Unpack the nodes per entity
02422             if( MBVERTEX != this_type && num_ents2 )
02423             {
02424                 UNPACK_INT( buff_ptr, verts_per_entity );
02425             }
02426 
02427             std::cerr << "Type: " << CN::EntityTypeName( this_type ) << "; num_ents = " << num_ents2;
02428             if( MBVERTEX != this_type ) std::cerr << "; verts_per_ent = " << verts_per_entity;
02429             std::cerr << std::endl;
02430             if( num_ents2 < 0 || num_ents2 > msg_length )
02431             {
02432                 std::cerr << "Wrong number of entities, returning." << std::endl;
02433                 return MB_FAILURE;
02434             }
02435 
02436             for( int e = 0; e < num_ents2; e++ )
02437             {
02438                 // Check for existing entity, otherwise make new one
02439                 if( MBVERTEX == this_type )
02440                 {
02441                     double coords[3];
02442                     UNPACK_DBLS( buff_ptr, coords, 3 );
02443                     std::cerr << "xyz = " << coords[0] << ", " << coords[1] << ", " << coords[2] << std::endl;
02444                 }
02445                 else
02446                 {
02447                     EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
02448                     assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
02449                     UNPACK_EH( buff_ptr, connect, verts_per_entity );
02450 
02451                     // Update connectivity to local handles
02452                     std::cerr << "Connectivity: ";
02453                     for( k = 0; k < verts_per_entity; k++ )
02454                         std::cerr << connect[k] << " ";
02455                     std::cerr << std::endl;
02456                 }
02457 
02458                 if( buff_ptr - orig_ptr > msg_length )
02459                 {
02460                     std::cerr << "End of buffer..." << std::endl;
02461                     std::cerr.flush();
02462                     return MB_FAILURE;
02463                 }
02464             }
02465         }
02466     }
02467     else if( MB_MESG_REMOTEH_SIZE == mesg_tag || MB_MESG_REMOTEH_LARGE == mesg_tag )
02468     {
02469         UNPACK_INT( buff_ptr, num_ents );
02470         std::cerr << num_ents << " entities..." << std::endl;
02471         if( 0 > num_ents || num_ents > msg_length )
02472         {
02473             std::cerr << "Wrong number of entities, returning." << std::endl;
02474             return MB_FAILURE;
02475         }
02476         std::vector< EntityHandle > L1hloc( num_ents ), L1hrem( num_ents );
02477         std::vector< int > L1p( num_ents );
02478         UNPACK_INTS( buff_ptr, &L1p[0], num_ents );
02479         UNPACK_EH( buff_ptr, &L1hrem[0], num_ents );
02480         UNPACK_EH( buff_ptr, &L1hloc[0], num_ents );
02481         std::cerr << num_ents << " Entity pairs; hremote/hlocal/proc: " << std::endl;
02482         for( int i = 0; i < num_ents; i++ )
02483         {
02484             EntityType etype = TYPE_FROM_HANDLE( L1hloc[i] );
02485             std::cerr << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hrem[i] ) << ", "
02486                       << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hloc[i] ) << ", " << L1p[i] << std::endl;
02487         }
02488 
02489         if( buff_ptr - orig_ptr > msg_length )
02490         {
02491             std::cerr << "End of buffer..." << std::endl;
02492             std::cerr.flush();
02493             return MB_FAILURE;
02494         }
02495     }
02496     else if( mesg_tag == MB_MESG_TAGS_SIZE || mesg_tag == MB_MESG_TAGS_LARGE )
02497     {
02498         int num_tags, dum1, data_type, tag_size;
02499         UNPACK_INT( buff_ptr, num_tags );
02500         std::cerr << "Number of tags = " << num_tags << std::endl;
02501         for( int i = 0; i < num_tags; i++ )
02502         {
02503             std::cerr << "Tag " << i << ":" << std::endl;
02504             UNPACK_INT( buff_ptr, tag_size );
02505             UNPACK_INT( buff_ptr, dum1 );
02506             UNPACK_INT( buff_ptr, data_type );
02507             std::cerr << "Tag size, type, data type = " << tag_size << ", " << dum1 << ", " << data_type << std::endl;
02508             UNPACK_INT( buff_ptr, dum1 );
02509             std::cerr << "Default value size = " << dum1 << std::endl;
02510             buff_ptr += dum1;
02511             UNPACK_INT( buff_ptr, dum1 );
02512             std::string name( (char*)buff_ptr, dum1 );
02513             std::cerr << "Tag name = " << name.c_str() << std::endl;
02514             buff_ptr += dum1;
02515             UNPACK_INT( buff_ptr, num_ents );
02516             std::cerr << "Number of ents = " << num_ents << std::endl;
02517             std::vector< EntityHandle > tmp_buff( num_ents );
02518             UNPACK_EH( buff_ptr, &tmp_buff[0], num_ents );
02519             int tot_length = 0;
02520             for( int j = 0; j < num_ents; j++ )
02521             {
02522                 EntityType etype = TYPE_FROM_HANDLE( tmp_buff[j] );
02523                 std::cerr << CN::EntityTypeName( etype ) << " " << ID_FROM_HANDLE( tmp_buff[j] ) << ", tag = ";
02524                 if( tag_size == MB_VARIABLE_LENGTH )
02525                 {
02526                     UNPACK_INT( buff_ptr, dum1 );
02527                     tot_length += dum1;
02528                     std::cerr << "(variable, length = " << dum1 << ")" << std::endl;
02529                 }
02530                 else if( data_type == MB_TYPE_DOUBLE )
02531                 {
02532                     double dum_dbl;
02533                     UNPACK_DBL( buff_ptr, dum_dbl );
02534                     std::cerr << dum_dbl << std::endl;
02535                 }
02536                 else if( data_type == MB_TYPE_INTEGER )
02537                 {
02538                     int dum_int;
02539                     UNPACK_INT( buff_ptr, dum_int );
02540                     std::cerr << dum_int << std::endl;
02541                 }
02542                 else if( data_type == MB_TYPE_OPAQUE )
02543                 {
02544                     std::cerr << "(opaque)" << std::endl;
02545                     buff_ptr += tag_size;
02546                 }
02547                 else if( data_type == MB_TYPE_HANDLE )
02548                 {
02549                     EntityHandle dum_eh;
02550                     UNPACK_EH( buff_ptr, &dum_eh, 1 );
02551                     std::cerr << dum_eh << std::endl;
02552                 }
02553                 else if( data_type == MB_TYPE_BIT )
02554                 {
02555                     std::cerr << "(bit)" << std::endl;
02556                     buff_ptr += tag_size;
02557                 }
02558             }
02559             if( tag_size == MB_VARIABLE_LENGTH ) buff_ptr += tot_length;
02560         }
02561     }
02562     else
02563     {
02564         assert( false );
02565         return MB_FAILURE;
02566     }
02567 
02568     std::cerr.flush();
02569 
02570     return MB_SUCCESS;
02571 }
02572 
02573 ErrorCode ParallelComm::list_entities( const EntityHandle* ents, int num_ents )
02574 {
02575     if( NULL == ents )
02576     {
02577         Range shared_ents;
02578         std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) );
02579         shared_ents.print( "Shared entities:\n" );
02580         return MB_SUCCESS;
02581     }
02582 
02583     unsigned char pstat;
02584     EntityHandle tmp_handles[MAX_SHARING_PROCS];
02585     int tmp_procs[MAX_SHARING_PROCS];
02586     unsigned int num_ps;
02587     ErrorCode result;
02588 
02589     for( int i = 0; i < num_ents; i++ )
02590     {
02591         result = mbImpl->list_entities( ents + i, 1 );MB_CHK_ERR( result );
02592         double coords[3];
02593         result = mbImpl->get_coords( ents + i, 1, coords );
02594         std::cout << " coords: " << coords[0] << " " << coords[1] << " " << coords[2] << "\n";
02595 
02596         result = get_sharing_data( ents[i], tmp_procs, tmp_handles, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
02597 
02598         std::cout << "Pstatus: ";
02599         if( !num_ps )
02600             std::cout << "local " << std::endl;
02601         else
02602         {
02603             if( pstat & PSTATUS_NOT_OWNED ) std::cout << "NOT_OWNED; ";
02604             if( pstat & PSTATUS_SHARED ) std::cout << "SHARED; ";
02605             if( pstat & PSTATUS_MULTISHARED ) std::cout << "MULTISHARED; ";
02606             if( pstat & PSTATUS_INTERFACE ) std::cout << "INTERFACE; ";
02607             if( pstat & PSTATUS_GHOST ) std::cout << "GHOST; ";
02608             std::cout << std::endl;
02609             for( unsigned int j = 0; j < num_ps; j++ )
02610             {
02611                 std::cout << "  proc " << tmp_procs[j] << " id (handle) " << mbImpl->id_from_handle( tmp_handles[j] )
02612                           << "(" << tmp_handles[j] << ")" << std::endl;
02613             }
02614         }
02615         std::cout << std::endl;
02616     }
02617 
02618     return MB_SUCCESS;
02619 }
02620 
02621 ErrorCode ParallelComm::list_entities( const Range& ents )
02622 {
02623     for( Range::iterator rit = ents.begin(); rit != ents.end(); ++rit )
02624         list_entities( &( *rit ), 1 );
02625 
02626     return MB_SUCCESS;
02627 }
02628 
02629 ErrorCode ParallelComm::update_remote_data( Range& local_range,
02630                                             Range& remote_range,
02631                                             int other_proc,
02632                                             const unsigned char add_pstat )
02633 {
02634     Range::iterator rit, rit2;
02635     ErrorCode result = MB_SUCCESS;
02636 
02637     // For each pair of local/remote handles:
02638     for( rit = local_range.begin(), rit2 = remote_range.begin(); rit != local_range.end(); ++rit, ++rit2 )
02639     {
02640         result = update_remote_data( *rit, &other_proc, &( *rit2 ), 1, add_pstat );MB_CHK_ERR( result );
02641     }
02642 
02643     return MB_SUCCESS;
02644 }
02645 
02646 ErrorCode ParallelComm::update_remote_data( const EntityHandle new_h,
02647                                             const int* ps,
02648                                             const EntityHandle* hs,
02649                                             const int num_ps,
02650                                             const unsigned char add_pstat
02651                                             // The following lines left in for future debugging, at least until I trust
02652                                             // this function; tjt, 10/4/2013
02653                                             //                                           , int *new_ps,
02654                                             //                                           EntityHandle *new_hs,
02655                                             //                                           int &new_numps,
02656                                             //                                           unsigned char &new_pstat
02657 )
02658 {
02659     // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0
02660     // in this function, so no need to initialize; sharing data does not include
02661     // this proc if shared with only one other
02662 
02663     // Following variables declared here to avoid compiler errors
02664     int new_numps;
02665     unsigned char new_pstat;
02666     std::vector< int > new_ps( MAX_SHARING_PROCS, -1 );
02667     std::vector< EntityHandle > new_hs( MAX_SHARING_PROCS, 0 );
02668 
02669     new_numps        = 0;
02670     ErrorCode result = get_sharing_data( new_h, &new_ps[0], &new_hs[0], new_pstat, new_numps );MB_CHK_SET_ERR( result, "Failed to get sharing data in update_remote_data" );
02671     int num_exist = new_numps;
02672 
02673     // Add new pstat info to the flag
02674     new_pstat |= add_pstat;
02675 
02676     /*
02677     #define plist(str, lst, siz)                                          \
02678         std::cout << str << "(";                                          \
02679         for (int i = 0; i < (int)siz; i++) std::cout << lst[i] << " ";    \
02680         std::cout << ") ";                                                \
02681 
02682         std::cout << "update_remote_data: rank = " << rank() << ", new_h = " << new_h << std::endl;
02683         std::string ostr;
02684         plist("ps", ps, num_ps);
02685         plist("hs", hs, num_ps);
02686         print_pstatus(add_pstat, ostr);
02687         std::cout << ", add_pstat = " << ostr.c_str() << std::endl;
02688         plist("tag_ps", new_ps, new_numps);
02689         plist("tag_hs", new_hs, new_numps);
02690         assert(new_numps <= size());
02691         print_pstatus(new_pstat, ostr);
02692         std::cout << ", tag_pstat=" << ostr.c_str() << std::endl;
02693     */
02694 
02695 #ifndef NDEBUG
02696     {
02697         // Check for duplicates in proc list
02698         std::set< unsigned int > dumprocs;
02699         unsigned int dp = 0;
02700         for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
02701             dumprocs.insert( ps[dp] );
02702         assert( dp == dumprocs.size() );
02703     }
02704 #endif
02705 
02706     // If only one sharer and I'm the owner, insert myself in the list;
02707     // otherwise, my data is checked at the end
02708     if( 1 == new_numps && !( new_pstat & PSTATUS_NOT_OWNED ) )
02709     {
02710         new_hs[1] = new_hs[0];
02711         new_ps[1] = new_ps[0];
02712         new_hs[0] = new_h;
02713         new_ps[0] = rank();
02714         new_numps = 2;
02715     }
02716 
02717     // Now put passed-in data onto lists
02718     int idx;
02719     for( int i = 0; i < num_ps; i++ )
02720     {
02721         idx = std::find( &new_ps[0], &new_ps[0] + new_numps, ps[i] ) - &new_ps[0];
02722         if( idx < new_numps )
02723         {
02724             if( !new_hs[idx] && hs[i] )
02725                 // h on list is 0 and passed-in h is non-zero, replace it
02726                 new_hs[idx] = hs[i];
02727             else
02728                 assert( !hs[i] || new_hs[idx] == hs[i] );
02729         }
02730         else
02731         {
02732             if( new_numps + 1 == MAX_SHARING_PROCS )
02733             {
02734                 MB_SET_ERR( MB_FAILURE, "Exceeded MAX_SHARING_PROCS for "
02735                                             << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) << ' '
02736                                             << ID_FROM_HANDLE( new_h ) << " in process " << rank() );
02737             }
02738             new_ps[new_numps] = ps[i];
02739             new_hs[new_numps] = hs[i];
02740             new_numps++;
02741         }
02742     }
02743 
02744     // Add myself, if it isn't there already
02745     idx = std::find( &new_ps[0], &new_ps[0] + new_numps, rank() ) - &new_ps[0];
02746     if( idx == new_numps )
02747     {
02748         new_ps[new_numps] = rank();
02749         new_hs[new_numps] = new_h;
02750         new_numps++;
02751     }
02752     else if( !new_hs[idx] && new_numps > 2 )
02753         new_hs[idx] = new_h;
02754 
02755     // Proc list is complete; update for shared, multishared
02756     if( new_numps > 1 )
02757     {
02758         if( new_numps > 2 ) new_pstat |= PSTATUS_MULTISHARED;
02759         new_pstat |= PSTATUS_SHARED;
02760     }
02761 
02762     /*
02763         plist("new_ps", new_ps, new_numps);
02764         plist("new_hs", new_hs, new_numps);
02765         print_pstatus(new_pstat, ostr);
02766         std::cout << ", new_pstat=" << ostr.c_str() << std::endl;
02767         std::cout << std::endl;
02768     */
02769 
02770     result = set_sharing_data( new_h, new_pstat, num_exist, new_numps, &new_ps[0], &new_hs[0] );MB_CHK_SET_ERR( result, "Failed to set sharing data in update_remote_data" );
02771 
02772     if( new_pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
02773 
02774     return MB_SUCCESS;
02775 }
02776 
02777 ErrorCode ParallelComm::update_remote_data_old( const EntityHandle new_h,
02778                                                 const int* ps,
02779                                                 const EntityHandle* hs,
02780                                                 const int num_ps,
02781                                                 const unsigned char add_pstat )
02782 {
02783     EntityHandle tag_hs[MAX_SHARING_PROCS];
02784     int tag_ps[MAX_SHARING_PROCS];
02785     unsigned char pstat;
02786     // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0
02787     // in this function, so no need to initialize
02788     unsigned int num_exist;
02789     ErrorCode result = get_sharing_data( new_h, tag_ps, tag_hs, pstat, num_exist );MB_CHK_ERR( result );
02790 
02791 #ifndef NDEBUG
02792     {
02793         // Check for duplicates in proc list
02794         std::set< unsigned int > dumprocs;
02795         unsigned int dp = 0;
02796         for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
02797             dumprocs.insert( ps[dp] );
02798         assert( dp == dumprocs.size() );
02799     }
02800 #endif
02801 
02802     // Add any new sharing data
02803     bool changed = false;
02804     int idx;
02805     if( !num_exist )
02806     {
02807         // Just take what caller passed
02808         memcpy( tag_ps, ps, num_ps * sizeof( int ) );
02809         memcpy( tag_hs, hs, num_ps * sizeof( EntityHandle ) );
02810         num_exist = num_ps;
02811         // If it's only one, hopefully I'm not there yet...
02812         assert( "I shouldn't be the only proc there." && ( 1 != num_exist || ps[0] != (int)procConfig.proc_rank() ) );
02813         changed = true;
02814     }
02815     else
02816     {
02817         for( int i = 0; i < num_ps; i++ )
02818         {
02819             idx = std::find( tag_ps, tag_ps + num_exist, ps[i] ) - tag_ps;
02820             if( idx == (int)num_exist )
02821             {
02822                 if( num_exist == MAX_SHARING_PROCS )
02823                 {
02824                     std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) )
02825                               << ' ' << ID_FROM_HANDLE( new_h ) << " in process " << proc_config().proc_rank()
02826                               << std::endl;
02827                     std::cerr.flush();
02828                     MPI_Abort( proc_config().proc_comm(), 66 );
02829                 }
02830 
02831                 // If there's only 1 sharing proc, and it's not me, then
02832                 // we'll end up with 3; add me to the front
02833                 if( !i && num_ps == 1 && num_exist == 1 && ps[0] != (int)procConfig.proc_rank() )
02834                 {
02835                     int j = 1;
02836                     // If I own this entity, put me at front, otherwise after first
02837                     if( !( pstat & PSTATUS_NOT_OWNED ) )
02838                     {
02839                         tag_ps[1] = tag_ps[0];
02840                         tag_hs[1] = tag_hs[0];
02841                         j         = 0;
02842                     }
02843                     tag_ps[j] = procConfig.proc_rank();
02844                     tag_hs[j] = new_h;
02845                     num_exist++;
02846                 }
02847 
02848                 tag_ps[num_exist] = ps[i];
02849                 tag_hs[num_exist] = hs[i];
02850                 num_exist++;
02851                 changed = true;
02852             }
02853             else if( 0 == tag_hs[idx] )
02854             {
02855                 tag_hs[idx] = hs[i];
02856                 changed     = true;
02857             }
02858             else if( 0 != hs[i] )
02859             {
02860                 assert( hs[i] == tag_hs[idx] );
02861             }
02862         }
02863     }
02864 
02865     // Adjust for interface layer if necessary
02866     if( add_pstat & PSTATUS_INTERFACE )
02867     {
02868         idx = std::min_element( tag_ps, tag_ps + num_exist ) - tag_ps;
02869         if( idx )
02870         {
02871             int tag_proc       = tag_ps[idx];
02872             tag_ps[idx]        = tag_ps[0];
02873             tag_ps[0]          = tag_proc;
02874             EntityHandle tag_h = tag_hs[idx];
02875             tag_hs[idx]        = tag_hs[0];
02876             tag_hs[0]          = tag_h;
02877             changed            = true;
02878             if( tag_ps[0] != (int)procConfig.proc_rank() ) pstat |= PSTATUS_NOT_OWNED;
02879         }
02880     }
02881 
02882     if( !changed ) return MB_SUCCESS;
02883 
02884     assert( "interface entities should have > 1 proc" && ( !( add_pstat & PSTATUS_INTERFACE ) || num_exist > 1 ) );
02885     assert( "ghost entities should have > 1 proc" && ( !( add_pstat & PSTATUS_GHOST ) || num_exist > 1 ) );
02886 
02887     // If it's multi-shared and we created the entity in this unpack,
02888     // local handle probably isn't in handle list yet
02889     if( num_exist > 2 )
02890     {
02891         idx = std::find( tag_ps, tag_ps + num_exist, procConfig.proc_rank() ) - tag_ps;
02892         assert( idx < (int)num_exist );
02893         if( !tag_hs[idx] ) tag_hs[idx] = new_h;
02894     }
02895 
02896     int tag_p;
02897     EntityHandle tag_h;
02898 
02899     // Update pstat
02900     pstat |= add_pstat;
02901 
02902     if( num_exist > 2 )
02903         pstat |= ( PSTATUS_MULTISHARED | PSTATUS_SHARED );
02904     else if( num_exist > 0 )
02905         pstat |= PSTATUS_SHARED;
02906 
02907     //    compare_remote_data(new_h, num_ps, hs, ps, add_pstat,
02908     //                        num_exist, tag_hs, tag_ps, pstat);
02909 
02910     // Reset single shared proc/handle if was shared and moving to multi-shared
02911     if( num_exist > 2 && !( pstat & PSTATUS_MULTISHARED ) && ( pstat & PSTATUS_SHARED ) )
02912     {
02913         // Must remove sharedp/h first, which really means set to default value
02914         tag_p  = -1;
02915         result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, &tag_p );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
02916         tag_h  = 0;
02917         result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, &tag_h );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
02918     }
02919 
02920     // Set sharing tags
02921     if( num_exist > 2 )
02922     {
02923         std::fill( tag_ps + num_exist, tag_ps + MAX_SHARING_PROCS, -1 );
02924         std::fill( tag_hs + num_exist, tag_hs + MAX_SHARING_PROCS, 0 );
02925         result = mbImpl->tag_set_data( sharedps_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" );
02926         result = mbImpl->tag_set_data( sharedhs_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" );
02927 
02928 #ifndef NDEBUG
02929         {
02930             // Check for duplicates in proc list
02931             std::set< unsigned int > dumprocs;
02932             unsigned int dp = 0;
02933             for( ; dp < num_exist && -1 != tag_ps[dp]; dp++ )
02934                 dumprocs.insert( tag_ps[dp] );
02935             assert( dp == dumprocs.size() );
02936         }
02937 #endif
02938     }
02939     else if( num_exist == 2 || num_exist == 1 )
02940     {
02941         if( tag_ps[0] == (int)procConfig.proc_rank() )
02942         {
02943             assert( 2 == num_exist && tag_ps[1] != (int)procConfig.proc_rank() );
02944             tag_ps[0] = tag_ps[1];
02945             tag_hs[0] = tag_hs[1];
02946         }
02947         assert( tag_ps[0] != -1 && tag_hs[0] != 0 );
02948         result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
02949         result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
02950     }
02951 
02952     // Now set new pstatus
02953     result = mbImpl->tag_set_data( pstatus_tag(), &new_h, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
02954 
02955     if( pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
02956 
02957     return MB_SUCCESS;
02958 }
02959 
02960 ErrorCode ParallelComm::get_sharing_data( const Range& entities, std::set< int >& procs, int operation )
02961 {
02962     // Get the union or intersection of sharing data for multiple entities
02963     ErrorCode result;
02964     int sp2[MAX_SHARING_PROCS];
02965     int num_ps;
02966     unsigned char pstat;
02967     std::set< int > tmp_procs;
02968     procs.clear();
02969 
02970     for( Range::const_iterator rit = entities.begin(); rit != entities.end(); ++rit )
02971     {
02972         // Get sharing procs
02973         result = get_sharing_data( *rit, sp2, NULL, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_sharing_data" );
02974         if( !( pstat & PSTATUS_SHARED ) && Interface::INTERSECT == operation )
02975         {
02976             procs.clear();
02977             return MB_SUCCESS;
02978         }
02979 
02980         if( rit == entities.begin() )
02981         {
02982             std::copy( sp2, sp2 + num_ps, std::inserter( procs, procs.begin() ) );
02983         }
02984         else
02985         {
02986             std::sort( sp2, sp2 + num_ps );
02987             tmp_procs.clear();
02988             if( Interface::UNION == operation )
02989                 std::set_union( procs.begin(), procs.end(), sp2, sp2 + num_ps,
02990                                 std::inserter( tmp_procs, tmp_procs.end() ) );
02991             else if( Interface::INTERSECT == operation )
02992                 std::set_intersection( procs.begin(), procs.end(), sp2, sp2 + num_ps,
02993                                        std::inserter( tmp_procs, tmp_procs.end() ) );
02994             else
02995             {
02996                 assert( "Unknown operation." && false );
02997                 return MB_FAILURE;
02998             }
02999             procs.swap( tmp_procs );
03000         }
03001         if( Interface::INTERSECT == operation && procs.empty() ) return MB_SUCCESS;
03002     }
03003 
03004     return MB_SUCCESS;
03005 }
03006 
03007 ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity,
03008                                           int* ps,
03009                                           EntityHandle* hs,
03010                                           unsigned char& pstat,
03011                                           unsigned int& num_ps )
03012 {
03013     ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
03014     if( pstat & PSTATUS_MULTISHARED )
03015     {
03016         result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
03017         if( hs )
03018         {
03019             result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
03020         }
03021         num_ps = std::find( ps, ps + MAX_SHARING_PROCS, -1 ) - ps;
03022     }
03023     else if( pstat & PSTATUS_SHARED )
03024     {
03025         result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
03026         if( hs )
03027         {
03028             result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
03029             hs[1] = 0;
03030         }
03031         // Initialize past end of data
03032         ps[1]  = -1;
03033         num_ps = 1;
03034     }
03035     else
03036     {
03037         ps[0] = -1;
03038         if( hs ) hs[0] = 0;
03039         num_ps = 0;
03040     }
03041 
03042     assert( MAX_SHARING_PROCS >= num_ps );
03043 
03044     return MB_SUCCESS;
03045 }
03046 
03047 ErrorCode ParallelComm::find_existing_entity( const bool is_iface,
03048                                               const int owner_p,
03049                                               const EntityHandle owner_h,
03050                                               const int num_ps,
03051                                               const EntityHandle* connect,
03052                                               const int num_connect,
03053                                               const EntityType this_type,
03054                                               std::vector< EntityHandle >& L2hloc,
03055                                               std::vector< EntityHandle >& L2hrem,
03056                                               std::vector< unsigned int >& L2p,
03057                                               EntityHandle& new_h )
03058 {
03059     new_h = 0;
03060     if( !is_iface && num_ps > 2 )
03061     {
03062         for( unsigned int i = 0; i < L2hrem.size(); i++ )
03063         {
03064             if( L2hrem[i] == owner_h && owner_p == (int)L2p[i] )
03065             {
03066                 new_h = L2hloc[i];
03067                 return MB_SUCCESS;
03068             }
03069         }
03070     }
03071 
03072     // If we got here and it's a vertex, we don't need to look further
03073     if( MBVERTEX == this_type || !connect || !num_connect ) return MB_SUCCESS;
03074 
03075     Range tmp_range;
03076     ErrorCode result = mbImpl->get_adjacencies( connect, num_connect, CN::Dimension( this_type ), false, tmp_range );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
03077     if( !tmp_range.empty() )
03078     {
03079         // Found a corresponding entity - return target
03080         new_h = *tmp_range.begin();
03081     }
03082     else
03083     {
03084         new_h = 0;
03085     }
03086 
03087     return MB_SUCCESS;
03088 }
03089 
03090 ErrorCode ParallelComm::get_local_handles( const Range& remote_handles,
03091                                            Range& local_handles,
03092                                            const std::vector< EntityHandle >& new_ents )
03093 {
03094     std::vector< EntityHandle > rh_vec;
03095     rh_vec.reserve( remote_handles.size() );
03096     std::copy( remote_handles.begin(), remote_handles.end(), std::back_inserter( rh_vec ) );
03097     ErrorCode result = get_local_handles( &rh_vec[0], remote_handles.size(), new_ents );
03098     std::copy( rh_vec.begin(), rh_vec.end(), range_inserter( local_handles ) );
03099     return result;
03100 }
03101 
03102 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents )
03103 {
03104     std::vector< EntityHandle > tmp_ents;
03105     std::copy( new_ents.begin(), new_ents.end(), std::back_inserter( tmp_ents ) );
03106     return get_local_handles( from_vec, num_ents, tmp_ents );
03107 }
03108 
03109 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec,
03110                                            int num_ents,
03111                                            const std::vector< EntityHandle >& new_ents )
03112 {
03113     for( int i = 0; i < num_ents; i++ )
03114     {
03115         if( TYPE_FROM_HANDLE( from_vec[i] ) == MBMAXTYPE )
03116         {
03117             assert( ID_FROM_HANDLE( from_vec[i] ) < (int)new_ents.size() );
03118             from_vec[i] = new_ents[ID_FROM_HANDLE( from_vec[i] )];
03119         }
03120     }
03121 
03122     return MB_SUCCESS;
03123 }
03124 
03125 /*
03126 template <typename T> void
03127 insert_in_array(T* array, size_t array_size, size_t location, T value)
03128 {
03129   assert(location + 1 < array_size);
03130   for (size_t i = array_size - 1; i > location; i--)
03131     array[i] = array[i - 1];
03132   array[location] = value;
03133 }
03134 */
03135 
03136 ErrorCode ParallelComm::pack_range_map( Range& key_range, EntityHandle val_start, HandleMap& handle_map )
03137 {
03138     for( Range::const_pair_iterator key_it = key_range.const_pair_begin(); key_it != key_range.const_pair_end();
03139          ++key_it )
03140     {
03141         int tmp_num = ( *key_it ).second - ( *key_it ).first + 1;
03142         handle_map.insert( ( *key_it ).first, val_start, tmp_num );
03143         val_start += tmp_num;
03144     }
03145 
03146     return MB_SUCCESS;
03147 }
03148 
03149 ErrorCode ParallelComm::pack_sets( Range& entities, Buffer* buff, const bool store_remote_handles, const int to_proc )
03150 {
03151     // SETS:
03152     // . #sets
03153     // . for each set:
03154     //   - options[#sets] (unsigned int)
03155     //   - if (unordered) set range
03156     //   - else if ordered
03157     //     . #ents in set
03158     //     . handles[#ents]
03159     //   - #parents
03160     //   - if (#parents) handles[#parents]
03161     //   - #children
03162     //   - if (#children) handles[#children]
03163 
03164     // Now the sets; assume any sets the application wants to pass are in the entities list
03165     ErrorCode result;
03166     Range all_sets = entities.subset_by_type( MBENTITYSET );
03167 
03168     int buff_size = estimate_sets_buffer_size( all_sets, store_remote_handles );
03169     if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate sets buffer size" );
03170     buff->check_space( buff_size );
03171 
03172     // Number of sets
03173     PACK_INT( buff->buff_ptr, all_sets.size() );
03174 
03175     // Options for all sets
03176     std::vector< unsigned int > options( all_sets.size() );
03177     Range::iterator rit;
03178     std::vector< EntityHandle > members;
03179     int i;
03180     for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
03181     {
03182         result = mbImpl->get_meshset_options( *rit, options[i] );MB_CHK_SET_ERR( result, "Failed to get meshset options" );
03183     }
03184     buff->check_space( all_sets.size() * sizeof( unsigned int ) );
03185     PACK_VOID( buff->buff_ptr, &options[0], all_sets.size() * sizeof( unsigned int ) );
03186 
03187     // Pack parallel geometry unique id
03188     if( !all_sets.empty() )
03189     {
03190         Tag uid_tag;
03191         int n_sets  = all_sets.size();
03192         bool b_pack = false;
03193         std::vector< int > id_data( n_sets );
03194         result =
03195             mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
03196 
03197         result = mbImpl->tag_get_data( uid_tag, all_sets, &id_data[0] );
03198         if( MB_TAG_NOT_FOUND != result )
03199         {
03200             if( MB_SUCCESS != result ) MB_SET_ERR( result, "Failed to get parallel geometry unique ids" );
03201             for( i = 0; i < n_sets; i++ )
03202             {
03203                 if( id_data[i] != 0 )
03204                 {
03205                     b_pack = true;
03206                     break;
03207                 }
03208             }
03209         }
03210 
03211         if( b_pack )
03212         {  // If you find
03213             buff->check_space( ( n_sets + 1 ) * sizeof( int ) );
03214             PACK_INT( buff->buff_ptr, n_sets );
03215             PACK_INTS( buff->buff_ptr, &id_data[0], n_sets );
03216         }
03217         else
03218         {
03219             buff->check_space( sizeof( int ) );
03220             PACK_INT( buff->buff_ptr, 0 );
03221         }
03222     }
03223 
03224     // Vectors/ranges
03225     std::vector< EntityHandle > entities_vec( entities.size() );
03226     std::copy( entities.begin(), entities.end(), entities_vec.begin() );
03227     for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
03228     {
03229         members.clear();
03230         result = mbImpl->get_entities_by_handle( *rit, members );MB_CHK_SET_ERR( result, "Failed to get entities in ordered set" );
03231         result =
03232             get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
03233         buff->check_space( members.size() * sizeof( EntityHandle ) + sizeof( int ) );
03234         PACK_INT( buff->buff_ptr, members.size() );
03235         PACK_EH( buff->buff_ptr, &members[0], members.size() );
03236     }
03237 
03238     // Pack parent/child sets
03239     if( !store_remote_handles )
03240     {  // Only works not store remote handles
03241         // Pack numbers of parents/children
03242         unsigned int tot_pch = 0;
03243         int num_pch;
03244         buff->check_space( 2 * all_sets.size() * sizeof( int ) );
03245         for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
03246         {
03247             // Pack parents
03248             result = mbImpl->num_parent_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num parents" );
03249             PACK_INT( buff->buff_ptr, num_pch );
03250             tot_pch += num_pch;
03251             result = mbImpl->num_child_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num children" );
03252             PACK_INT( buff->buff_ptr, num_pch );
03253             tot_pch += num_pch;
03254         }
03255 
03256         // Now pack actual parents/children
03257         members.clear();
03258         members.reserve( tot_pch );
03259         std::vector< EntityHandle > tmp_pch;
03260         for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
03261         {
03262             result = mbImpl->get_parent_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get parents" );
03263             std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
03264             tmp_pch.clear();
03265             result = mbImpl->get_child_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get children" );
03266             std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
03267             tmp_pch.clear();
03268         }
03269         assert( members.size() == tot_pch );
03270         if( !members.empty() )
03271         {
03272             result = get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc,
03273                                          entities_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles for set parent/child sets" );
03274 #ifndef NDEBUG
03275             // Check that all handles are either sets or maxtype
03276             for( unsigned int __j = 0; __j < members.size(); __j++ )
03277                 assert( ( TYPE_FROM_HANDLE( members[__j] ) == MBMAXTYPE &&
03278                           ID_FROM_HANDLE( members[__j] ) < (int)entities.size() ) ||
03279                         TYPE_FROM_HANDLE( members[__j] ) == MBENTITYSET );
03280 #endif
03281             buff->check_space( members.size() * sizeof( EntityHandle ) );
03282             PACK_EH( buff->buff_ptr, &members[0], members.size() );
03283         }
03284     }
03285     else
03286     {
03287         buff->check_space( 2 * all_sets.size() * sizeof( int ) );
03288         for( rit = all_sets.begin(); rit != all_sets.end(); ++rit )
03289         {
03290             PACK_INT( buff->buff_ptr, 0 );
03291             PACK_INT( buff->buff_ptr, 0 );
03292         }
03293     }
03294 
03295     // Pack the handles
03296     if( store_remote_handles && !all_sets.empty() )
03297     {
03298         buff_size = RANGE_SIZE( all_sets );
03299         buff->check_space( buff_size );
03300         PACK_RANGE( buff->buff_ptr, all_sets );
03301     }
03302 
03303     myDebug->tprintf( 4, "Done packing sets.\n" );
03304 
03305     buff->set_stored_size();
03306 
03307     return MB_SUCCESS;
03308 }
03309 
03310 ErrorCode ParallelComm::unpack_sets( unsigned char*& buff_ptr,
03311                                      std::vector< EntityHandle >& entities,
03312                                      const bool store_remote_handles,
03313                                      const int from_proc )
03314 {
03315     // Now the sets; assume any sets the application wants to pass are in the entities list
03316     ErrorCode result;
03317 
03318     bool no_sets = ( entities.empty() || ( mbImpl->type_from_handle( *entities.rbegin() ) == MBENTITYSET ) );
03319 
03320     Range new_sets;
03321     int num_sets;
03322     UNPACK_INT( buff_ptr, num_sets );
03323 
03324     if( !num_sets ) return MB_SUCCESS;
03325 
03326     int i;
03327     Range::const_iterator rit;
03328     std::vector< EntityHandle > members;
03329     int num_ents;
03330     std::vector< unsigned int > options_vec( num_sets );
03331     // Option value
03332     if( num_sets ) UNPACK_VOID( buff_ptr, &options_vec[0], num_sets * sizeof( unsigned int ) );
03333 
03334     // Unpack parallel geometry unique id
03335     int n_uid;
03336     UNPACK_INT( buff_ptr, n_uid );
03337     if( n_uid > 0 && n_uid != num_sets )
03338     {
03339         std::cerr << "The number of Parallel geometry unique ids should be same." << std::endl;
03340     }
03341 
03342     if( n_uid > 0 )
03343     {  // If parallel geometry unique id is packed
03344         std::vector< int > uids( n_uid );
03345         UNPACK_INTS( buff_ptr, &uids[0], n_uid );
03346 
03347         Tag uid_tag;
03348         result =
03349             mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
03350 
03351         // Find existing sets
03352         for( i = 0; i < n_uid; i++ )
03353         {
03354             EntityHandle set_handle;
03355             Range temp_sets;
03356             void* tag_vals[] = { &uids[i] };
03357             if( uids[i] > 0 )
03358             {
03359                 result = mbImpl->get_entities_by_type_and_tag( 0, MBENTITYSET, &uid_tag, tag_vals, 1, temp_sets );
03360             }
03361             if( !temp_sets.empty() )
03362             {  // Existing set
03363                 set_handle = *temp_sets.begin();
03364             }
03365             else
03366             {  // Create a new set
03367                 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
03368                 result = mbImpl->tag_set_data( uid_tag, &set_handle, 1, &uids[i] );MB_CHK_SET_ERR( result, "Failed to set parallel geometry unique ids" );
03369             }
03370             new_sets.insert( set_handle );
03371         }
03372     }
03373     else
03374     {
03375         // Create sets
03376         for( i = 0; i < num_sets; i++ )
03377         {
03378             EntityHandle set_handle;
03379             result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
03380 
03381             // Make sure new sets handles are monotonically increasing
03382             assert( set_handle > *new_sets.rbegin() );
03383             new_sets.insert( set_handle );
03384         }
03385     }
03386 
03387     std::copy( new_sets.begin(), new_sets.end(), std::back_inserter( entities ) );
03388     // Only need to sort if we came in with no sets on the end
03389     if( !no_sets ) std::sort( entities.begin(), entities.end() );
03390 
03391     for( rit = new_sets.begin(), i = 0; rit != new_sets.end(); ++rit, i++ )
03392     {
03393         // Unpack entities as vector, with length
03394         UNPACK_INT( buff_ptr, num_ents );
03395         members.resize( num_ents );
03396         if( num_ents ) UNPACK_EH( buff_ptr, &members[0], num_ents );
03397         result = get_local_handles( &members[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for ordered set contents" );
03398         result = mbImpl->add_entities( *rit, &members[0], num_ents );MB_CHK_SET_ERR( result, "Failed to add ents to ordered set in unpack" );
03399     }
03400 
03401     std::vector< int > num_pch( 2 * new_sets.size() );
03402     std::vector< int >::iterator vit;
03403     int tot_pch = 0;
03404     for( vit = num_pch.begin(); vit != num_pch.end(); ++vit )
03405     {
03406         UNPACK_INT( buff_ptr, *vit );
03407         tot_pch += *vit;
03408     }
03409 
03410     members.resize( tot_pch );
03411     UNPACK_EH( buff_ptr, &members[0], tot_pch );
03412     result = get_local_handles( &members[0], tot_pch, entities );MB_CHK_SET_ERR( result, "Failed to get local handle for parent/child sets" );
03413 
03414     int num               = 0;
03415     EntityHandle* mem_ptr = &members[0];
03416     for( rit = new_sets.begin(); rit != new_sets.end(); ++rit )
03417     {
03418         // Unpack parents/children
03419         int num_par = num_pch[num++], num_child = num_pch[num++];
03420         if( num_par + num_child )
03421         {
03422             for( i = 0; i < num_par; i++ )
03423             {
03424                 assert( 0 != mem_ptr[i] );
03425                 result = mbImpl->add_parent_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add parent to set in unpack" );
03426             }
03427             mem_ptr += num_par;
03428             for( i = 0; i < num_child; i++ )
03429             {
03430                 assert( 0 != mem_ptr[i] );
03431                 result = mbImpl->add_child_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add child to set in unpack" );
03432             }
03433             mem_ptr += num_child;
03434         }
03435     }
03436 
03437     // Unpack source handles
03438     Range dum_range;
03439     if( store_remote_handles && !new_sets.empty() )
03440     {
03441         UNPACK_RANGE( buff_ptr, dum_range );
03442         result = update_remote_data( new_sets, dum_range, from_proc, 0 );MB_CHK_SET_ERR( result, "Failed to set sharing data for sets" );
03443     }
03444 
03445     myDebug->tprintf( 4, "Done unpacking sets." );
03446 
03447     return MB_SUCCESS;
03448 }
03449 
03450 ErrorCode ParallelComm::pack_adjacencies( Range& /*entities*/,
03451                                           Range::const_iterator& /*start_rit*/,
03452                                           Range& /*whole_range*/,
03453                                           unsigned char*& /*buff_ptr*/,
03454                                           int& /*count*/,
03455                                           const bool /*just_count*/,
03456                                           const bool /*store_handles*/,
03457                                           const int /*to_proc*/ )
03458 {
03459     return MB_FAILURE;
03460 }
03461 
03462 ErrorCode ParallelComm::unpack_adjacencies( unsigned char*& /*buff_ptr*/,
03463                                             Range& /*entities*/,
03464                                             const bool /*store_handles*/,
03465                                             const int /*from_proc*/ )
03466 {
03467     return MB_FAILURE;
03468 }
03469 
03470 ErrorCode ParallelComm::pack_tags( Range& entities,
03471                                    const std::vector< Tag >& src_tags,
03472                                    const std::vector< Tag >& dst_tags,
03473                                    const std::vector< Range >& tag_ranges,
03474                                    Buffer* buff,
03475                                    const bool store_remote_handles,
03476                                    const int to_proc )
03477 {
03478     ErrorCode result;
03479     std::vector< Tag >::const_iterator tag_it, dst_it;
03480     std::vector< Range >::const_iterator rit;
03481     int count = 0;
03482 
03483     for( tag_it = src_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); ++tag_it, ++rit )
03484     {
03485         result = packed_tag_size( *tag_it, *rit, count );
03486         if( MB_SUCCESS != result ) return result;
03487     }
03488 
03489     // Number of tags
03490     count += sizeof( int );
03491 
03492     buff->check_space( count );
03493 
03494     PACK_INT( buff->buff_ptr, src_tags.size() );
03495 
03496     std::vector< EntityHandle > entities_vec( entities.size() );
03497     std::copy( entities.begin(), entities.end(), entities_vec.begin() );
03498 
03499     for( tag_it = src_tags.begin(), dst_it = dst_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end();
03500          ++tag_it, ++dst_it, ++rit )
03501     {
03502         result = pack_tag( *tag_it, *dst_it, *rit, entities_vec, buff, store_remote_handles, to_proc );
03503         if( MB_SUCCESS != result ) return result;
03504     }
03505 
03506     myDebug->tprintf( 4, "Done packing tags." );
03507 
03508     buff->set_stored_size();
03509 
03510     return MB_SUCCESS;
03511 }
03512 
03513 ErrorCode ParallelComm::packed_tag_size( Tag tag, const Range& tagged_entities, int& count )
03514 {
03515     // For dense tags, compute size assuming all entities have that tag
03516     // For sparse tags, get number of entities w/ that tag to compute size
03517 
03518     std::vector< int > var_len_sizes;
03519     std::vector< const void* > var_len_values;
03520 
03521     // Default value
03522     count += sizeof( int );
03523     if( NULL != tag->get_default_value() ) count += tag->get_default_value_size();
03524 
03525     // Size, type, data type
03526     count += 3 * sizeof( int );
03527 
03528     // Name
03529     count += sizeof( int );
03530     count += tag->get_name().size();
03531 
03532     // Range of tag
03533     count += sizeof( int ) + tagged_entities.size() * sizeof( EntityHandle );
03534 
03535     if( tag->get_size() == MB_VARIABLE_LENGTH )
03536     {
03537         const int num_ent = tagged_entities.size();
03538         // Send a tag size for each entity
03539         count += num_ent * sizeof( int );
03540         // Send tag data for each entity
03541         var_len_sizes.resize( num_ent );
03542         var_len_values.resize( num_ent );
03543         ErrorCode result =
03544             tag->get_data( sequenceManager, errorHandler, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get lenghts of variable-length tag values" );
03545         count += std::accumulate( var_len_sizes.begin(), var_len_sizes.end(), 0 );
03546     }
03547     else
03548     {
03549         // Tag data values for range or vector
03550         count += tagged_entities.size() * tag->get_size();
03551     }
03552 
03553     return MB_SUCCESS;
03554 }
03555 
03556 ErrorCode ParallelComm::pack_tag( Tag src_tag,
03557                                   Tag dst_tag,
03558                                   const Range& tagged_entities,
03559                                   const std::vector< EntityHandle >& whole_vec,
03560                                   Buffer* buff,
03561                                   const bool store_remote_handles,
03562                                   const int to_proc )
03563 {
03564     ErrorCode result;
03565     std::vector< int > var_len_sizes;
03566     std::vector< const void* > var_len_values;
03567 
03568     if( src_tag != dst_tag )
03569     {
03570         if( dst_tag->get_size() != src_tag->get_size() ) return MB_TYPE_OUT_OF_RANGE;
03571         if( dst_tag->get_data_type() != src_tag->get_data_type() && dst_tag->get_data_type() != MB_TYPE_OPAQUE &&
03572             src_tag->get_data_type() != MB_TYPE_OPAQUE )
03573             return MB_TYPE_OUT_OF_RANGE;
03574     }
03575 
03576     // Size, type, data type
03577     buff->check_space( 3 * sizeof( int ) );
03578     PACK_INT( buff->buff_ptr, src_tag->get_size() );
03579     TagType this_type;
03580     result = mbImpl->tag_get_type( dst_tag, this_type );
03581     PACK_INT( buff->buff_ptr, (int)this_type );
03582     DataType data_type = src_tag->get_data_type();
03583     PACK_INT( buff->buff_ptr, (int)data_type );
03584     int type_size = TagInfo::size_from_data_type( data_type );
03585 
03586     // Default value
03587     if( NULL == src_tag->get_default_value() )
03588     {
03589         buff->check_space( sizeof( int ) );
03590         PACK_INT( buff->buff_ptr, 0 );
03591     }
03592     else
03593     {
03594         buff->check_space( src_tag->get_default_value_size() );
03595         PACK_BYTES( buff->buff_ptr, src_tag->get_default_value(), src_tag->get_default_value_size() );
03596     }
03597 
03598     // Name
03599     buff->check_space( src_tag->get_name().size() );
03600     PACK_BYTES( buff->buff_ptr, dst_tag->get_name().c_str(), dst_tag->get_name().size() );
03601 
03602     myDebug->tprintf( 4, "Packing tag \"%s\"", src_tag->get_name().c_str() );
03603     if( src_tag != dst_tag ) myDebug->tprintf( 4, " (as tag \"%s\")", dst_tag->get_name().c_str() );
03604     myDebug->tprintf( 4, "\n" );
03605 
03606     // Pack entities
03607     buff->check_space( tagged_entities.size() * sizeof( EntityHandle ) + sizeof( int ) );
03608     PACK_INT( buff->buff_ptr, tagged_entities.size() );
03609     std::vector< EntityHandle > dum_tagged_entities( tagged_entities.size() );
03610     result = get_remote_handles( store_remote_handles, tagged_entities, &dum_tagged_entities[0], to_proc, whole_vec );
03611     if( MB_SUCCESS != result )
03612     {
03613         if( myDebug->get_verbosity() == 3 )
03614         {
03615             std::cerr << "Failed to get remote handles for tagged entities:" << std::endl;
03616             tagged_entities.print( "  " );
03617         }
03618         MB_SET_ERR( result, "Failed to get remote handles for tagged entities" );
03619     }
03620 
03621     PACK_EH( buff->buff_ptr, &dum_tagged_entities[0], dum_tagged_entities.size() );
03622 
03623     const size_t num_ent = tagged_entities.size();
03624     if( src_tag->get_size() == MB_VARIABLE_LENGTH )
03625     {
03626         var_len_sizes.resize( num_ent, 0 );
03627         var_len_values.resize( num_ent, 0 );
03628         result = mbImpl->tag_get_by_ptr( src_tag, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get variable-length tag data in pack_tags" );
03629         buff->check_space( num_ent * sizeof( int ) );
03630         PACK_INTS( buff->buff_ptr, &var_len_sizes[0], num_ent );
03631         for( unsigned int i = 0; i < num_ent; i++ )
03632         {
03633             buff->check_space( var_len_sizes[i] );
03634             PACK_VOID( buff->buff_ptr, var_len_values[i], type_size * var_len_sizes[i] );
03635         }
03636     }
03637     else
03638     {
03639         buff->check_space( num_ent * src_tag->get_size() );
03640         // Should be OK to read directly into buffer, since tags are untyped and
03641         // handled by memcpy
03642         result = mbImpl->tag_get_data( src_tag, tagged_entities, buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get tag data in pack_tags" );
03643         buff->buff_ptr += num_ent * src_tag->get_size();
03644         PC( num_ent * src_tag->get_size(), " void" );
03645     }
03646 
03647     return MB_SUCCESS;
03648 }
03649 
03650 ErrorCode ParallelComm::get_tag_send_list( const Range& whole_range,
03651                                            std::vector< Tag >& all_tags,
03652                                            std::vector< Range >& tag_ranges )
03653 {
03654     std::vector< Tag > tmp_tags;
03655     ErrorCode result = mbImpl->tag_get_tags( tmp_tags );MB_CHK_SET_ERR( result, "Failed to get tags in pack_tags" );
03656 
03657     std::vector< Tag >::iterator tag_it;
03658     for( tag_it = tmp_tags.begin(); tag_it != tmp_tags.end(); ++tag_it )
03659     {
03660         std::string tag_name;
03661         result = mbImpl->tag_get_name( *tag_it, tag_name );
03662         if( tag_name.c_str()[0] == '_' && tag_name.c_str()[1] == '_' ) continue;
03663 
03664         Range tmp_range;
03665         result = ( *tag_it )->get_tagged_entities( sequenceManager, tmp_range );MB_CHK_SET_ERR( result, "Failed to get entities for tag in pack_tags" );
03666         tmp_range = intersect( tmp_range, whole_range );
03667 
03668         if( tmp_range.empty() ) continue;
03669 
03670         // OK, we'll be sending this tag
03671         all_tags.push_back( *tag_it );
03672         tag_ranges.push_back( Range() );
03673         tag_ranges.back().swap( tmp_range );
03674     }
03675 
03676     return MB_SUCCESS;
03677 }
03678 
03679 ErrorCode ParallelComm::unpack_tags( unsigned char*& buff_ptr,
03680                                      std::vector< EntityHandle >& entities,
03681                                      const bool /*store_remote_handles*/,
03682                                      const int /*from_proc*/,
03683                                      const MPI_Op* const mpi_op )
03684 {
03685     // Tags
03686     // Get all the tags
03687     // For dense tags, compute size assuming all entities have that tag
03688     // For sparse tags, get number of entities w/ that tag to compute size
03689 
03690     ErrorCode result;
03691 
03692     int num_tags;
03693     UNPACK_INT( buff_ptr, num_tags );
03694     std::vector< const void* > var_len_vals;
03695     std::vector< unsigned char > dum_vals;
03696     std::vector< EntityHandle > dum_ehvals;
03697 
03698     for( int i = 0; i < num_tags; i++ )
03699     {
03700         // Tag handle
03701         Tag tag_handle;
03702 
03703         // Size, data type
03704         int tag_size, tag_data_type, tag_type;
03705         UNPACK_INT( buff_ptr, tag_size );
03706         UNPACK_INT( buff_ptr, tag_type );
03707         UNPACK_INT( buff_ptr, tag_data_type );
03708 
03709         // Default value
03710         int def_val_size;
03711         UNPACK_INT( buff_ptr, def_val_size );
03712         void* def_val_ptr = NULL;
03713         if( def_val_size )
03714         {
03715             def_val_ptr = buff_ptr;
03716             buff_ptr += def_val_size;
03717             UPC( tag_size, " void" );
03718         }
03719 
03720         // Name
03721         int name_len;
03722         UNPACK_INT( buff_ptr, name_len );
03723         std::string tag_name( reinterpret_cast< char* >( buff_ptr ), name_len );
03724         buff_ptr += name_len;
03725         UPC( 64, " chars" );
03726 
03727         myDebug->tprintf( 4, "Unpacking tag %s\n", tag_name.c_str() );
03728 
03729         // Create the tag
03730         if( tag_size == MB_VARIABLE_LENGTH )
03731             result = mbImpl->tag_get_handle( tag_name.c_str(), def_val_size, (DataType)tag_data_type, tag_handle,
03732                                              MB_TAG_VARLEN | MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
03733         else
03734             result = mbImpl->tag_get_handle( tag_name.c_str(), tag_size, (DataType)tag_data_type, tag_handle,
03735                                              MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
03736         if( MB_SUCCESS != result ) return result;
03737 
03738         // Get handles and convert to local handles
03739         int num_ents;
03740         UNPACK_INT( buff_ptr, num_ents );
03741         std::vector< EntityHandle > dum_ents( num_ents );
03742         UNPACK_EH( buff_ptr, &dum_ents[0], num_ents );
03743 
03744         // In this case handles are indices into new entity range; need to convert
03745         // to local handles
03746         result = get_local_handles( &dum_ents[0], num_ents, entities );MB_CHK_SET_ERR( result, "Unable to convert to local handles" );
03747 
03748         // If it's a handle type, also convert tag vals in-place in buffer
03749         if( MB_TYPE_HANDLE == tag_type )
03750         {
03751             dum_ehvals.resize( num_ents );
03752             UNPACK_EH( buff_ptr, &dum_ehvals[0], num_ents );
03753             result = get_local_handles( &dum_ehvals[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for tag vals" );
03754         }
03755 
03756         DataType data_type;
03757         mbImpl->tag_get_data_type( tag_handle, data_type );
03758         int type_size = TagInfo::size_from_data_type( data_type );
03759 
03760         if( !dum_ents.empty() )
03761         {
03762             if( tag_size == MB_VARIABLE_LENGTH )
03763             {
03764                 // Be careful of alignment here. If the integers are aligned
03765                 // in the buffer, we can use them directly. Otherwise we must
03766                 // copy them.
03767                 std::vector< int > var_lengths( num_ents );
03768                 UNPACK_INTS( buff_ptr, &var_lengths[0], num_ents );
03769                 UPC( sizeof( int ) * num_ents, " void" );
03770 
03771                 // Get pointers into buffer for each tag value
03772                 var_len_vals.resize( num_ents );
03773                 for( std::vector< EntityHandle >::size_type j = 0; j < (std::vector< EntityHandle >::size_type)num_ents;
03774                      j++ )
03775                 {
03776                     var_len_vals[j] = buff_ptr;
03777                     buff_ptr += var_lengths[j] * type_size;
03778                     UPC( var_lengths[j], " void" );
03779                 }
03780                 result =
03781                     mbImpl->tag_set_by_ptr( tag_handle, &dum_ents[0], num_ents, &var_len_vals[0], &var_lengths[0] );MB_CHK_SET_ERR( result, "Failed to set tag data when unpacking variable-length tag" );
03782             }
03783             else
03784             {
03785                 // Get existing values of dst tag
03786                 dum_vals.resize( tag_size * num_ents );
03787                 if( mpi_op )
03788                 {
03789                     int tag_length;
03790                     result = mbImpl->tag_get_length( tag_handle, tag_length );MB_CHK_SET_ERR( result, "Failed to get tag length" );
03791                     result = mbImpl->tag_get_data( tag_handle, &dum_ents[0], num_ents, &dum_vals[0] );MB_CHK_SET_ERR( result, "Failed to get existing value of dst tag on entities" );
03792                     result = reduce_void( tag_data_type, *mpi_op, tag_length * num_ents, &dum_vals[0], buff_ptr );MB_CHK_SET_ERR( result, "Failed to perform mpi op on dst tags" );
03793                 }
03794                 result = mbImpl->tag_set_data( tag_handle, &dum_ents[0], num_ents, buff_ptr );MB_CHK_SET_ERR( result, "Failed to set range-based tag data when unpacking tag" );
03795                 buff_ptr += num_ents * tag_size;
03796                 UPC( num_ents * tag_size, " void" );
03797             }
03798         }
03799     }
03800 
03801     myDebug->tprintf( 4, "Done unpacking tags.\n" );
03802 
03803     return MB_SUCCESS;
03804 }
03805 
03806 template < class T >
03807 T LAND( const T& arg1, const T& arg2 )
03808 {
03809     return arg1 && arg2;
03810 }
03811 template < class T >
03812 T LOR( const T& arg1, const T& arg2 )
03813 {
03814     return arg1 || arg2;
03815 }
03816 template < class T >
03817 T LXOR( const T& arg1, const T& arg2 )
03818 {
03819     return ( ( arg1 && !arg2 ) || ( !arg1 && arg2 ) );
03820 }
03821 template < class T >
03822 T MAX( const T& arg1, const T& arg2 )
03823 {
03824     return ( arg1 > arg2 ? arg1 : arg2 );
03825 }
03826 template < class T >
03827 T MIN( const T& arg1, const T& arg2 )
03828 {
03829     return ( arg1 < arg2 ? arg1 : arg2 );
03830 }
03831 template < class T >
03832 T ADD( const T& arg1, const T& arg2 )
03833 {
03834     return arg1 + arg2;
03835 }
03836 template < class T >
03837 T MULT( const T& arg1, const T& arg2 )
03838 {
03839     return arg1 * arg2;
03840 }
03841 
03842 template < class T >
03843 ErrorCode ParallelComm::reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals )
03844 {
03845     T* old_tmp = reinterpret_cast< T* >( old_vals );
03846     // T *new_tmp = reinterpret_cast<T*>(new_vals);
03847     // new vals pointer needs to be aligned , some compilers will optimize and will shift
03848 
03849     std::vector< T > new_values;
03850     new_values.resize( num_ents );
03851     memcpy( &new_values[0], new_vals, num_ents * sizeof( T ) );
03852     T* new_tmp = &new_values[0];
03853 
03854     if( mpi_op == MPI_SUM )
03855         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, ADD< T > );
03856     else if( mpi_op == MPI_PROD )
03857         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MULT< T > );
03858     else if( mpi_op == MPI_MAX )
03859         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MAX< T > );
03860     else if( mpi_op == MPI_MIN )
03861         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MIN< T > );
03862     else if( mpi_op == MPI_LAND )
03863         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LAND< T > );
03864     else if( mpi_op == MPI_LOR )
03865         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LOR< T > );
03866     else if( mpi_op == MPI_LXOR )
03867         std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LXOR< T > );
03868     else if( mpi_op == MPI_BAND || mpi_op == MPI_BOR || mpi_op == MPI_BXOR )
03869     {
03870         std::cerr << "Bitwise operations not allowed in tag reductions." << std::endl;
03871         return MB_FAILURE;
03872     }
03873     else if( mpi_op != MPI_OP_NULL )
03874     {
03875         std::cerr << "Unknown MPI operation type." << std::endl;
03876         return MB_TYPE_OUT_OF_RANGE;
03877     }
03878 
03879     // copy now the result back where it should be
03880     memcpy( new_vals, new_tmp, num_ents * sizeof( T ) );
03881     std::vector< T >().swap( new_values );  // way to release allocated vector
03882 
03883     return MB_SUCCESS;
03884 }
03885 
03886 ErrorCode ParallelComm::reduce_void( int tag_data_type,
03887                                      const MPI_Op mpi_op,
03888                                      int num_ents,
03889                                      void* old_vals,
03890                                      void* new_vals )
03891 {
03892     ErrorCode result;
03893     switch( tag_data_type )
03894     {
03895         case MB_TYPE_INTEGER:
03896             result = reduce< int >( mpi_op, num_ents, old_vals, new_vals );
03897             break;
03898         case MB_TYPE_DOUBLE:
03899             result = reduce< double >( mpi_op, num_ents, old_vals, new_vals );
03900             break;
03901         case MB_TYPE_BIT:
03902             result = reduce< unsigned char >( mpi_op, num_ents, old_vals, new_vals );
03903             break;
03904         default:
03905             result = MB_SUCCESS;
03906             break;
03907     }
03908 
03909     return result;
03910 }
03911 
03912 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, int resolve_dim, int shared_dim, const Tag* id_tag )
03913 {
03914     ErrorCode result;
03915     Range proc_ents;
03916 
03917     // Check for structured mesh, and do it differently if it is
03918     ScdInterface* scdi;
03919     result = mbImpl->query_interface( scdi );
03920     if( scdi )
03921     {
03922         result = scdi->tag_shared_vertices( this, this_set );
03923         if( MB_SUCCESS == result )
03924         {
03925             myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
03926             return result;
03927         }
03928     }
03929 
03930     if( 0 == this_set )
03931     {
03932         // Get the entities in the partition sets
03933         for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit )
03934         {
03935             Range tmp_ents;
03936             result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true );
03937             if( MB_SUCCESS != result ) return result;
03938             proc_ents.merge( tmp_ents );
03939         }
03940     }
03941     else
03942     {
03943         result = mbImpl->get_entities_by_handle( this_set, proc_ents, true );
03944         if( MB_SUCCESS != result ) return result;
03945     }
03946 
03947     // Resolve dim is maximal dim of entities in proc_ents
03948     if( -1 == resolve_dim )
03949     {
03950         if( !proc_ents.empty() ) resolve_dim = mbImpl->dimension_from_handle( *proc_ents.rbegin() );
03951     }
03952 
03953     // proc_ents should all be of same dimension
03954     if( resolve_dim > shared_dim &&
03955         mbImpl->dimension_from_handle( *proc_ents.rbegin() ) != mbImpl->dimension_from_handle( *proc_ents.begin() ) )
03956     {
03957         Range::iterator lower = proc_ents.lower_bound( CN::TypeDimensionMap[0].first ),
03958                         upper = proc_ents.upper_bound( CN::TypeDimensionMap[resolve_dim - 1].second );
03959         proc_ents.erase( lower, upper );
03960     }
03961 
03962     // Must call even if we don't have any entities, to make sure
03963     // collective comm'n works
03964     return resolve_shared_ents( this_set, proc_ents, resolve_dim, shared_dim, NULL, id_tag );
03965 }
03966 
03967 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set,
03968                                              Range& proc_ents,
03969                                              int resolve_dim,
03970                                              int shared_dim,
03971                                              Range* skin_ents,
03972                                              const Tag* id_tag )
03973 {
03974 #ifdef MOAB_HAVE_MPE
03975     if( myDebug->get_verbosity() == 2 )
03976     {
03977         define_mpe();
03978         MPE_Log_event( RESOLVE_START, procConfig.proc_rank(), "Entering resolve_shared_ents." );
03979     }
03980 #endif
03981 
03982     ErrorCode result;
03983     myDebug->tprintf( 1, "Resolving shared entities.\n" );
03984 
03985     if( resolve_dim < shared_dim )
03986     {
03987         MB_SET_ERR( MB_FAILURE, "MOAB does not support vertex-based partitions, only element-based ones" );
03988     }
03989 
03990     if( -1 == shared_dim )
03991     {
03992         if( !proc_ents.empty() )
03993             shared_dim = mbImpl->dimension_from_handle( *proc_ents.begin() ) - 1;
03994         else if( resolve_dim == 3 )
03995             shared_dim = 2;
03996     }
03997     int max_global_resolve_dim = -1;
03998     int err = MPI_Allreduce( &resolve_dim, &max_global_resolve_dim, 1, MPI_INT, MPI_MAX, proc_config().proc_comm() );
03999     if( MPI_SUCCESS != err )
04000     {
04001         MB_SET_ERR( MB_FAILURE, "Unable to guess global resolve_dim" );
04002     }
04003     if( shared_dim < 0 || resolve_dim < 0 )
04004     {
04005         // MB_SET_ERR(MB_FAILURE, "Unable to guess shared_dim or resolve_dim");
04006         resolve_dim = max_global_resolve_dim;
04007         shared_dim  = resolve_dim - 1;
04008     }
04009 
04010     if( resolve_dim < 0 || shared_dim < 0 ) return MB_SUCCESS;
04011     // no task has any mesh, get out
04012 
04013     // Get the skin entities by dimension
04014     Range tmp_skin_ents[4];
04015 
04016     // Get the entities to be skinned
04017     // Find the skin
04018     int skin_dim = resolve_dim - 1;
04019     if( !skin_ents )
04020     {
04021         skin_ents              = tmp_skin_ents;
04022         skin_ents[resolve_dim] = proc_ents;
04023         Skinner skinner( mbImpl );
04024         result =
04025             skinner.find_skin( this_set, skin_ents[skin_dim + 1], false, skin_ents[skin_dim], NULL, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
04026         myDebug->tprintf( 1, "Found skin:   skin_dim: %d resolve_dim: %d , now resolving.\n", skin_dim, resolve_dim );
04027         myDebug->tprintf( 3, "skin_ents[0].size(): %d skin_ents[1].size(): %d  \n", (int)skin_ents[0].size(),
04028                           (int)skin_ents[1].size() );
04029         // Get entities adjacent to skin ents from shared_dim down to zero
04030         for( int this_dim = skin_dim - 1; this_dim >= 0; this_dim-- )
04031         {
04032             result =
04033                 mbImpl->get_adjacencies( skin_ents[skin_dim], this_dim, true, skin_ents[this_dim], Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
04034 
04035             if( this_set && skin_dim == 2 && this_dim == 1 )
04036             {
04037                 result = mbImpl->add_entities( this_set, skin_ents[this_dim] );MB_CHK_ERR( result );
04038             }
04039         }
04040     }
04041     else if( skin_ents[resolve_dim].empty() )
04042         skin_ents[resolve_dim] = proc_ents;
04043 
04044     // Global id tag
04045     Tag gid_tag;
04046     if( id_tag )
04047         gid_tag = *id_tag;
04048     else
04049     {
04050         bool tag_created = false;
04051         int def_val      = -1;
04052         result = mbImpl->tag_get_handle( GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE | MB_TAG_CREAT,
04053                                          &def_val, &tag_created );
04054         if( MB_ALREADY_ALLOCATED != result && MB_SUCCESS != result )
04055         {
04056             MB_SET_ERR( result, "Failed to create/get gid tag handle" );
04057         }
04058         else if( tag_created )
04059         {
04060             // Just created it, so we need global ids
04061             result = assign_global_ids( this_set, skin_dim + 1, true, true, true );MB_CHK_SET_ERR( result, "Failed to assign global ids" );
04062         }
04063     }
04064 
04065     DataType tag_type;
04066     result = mbImpl->tag_get_data_type( gid_tag, tag_type );MB_CHK_SET_ERR( result, "Failed to get tag data type" );
04067     int bytes_per_tag;
04068     result = mbImpl->tag_get_bytes( gid_tag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed to get number of bytes per tag" );
04069     // On 64 bits, long and int are different
04070     // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?)
04071 
04072     // Get gids for skin ents in a vector, to pass to gs
04073     std::vector< long > lgid_data( skin_ents[0].size() );
04074     // Size is either long or int
04075     // On 64 bit is 8 or 4
04076     if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
04077     {  // It is a special id tag
04078         result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &lgid_data[0] );MB_CHK_SET_ERR( result, "Couldn't get gid tag for skin vertices" );
04079     }
04080     else if( 4 == bytes_per_tag )
04081     {  // Must be GLOBAL_ID tag or 32 bits ...
04082         std::vector< int > gid_data( lgid_data.size() );
04083         result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &gid_data[0] );MB_CHK_SET_ERR( result, "Failed to get gid tag for skin vertices" );
04084         std::copy( gid_data.begin(), gid_data.end(), lgid_data.begin() );
04085     }
04086     else
04087     {
04088         // Not supported flag
04089         MB_SET_ERR( MB_FAILURE, "Unsupported id tag" );
04090     }
04091 
04092     // Put handles in vector for passing to gs setup
04093     std::vector< Ulong > handle_vec;  // Assumes that we can do conversion from Ulong to EntityHandle
04094     std::copy( skin_ents[0].begin(), skin_ents[0].end(), std::back_inserter( handle_vec ) );
04095 
04096 #ifdef MOAB_HAVE_MPE
04097     if( myDebug->get_verbosity() == 2 )
04098     {
04099         MPE_Log_event( SHAREDV_START, procConfig.proc_rank(), "Creating crystal router." );
04100     }
04101 #endif
04102 
04103     // Get a crystal router
04104     gs_data::crystal_data* cd = procConfig.crystal_router();
04105 
04106     /*
04107     // Get total number of entities; will overshoot highest global id, but
04108     // that's OK
04109     int num_total[2] = {0, 0}, num_local[2] = {0, 0};
04110     result = mbImpl->get_number_entities_by_dimension(this_set, 0, num_local);
04111     if (MB_SUCCESS != result)return result;
04112     int failure = MPI_Allreduce(num_local, num_total, 1,
04113     MPI_INT, MPI_SUM, procConfig.proc_comm());
04114     if (failure) {
04115       MB_SET_ERR(MB_FAILURE, "Allreduce for total number of shared ents failed");
04116     }
04117     */
04118     // Call gather-scatter to get shared ids & procs
04119     gs_data* gsd = new gs_data();
04120     // assert(sizeof(ulong_) == sizeof(EntityHandle));
04121     result = gsd->initialize( skin_ents[0].size(), &lgid_data[0], &handle_vec[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
04122 
04123     // Get shared proc tags
04124     Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
04125     result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
04126 
04127     // Load shared verts into a tuple, then sort by index
04128     TupleList shared_verts;
04129     shared_verts.initialize( 2, 0, 1, 0, skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ) );
04130     shared_verts.enableWriteAccess();
04131 
04132     unsigned int i = 0, j = 0;
04133     for( unsigned int p = 0; p < gsd->nlinfo->_np; p++ )
04134         for( unsigned int np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
04135         {
04136             shared_verts.vi_wr[i++] = gsd->nlinfo->_sh_ind[j];
04137             shared_verts.vi_wr[i++] = gsd->nlinfo->_target[p];
04138             shared_verts.vul_wr[j]  = gsd->nlinfo->_ulabels[j];
04139             j++;
04140             shared_verts.inc_n();
04141         }
04142 
04143     myDebug->tprintf( 3, " shared verts size %d \n", (int)shared_verts.get_n() );
04144 
04145     int max_size = skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 );
04146     moab::TupleList::buffer sort_buffer;
04147     sort_buffer.buffer_init( max_size );
04148     shared_verts.sort( 0, &sort_buffer );
04149     sort_buffer.reset();
04150 
04151     // Set sharing procs and handles tags on skin ents
04152     int maxp = -1;
04153     std::vector< int > sharing_procs( MAX_SHARING_PROCS );
04154     std::fill( sharing_procs.begin(), sharing_procs.end(), maxp );
04155     j = 0;
04156     i = 0;
04157 
04158     // Get ents shared by 1 or n procs
04159     std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
04160     Range proc_verts;
04161     result = mbImpl->get_adjacencies( proc_ents, 0, false, proc_verts, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get proc_verts" );
04162 
04163     myDebug->print( 3, " resolve shared ents:  proc verts ", proc_verts );
04164     result = tag_shared_verts( shared_verts, skin_ents, proc_nvecs, proc_verts );MB_CHK_SET_ERR( result, "Failed to tag shared verts" );
04165 
04166 #ifdef MOAB_HAVE_MPE
04167     if( myDebug->get_verbosity() == 2 )
04168     {
04169         MPE_Log_event( SHAREDV_END, procConfig.proc_rank(), "Finished tag_shared_verts." );
04170     }
04171 #endif
04172 
04173     // Get entities shared by 1 or n procs
04174     result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );MB_CHK_SET_ERR( result, "Failed to tag shared entities" );
04175 
04176     shared_verts.reset();
04177 
04178     if( myDebug->get_verbosity() > 0 )
04179     {
04180         for( std::map< std::vector< int >, std::vector< EntityHandle > >::const_iterator mit = proc_nvecs.begin();
04181              mit != proc_nvecs.end(); ++mit )
04182         {
04183             myDebug->tprintf( 1, "Iface: " );
04184             for( std::vector< int >::const_iterator vit = ( mit->first ).begin(); vit != ( mit->first ).end(); ++vit )
04185                 myDebug->printf( 1, " %d", *vit );
04186             myDebug->print( 1, "\n" );
04187         }
04188     }
04189 
04190     // Create the sets for each interface; store them as tags on
04191     // the interface instance
04192     Range iface_sets;
04193     result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" );
04194 
04195     // Establish comm procs and buffers for them
04196     std::set< unsigned int > procs;
04197     result = get_interface_procs( procs, true );MB_CHK_SET_ERR( result, "Failed to get interface procs" );
04198 
04199 #ifndef NDEBUG
04200     result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Shared handle check failed after interface vertex exchange" );
04201 #endif
04202 
04203     // Resolve shared entity remote handles; implemented in ghost cell exchange
04204     // code because it's so similar
04205     result = exchange_ghost_cells( -1, -1, 0, 0, true, true );MB_CHK_SET_ERR( result, "Failed to resolve shared entity remote handles" );
04206 
04207     // Now build parent/child links for interface sets
04208     result = create_iface_pc_links();MB_CHK_SET_ERR( result, "Failed to create interface parent/child links" );
04209 
04210     gsd->reset();
04211     delete gsd;
04212 
04213 #ifdef MOAB_HAVE_MPE
04214     if( myDebug->get_verbosity() == 2 )
04215     {
04216         MPE_Log_event( RESOLVE_END, procConfig.proc_rank(), "Exiting resolve_shared_ents." );
04217     }
04218 #endif
04219 
04220     // std::ostringstream ent_str;
04221     // ent_str << "mesh." << procConfig.proc_rank() << ".h5m";
04222     // mbImpl->write_mesh(ent_str.str().c_str());
04223 
04224     // Done
04225     return result;
04226 }
04227 
04228 void ParallelComm::define_mpe()
04229 {
04230 #ifdef MOAB_HAVE_MPE
04231     if( myDebug->get_verbosity() == 2 )
04232     {
04233         // Define mpe states used for logging
04234         int success;
04235         MPE_Log_get_state_eventIDs( &IFACE_START, &IFACE_END );
04236         MPE_Log_get_state_eventIDs( &GHOST_START, &GHOST_END );
04237         MPE_Log_get_state_eventIDs( &SHAREDV_START, &SHAREDV_END );
04238         MPE_Log_get_state_eventIDs( &RESOLVE_START, &RESOLVE_END );
04239         MPE_Log_get_state_eventIDs( &ENTITIES_START, &ENTITIES_END );
04240         MPE_Log_get_state_eventIDs( &RHANDLES_START, &RHANDLES_END );
04241         MPE_Log_get_state_eventIDs( &OWNED_START, &OWNED_END );
04242         success = MPE_Describe_state( IFACE_START, IFACE_END, "Resolve interface ents", "green" );
04243         assert( MPE_LOG_OK == success );
04244         success = MPE_Describe_state( GHOST_START, GHOST_END, "Exchange ghost ents", "red" );
04245         assert( MPE_LOG_OK == success );
04246         success = MPE_Describe_state( SHAREDV_START, SHAREDV_END, "Resolve interface vertices", "blue" );
04247         assert( MPE_LOG_OK == success );
04248         success = MPE_Describe_state( RESOLVE_START, RESOLVE_END, "Resolve shared ents", "purple" );
04249         assert( MPE_LOG_OK == success );
04250         success = MPE_Describe_state( ENTITIES_START, ENTITIES_END, "Exchange shared ents", "yellow" );
04251         assert( MPE_LOG_OK == success );
04252         success = MPE_Describe_state( RHANDLES_START, RHANDLES_END, "Remote handles", "cyan" );
04253         assert( MPE_LOG_OK == success );
04254         success = MPE_Describe_state( OWNED_START, OWNED_END, "Exchange owned ents", "black" );
04255         assert( MPE_LOG_OK == success );
04256     }
04257 #endif
04258 }
04259 
04260 ErrorCode ParallelComm::resolve_shared_ents( ParallelComm** pc,
04261                                              const unsigned int np,
04262                                              EntityHandle this_set,
04263                                              const int part_dim )
04264 {
04265     std::vector< Range > verts( np );
04266     int tot_verts = 0;
04267     unsigned int p, i, j, v;
04268     ErrorCode rval;
04269     for( p = 0; p < np; p++ )
04270     {
04271         Skinner skinner( pc[p]->get_moab() );
04272         Range part_ents, skin_ents;
04273         rval = pc[p]->get_moab()->get_entities_by_dimension( this_set, part_dim, part_ents );
04274         if( MB_SUCCESS != rval ) return rval;
04275         rval = skinner.find_skin( this_set, part_ents, false, skin_ents, 0, true, true, true );
04276         if( MB_SUCCESS != rval ) return rval;
04277         rval = pc[p]->get_moab()->get_adjacencies( skin_ents, 0, true, verts[p], Interface::UNION );
04278         if( MB_SUCCESS != rval ) return rval;
04279         tot_verts += verts[p].size();
04280     }
04281 
04282     TupleList shared_ents;
04283     shared_ents.initialize( 2, 0, 1, 0, tot_verts );
04284     shared_ents.enableWriteAccess();
04285 
04286     i = 0;
04287     j = 0;
04288     std::vector< int > gids;
04289     Range::iterator rit;
04290     Tag gid_tag;
04291     for( p = 0; p < np; p++ )
04292     {
04293         gid_tag = pc[p]->get_moab()->globalId_tag();
04294 
04295         gids.resize( verts[p].size() );
04296         rval = pc[p]->get_moab()->tag_get_data( gid_tag, verts[p], &gids[0] );
04297         if( MB_SUCCESS != rval ) return rval;
04298 
04299         for( v = 0, rit = verts[p].begin(); v < gids.size(); v++, ++rit )
04300         {
04301             shared_ents.vi_wr[i++] = gids[v];
04302             shared_ents.vi_wr[i++] = p;
04303             shared_ents.vul_wr[j]  = *rit;
04304             j++;
04305             shared_ents.inc_n();
04306         }
04307     }
04308 
04309     moab::TupleList::buffer sort_buffer;
04310     sort_buffer.buffer_init( tot_verts );
04311     shared_ents.sort( 0, &sort_buffer );
04312     sort_buffer.reset();
04313 
04314     j = 0;
04315     i = 0;
04316     std::vector< EntityHandle > handles;
04317     std::vector< int > procs;
04318 
04319     while( i < shared_ents.get_n() )
04320     {
04321         handles.clear();
04322         procs.clear();
04323 
04324         // Count & accumulate sharing procs
04325         int this_gid = shared_ents.vi_rd[j];
04326         while( i < shared_ents.get_n() && shared_ents.vi_rd[j] == this_gid )
04327         {
04328             j++;
04329             procs.push_back( shared_ents.vi_rd[j++] );
04330             handles.push_back( shared_ents.vul_rd[i++] );
04331         }
04332         if( 1 == procs.size() ) continue;
04333 
04334         for( v = 0; v < procs.size(); v++ )
04335         {
04336             rval = pc[procs[v]]->update_remote_data( handles[v], &procs[0], &handles[0], procs.size(),
04337                                                      ( procs[0] == (int)pc[procs[v]]->rank()
04338                                                            ? PSTATUS_INTERFACE
04339                                                            : ( PSTATUS_NOT_OWNED | PSTATUS_INTERFACE ) ) );
04340             if( MB_SUCCESS != rval ) return rval;
04341         }
04342     }
04343 
04344     std::set< unsigned int > psets;
04345     for( p = 0; p < np; p++ )
04346     {
04347         rval = pc[p]->create_interface_sets( this_set, part_dim, part_dim - 1 );
04348         if( MB_SUCCESS != rval ) return rval;
04349         // Establish comm procs and buffers for them
04350         psets.clear();
04351         rval = pc[p]->get_interface_procs( psets, true );
04352         if( MB_SUCCESS != rval ) return rval;
04353     }
04354 
04355     shared_ents.reset();
04356 
04357     return MB_SUCCESS;
04358 }
04359 
04360 ErrorCode ParallelComm::tag_iface_entities()
04361 {
04362     ErrorCode result = MB_SUCCESS;
04363     Range iface_ents, tmp_ents, rmv_ents;
04364     std::vector< unsigned char > pstat;
04365     unsigned char set_pstat;
04366     Range::iterator rit2;
04367     unsigned int i;
04368 
04369     for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
04370     {
04371         iface_ents.clear();
04372 
04373         result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get interface set contents" );
04374         pstat.resize( iface_ents.size() );
04375         result = mbImpl->tag_get_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set entities" );
04376         result = mbImpl->tag_get_data( pstatus_tag(), &( *rit ), 1, &set_pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set" );
04377         rmv_ents.clear();
04378         for( rit2 = iface_ents.begin(), i = 0; rit2 != iface_ents.end(); ++rit2, i++ )
04379         {
04380             if( !( pstat[i] & PSTATUS_INTERFACE ) )
04381             {
04382                 rmv_ents.insert( *rit2 );
04383                 pstat[i] = 0x0;
04384             }
04385         }
04386         result = mbImpl->remove_entities( *rit, rmv_ents );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
04387 
04388         if( !( set_pstat & PSTATUS_NOT_OWNED ) ) continue;
04389         // If we're here, we need to set the notowned status on (remaining) set contents
04390 
04391         // Remove rmv_ents from the contents list
04392         iface_ents = subtract( iface_ents, rmv_ents );
04393         // Compress the pstat vector (removing 0x0's)
04394         std::remove_if( pstat.begin(), pstat.end(),
04395                         std::bind( std::equal_to< unsigned char >(), std::placeholders::_1, 0x0 ) );
04396         // std::bind2nd(std::equal_to<unsigned char>(), 0x0));
04397         // https://stackoverflow.com/questions/32739018/a-replacement-for-stdbind2nd
04398         // Fold the not_owned bit into remaining values
04399         unsigned int sz = iface_ents.size();
04400         for( i = 0; i < sz; i++ )
04401             pstat[i] |= PSTATUS_NOT_OWNED;
04402 
04403         // Set the tag on the entities
04404         result = mbImpl->tag_set_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus values for interface set entities" );
04405     }
04406 
04407     return MB_SUCCESS;
04408 }
04409 
04410 ErrorCode ParallelComm::set_pstatus_entities( Range& pstatus_ents,
04411                                               unsigned char pstatus_val,
04412                                               bool lower_dim_ents,
04413                                               bool verts_too,
04414                                               int operation )
04415 {
04416     std::vector< unsigned char > pstatus_vals( pstatus_ents.size() );
04417     Range all_ents, *range_ptr = &pstatus_ents;
04418     ErrorCode result;
04419     if( lower_dim_ents || verts_too )
04420     {
04421         all_ents      = pstatus_ents;
04422         range_ptr     = &all_ents;
04423         int start_dim = ( lower_dim_ents ? mbImpl->dimension_from_handle( *pstatus_ents.rbegin() ) - 1 : 0 );
04424         for( ; start_dim >= 0; start_dim-- )
04425         {
04426             result = mbImpl->get_adjacencies( all_ents, start_dim, true, all_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get adjacencies for pstatus entities" );
04427         }
04428     }
04429     if( Interface::UNION == operation )
04430     {
04431         result = mbImpl->tag_get_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
04432         for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
04433             pstatus_vals[i] |= pstatus_val;
04434     }
04435     else
04436     {
04437         for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
04438             pstatus_vals[i] = pstatus_val;
04439     }
04440     result = mbImpl->tag_set_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
04441 
04442     return MB_SUCCESS;
04443 }
04444 
04445 ErrorCode ParallelComm::set_pstatus_entities( EntityHandle* pstatus_ents,
04446                                               int num_ents,
04447                                               unsigned char pstatus_val,
04448                                               bool lower_dim_ents,
04449                                               bool verts_too,
04450                                               int operation )
04451 {
04452     std::vector< unsigned char > pstatus_vals( num_ents );
04453     ErrorCode result;
04454     if( lower_dim_ents || verts_too )
04455     {
04456         // In this case, call the range-based version
04457         Range tmp_range;
04458         std::copy( pstatus_ents, pstatus_ents + num_ents, range_inserter( tmp_range ) );
04459         return set_pstatus_entities( tmp_range, pstatus_val, lower_dim_ents, verts_too, operation );
04460     }
04461 
04462     if( Interface::UNION == operation )
04463     {
04464         result = mbImpl->tag_get_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
04465         for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
04466             pstatus_vals[i] |= pstatus_val;
04467     }
04468     else
04469     {
04470         for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
04471             pstatus_vals[i] = pstatus_val;
04472     }
04473     result = mbImpl->tag_set_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
04474 
04475     return MB_SUCCESS;
04476 }
04477 
04478 static size_t choose_owner_idx( const std::vector< unsigned >& proc_list )
04479 {
04480     // Try to assign owners randomly so we get a good distribution,
04481     // (note: specifying the same seed on all procs is essential)
04482     unsigned val = 0;
04483     for( size_t i = 0; i < proc_list.size(); i++ )
04484         val ^= proc_list[i];
04485     srand( (int)( val ) );
04486     return rand() % proc_list.size();
04487 }
04488 
04489 struct set_tuple
04490 {
04491     unsigned idx;
04492     unsigned proc;
04493     EntityHandle handle;
04494     inline bool operator<( set_tuple other ) const
04495     {
04496         return ( idx == other.idx ) ? ( proc < other.proc ) : ( idx < other.idx );
04497     }
04498 };
04499 
04500 ErrorCode ParallelComm::resolve_shared_sets( EntityHandle file, const Tag* idtag )
04501 {
04502     // Find all sets with any of the following tags:
04503     const char* const shared_set_tag_names[] = { GEOM_DIMENSION_TAG_NAME, MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME,
04504                                                  NEUMANN_SET_TAG_NAME, PARALLEL_PARTITION_TAG_NAME };
04505     int num_tags                             = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
04506     Range candidate_sets;
04507     ErrorCode result = MB_FAILURE;
04508 
04509     // If we're not given an ID tag to use to globally identify sets,
04510     // then fall back to using known tag values
04511     if( !idtag )
04512     {
04513         Tag gid, tag;
04514         gid = mbImpl->globalId_tag();
04515         if( NULL != gid ) result = mbImpl->tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tag );
04516         if( MB_SUCCESS == result )
04517         {
04518             for( int d = 0; d < 4; d++ )
04519             {
04520                 candidate_sets.clear();
04521                 const void* vals[] = { &d };
04522                 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, vals, 1, candidate_sets );
04523                 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, gid );
04524             }
04525         }
04526 
04527         for( int i = 1; i < num_tags; i++ )
04528         {
04529             result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag );
04530             if( MB_SUCCESS == result )
04531             {
04532                 candidate_sets.clear();
04533                 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets );
04534                 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, tag );
04535             }
04536         }
04537 
04538         return MB_SUCCESS;
04539     }
04540 
04541     for( int i = 0; i < num_tags; i++ )
04542     {
04543         Tag tag;
04544         result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag, MB_TAG_ANY );
04545         if( MB_SUCCESS != result ) continue;
04546 
04547         mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets, Interface::UNION );
04548     }
04549 
04550     // Find any additional sets that contain shared entities
04551     Range::iterator hint = candidate_sets.begin();
04552     Range all_sets;
04553     mbImpl->get_entities_by_type( file, MBENTITYSET, all_sets );
04554     all_sets           = subtract( all_sets, candidate_sets );
04555     Range::iterator it = all_sets.begin();
04556     while( it != all_sets.end() )
04557     {
04558         Range contents;
04559         mbImpl->get_entities_by_handle( *it, contents );
04560         contents.erase( contents.lower_bound( MBENTITYSET ), contents.end() );
04561         filter_pstatus( contents, PSTATUS_SHARED, PSTATUS_OR );
04562         if( contents.empty() )
04563         {
04564             ++it;
04565         }
04566         else
04567         {
04568             hint = candidate_sets.insert( hint, *it );
04569             it   = all_sets.erase( it );
04570         }
04571     }
04572 
04573     // Find any additionl sets that contain or are parents of potential shared sets
04574     Range prev_list = candidate_sets;
04575     while( !prev_list.empty() )
04576     {
04577         it = all_sets.begin();
04578         Range new_list;
04579         hint = new_list.begin();
04580         while( it != all_sets.end() )
04581         {
04582             Range contents;
04583             mbImpl->get_entities_by_type( *it, MBENTITYSET, contents );
04584             if( !intersect( prev_list, contents ).empty() )
04585             {
04586                 hint = new_list.insert( hint, *it );
04587                 it   = all_sets.erase( it );
04588             }
04589             else
04590             {
04591                 new_list.clear();
04592                 mbImpl->get_child_meshsets( *it, contents );
04593                 if( !intersect( prev_list, contents ).empty() )
04594                 {
04595                     hint = new_list.insert( hint, *it );
04596                     it   = all_sets.erase( it );
04597                 }
04598                 else
04599                 {
04600                     ++it;
04601                 }
04602             }
04603         }
04604 
04605         candidate_sets.merge( new_list );
04606         prev_list.swap( new_list );
04607     }
04608 
04609     return resolve_shared_sets( candidate_sets, *idtag );
04610 }
04611 
04612 #ifndef NDEBUG
04613 bool is_sorted_unique( std::vector< unsigned >& v )
04614 {
04615     for( size_t i = 1; i < v.size(); i++ )
04616         if( v[i - 1] >= v[i] ) return false;
04617     return true;
04618 }
04619 #endif
04620 
04621 ErrorCode ParallelComm::resolve_shared_sets( Range& sets, Tag idtag )
04622 {
04623     ErrorCode result;
04624     const unsigned rk = proc_config().proc_rank();
04625     MPI_Comm cm       = proc_config().proc_comm();
04626 
04627     // Build sharing list for all sets
04628 
04629     // Get ids for sets in a vector, to pass to gs
04630     std::vector< long > larray;  // Allocate sufficient space for longs
04631     std::vector< Ulong > handles;
04632     Range tmp_sets;
04633     // The id tag can be size 4 or size 8
04634     // Based on that, convert to int or to long, similarly to what we do
04635     // for resolving shared vertices;
04636     // This code must work on 32 bit too, where long is 4 bytes, also
04637     // so test first size 4, then we should be fine
04638     DataType tag_type;
04639     result = mbImpl->tag_get_data_type( idtag, tag_type );MB_CHK_SET_ERR( result, "Failed getting tag data type" );
04640     int bytes_per_tag;
04641     result = mbImpl->tag_get_bytes( idtag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed getting number of bytes per tag" );
04642     // On 64 bits, long and int are different
04643     // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?)
04644 
04645     for( Range::iterator rit = sets.begin(); rit != sets.end(); ++rit )
04646     {
04647         if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
04648         {  // It is a special id tag
04649             long dum;
04650             result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
04651             if( MB_SUCCESS == result )
04652             {
04653                 larray.push_back( dum );
04654                 handles.push_back( *rit );
04655                 tmp_sets.insert( tmp_sets.end(), *rit );
04656             }
04657         }
04658         else if( 4 == bytes_per_tag )
04659         {  // Must be GLOBAL_ID tag or MATERIAL_ID, etc
04660             int dum;
04661             result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
04662             if( MB_SUCCESS == result )
04663             {
04664                 larray.push_back( dum );
04665                 handles.push_back( *rit );
04666                 tmp_sets.insert( tmp_sets.end(), *rit );
04667             }
04668         }
04669     }
04670 
04671     const size_t nsets = handles.size();
04672 
04673     // Get handle array for sets
04674     // This is not true on windows machine, 64 bits: entity handle is 64 bit, long is 32
04675     // assert(sizeof(EntityHandle) <= sizeof(unsigned long));
04676 
04677     // Do communication of data
04678     gs_data::crystal_data* cd = procConfig.crystal_router();
04679     gs_data* gsd              = new gs_data();
04680     result                    = gsd->initialize( nsets, &larray[0], &handles[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
04681 
04682     // Convert from global IDs grouped by process rank to list
04683     // of <idx, rank> pairs so that we can sort primarily
04684     // by idx and secondarily by rank (we want lists of procs for each
04685     // idx, not lists if indices for each proc).
04686     size_t ntuple = 0;
04687     for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
04688         ntuple += gsd->nlinfo->_nshared[p];
04689     std::vector< set_tuple > tuples;
04690     tuples.reserve( ntuple );
04691     size_t j = 0;
04692     for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
04693     {
04694         for( unsigned np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
04695         {
04696             set_tuple t;
04697             t.idx    = gsd->nlinfo->_sh_ind[j];
04698             t.proc   = gsd->nlinfo->_target[p];
04699             t.handle = gsd->nlinfo->_ulabels[j];
04700             tuples.push_back( t );
04701             j++;
04702         }
04703     }
04704     std::sort( tuples.begin(), tuples.end() );
04705 
04706     // Release crystal router stuff
04707     gsd->reset();
04708     delete gsd;
04709 
04710     // Storing sharing data for each set
04711     size_t ti    = 0;
04712     unsigned idx = 0;
04713     std::vector< unsigned > procs;
04714     Range::iterator si = tmp_sets.begin();
04715     while( si != tmp_sets.end() && ti < tuples.size() )
04716     {
04717         assert( idx <= tuples[ti].idx );
04718         if( idx < tuples[ti].idx ) si += ( tuples[ti].idx - idx );
04719         idx = tuples[ti].idx;
04720 
04721         procs.clear();
04722         size_t ti_init = ti;
04723         while( ti < tuples.size() && tuples[ti].idx == idx )
04724         {
04725             procs.push_back( tuples[ti].proc );
04726             ++ti;
04727         }
04728         assert( is_sorted_unique( procs ) );
04729 
04730         result = sharedSetData->set_sharing_procs( *si, procs );
04731         if( MB_SUCCESS != result )
04732         {
04733             std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
04734             std::cerr.flush();
04735             MPI_Abort( cm, 1 );
04736         }
04737 
04738         // Add this proc to list of sharing procs in correct position
04739         // so that all procs select owner based on same list
04740         std::vector< unsigned >::iterator it = std::lower_bound( procs.begin(), procs.end(), rk );
04741         assert( it == procs.end() || *it > rk );
04742         procs.insert( it, rk );
04743         size_t owner_idx = choose_owner_idx( procs );
04744         EntityHandle owner_handle;
04745         if( procs[owner_idx] == rk )
04746             owner_handle = *si;
04747         else if( procs[owner_idx] > rk )
04748             owner_handle = tuples[ti_init + owner_idx - 1].handle;
04749         else
04750             owner_handle = tuples[ti_init + owner_idx].handle;
04751         result = sharedSetData->set_owner( *si, procs[owner_idx], owner_handle );
04752         if( MB_SUCCESS != result )
04753         {
04754             std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
04755             std::cerr.flush();
04756             MPI_Abort( cm, 1 );
04757         }
04758 
04759         ++si;
04760         ++idx;
04761     }
04762 
04763     return MB_SUCCESS;
04764 }
04765 // populate sets with ghost entities, if necessary
04766 ErrorCode ParallelComm::augment_default_sets_with_ghosts( EntityHandle file_set )
04767 {
04768     // gather all default sets we are interested in, material, neumann, etc
04769     // we will skip geometry sets, because they are not uniquely identified with their tag value
04770     // maybe we will add another tag, like category
04771 
04772     if( procConfig.proc_size() < 2 ) return MB_SUCCESS;  // no reason to stop by
04773     const char* const shared_set_tag_names[] = { MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, NEUMANN_SET_TAG_NAME,
04774                                                  PARALLEL_PARTITION_TAG_NAME };
04775 
04776     int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
04777 
04778     Range* rangeSets = new Range[num_tags];
04779     Tag* tags        = new Tag[num_tags + 1];  // one extra for global id tag, which is an int, so far
04780 
04781     int my_rank   = rank();
04782     int** tagVals = new int*[num_tags];
04783     for( int i = 0; i < num_tags; i++ )
04784         tagVals[i] = NULL;
04785     ErrorCode rval;
04786 
04787     // for each tag, we keep a local map, from the value to the actual set with that value
04788     // we assume that the tag values are unique, for a given set, otherwise we
04789     // do not know to which set to add the entity
04790 
04791     typedef std::map< int, EntityHandle > MVal;
04792     typedef std::map< int, EntityHandle >::iterator itMVal;
04793     MVal* localMaps = new MVal[num_tags];
04794 
04795     for( int i = 0; i < num_tags; i++ )
04796     {
04797 
04798         rval = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tags[i], MB_TAG_ANY );
04799         if( MB_SUCCESS != rval ) continue;
04800         rval = mbImpl->get_entities_by_type_and_tag( file_set, MBENTITYSET, &( tags[i] ), 0, 1, rangeSets[i],
04801                                                      Interface::UNION );MB_CHK_SET_ERR( rval, "can't get sets with a tag" );
04802 
04803         if( rangeSets[i].size() > 0 )
04804         {
04805             tagVals[i] = new int[rangeSets[i].size()];
04806             // fill up with the tag values
04807             rval = mbImpl->tag_get_data( tags[i], rangeSets[i], tagVals[i] );MB_CHK_SET_ERR( rval, "can't get set tag values" );
04808             // now for inverse mapping:
04809             for( int j = 0; j < (int)rangeSets[i].size(); j++ )
04810             {
04811                 localMaps[i][tagVals[i][j]] = rangeSets[i][j];
04812             }
04813         }
04814     }
04815     // get the global id tag too
04816     tags[num_tags] = mbImpl->globalId_tag();
04817 
04818     TupleList remoteEnts;
04819     // processor to send to, type of tag (0-mat,) tag value,     remote handle
04820     //                         1-diri
04821     //                         2-neum
04822     //                         3-part
04823     //
04824     int initialSize = (int)sharedEnts.size();  // estimate that on average, each shared ent
04825     // will be sent to one processor, for one tag
04826     // we will actually send only entities that are owned locally, and from those
04827     // only those that do have a special tag (material, neumann, etc)
04828     // if we exceed the capacity, we resize the tuple
04829     remoteEnts.initialize( 3, 0, 1, 0, initialSize );
04830     remoteEnts.enableWriteAccess();
04831 
04832     // now, for each owned entity, get the remote handle(s) and Proc(s), and verify if it
04833     // belongs to one of the sets; if yes, create a tuple and append it
04834 
04835     std::set< EntityHandle > own_and_sha;
04836     int ir = 0, jr = 0;
04837     for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
04838     {
04839         // ghosted eh
04840         EntityHandle geh = *vit;
04841         if( own_and_sha.find( geh ) != own_and_sha.end() )  // already encountered
04842             continue;
04843         int procs[MAX_SHARING_PROCS];
04844         EntityHandle handles[MAX_SHARING_PROCS];
04845         int nprocs;
04846         unsigned char pstat;
04847         rval = get_sharing_data( geh, procs, handles, pstat, nprocs );
04848         if( rval != MB_SUCCESS )
04849         {
04850             for( int i = 0; i < num_tags; i++ )
04851                 delete[] tagVals[i];
04852             delete[] tagVals;
04853 
04854             MB_CHK_SET_ERR( rval, "Failed to get sharing data" );
04855         }
04856         if( pstat & PSTATUS_NOT_OWNED ) continue;  // we will send info only for entities that we own
04857         own_and_sha.insert( geh );
04858         for( int i = 0; i < num_tags; i++ )
04859         {
04860             for( int j = 0; j < (int)rangeSets[i].size(); j++ )
04861             {
04862                 EntityHandle specialSet = rangeSets[i][j];  // this set has tag i, value tagVals[i][j];
04863                 if( mbImpl->contains_entities( specialSet, &geh, 1 ) )
04864                 {
04865                     // this ghosted entity is in a special set, so form the tuple
04866                     // to send to the processors that do not own this
04867                     for( int k = 0; k < nprocs; k++ )
04868                     {
04869                         if( procs[k] != my_rank )
04870                         {
04871                             if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
04872                             {
04873                                 // resize, so we do not overflow
04874                                 int oldSize = remoteEnts.get_max();
04875                                 // increase with 50% the capacity
04876                                 remoteEnts.resize( oldSize + oldSize / 2 + 1 );
04877                             }
04878                             remoteEnts.vi_wr[ir++]  = procs[k];       // send to proc
04879                             remoteEnts.vi_wr[ir++]  = i;              // for the tags [i] (0-3)
04880                             remoteEnts.vi_wr[ir++]  = tagVals[i][j];  // actual value of the tag
04881                             remoteEnts.vul_wr[jr++] = handles[k];
04882                             remoteEnts.inc_n();
04883                         }
04884                     }
04885                 }
04886             }
04887         }
04888         // if the local entity has a global id, send it too, so we avoid
04889         // another "exchange_tags" for global id
04890         int gid;
04891         rval = mbImpl->tag_get_data( tags[num_tags], &geh, 1, &gid );MB_CHK_SET_ERR( rval, "Failed to get global id" );
04892         if( gid != 0 )
04893         {
04894             for( int k = 0; k < nprocs; k++ )
04895             {
04896                 if( procs[k] != my_rank )
04897                 {
04898                     if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
04899                     {
04900                         // resize, so we do not overflow
04901                         int oldSize = remoteEnts.get_max();
04902                         // increase with 50% the capacity
04903                         remoteEnts.resize( oldSize + oldSize / 2 + 1 );
04904                     }
04905                     remoteEnts.vi_wr[ir++]  = procs[k];  // send to proc
04906                     remoteEnts.vi_wr[ir++]  = num_tags;  // for the tags [j] (4)
04907                     remoteEnts.vi_wr[ir++]  = gid;       // actual value of the tag
04908                     remoteEnts.vul_wr[jr++] = handles[k];
04909                     remoteEnts.inc_n();
04910                 }
04911             }
04912         }
04913     }
04914 
04915 #ifndef NDEBUG
04916     if( my_rank == 1 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 1, before augment routing" );
04917     MPI_Barrier( procConfig.proc_comm() );
04918     int sentEnts = remoteEnts.get_n();
04919     assert( ( sentEnts == jr ) && ( 3 * sentEnts == ir ) );
04920 #endif
04921     // exchange the info now, and send to
04922     gs_data::crystal_data* cd = this->procConfig.crystal_router();
04923     // All communication happens here; no other mpi calls
04924     // Also, this is a collective call
04925     rval = cd->gs_transfer( 1, remoteEnts, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" );
04926 #ifndef NDEBUG
04927     if( my_rank == 0 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 0, after augment routing" );
04928     MPI_Barrier( procConfig.proc_comm() );
04929 #endif
04930 
04931     // now process the data received from other processors
04932     int received = remoteEnts.get_n();
04933     for( int i = 0; i < received; i++ )
04934     {
04935         // int from = ents_to_delete.vi_rd[i];
04936         EntityHandle geh = (EntityHandle)remoteEnts.vul_rd[i];
04937         int from_proc    = remoteEnts.vi_rd[3 * i];
04938         if( my_rank == from_proc )
04939             std::cout << " unexpected receive from my rank " << my_rank << " during augmenting with ghosts\n ";
04940         int tag_type = remoteEnts.vi_rd[3 * i + 1];
04941         assert( ( 0 <= tag_type ) && ( tag_type <= num_tags ) );
04942         int value = remoteEnts.vi_rd[3 * i + 2];
04943         if( tag_type == num_tags )
04944         {
04945             // it is global id
04946             rval = mbImpl->tag_set_data( tags[num_tags], &geh, 1, &value );MB_CHK_SET_ERR( rval, "Error in setting gid tag" );
04947         }
04948         else
04949         {
04950             // now, based on value and tag type, see if we have that value in the map
04951             MVal& lmap = localMaps[tag_type];
04952             itMVal itm = lmap.find( value );
04953             if( itm == lmap.end() )
04954             {
04955                 // the value was not found yet in the local map, so we have to create the set
04956                 EntityHandle newSet;
04957                 rval = mbImpl->create_meshset( MESHSET_SET, newSet );MB_CHK_SET_ERR( rval, "can't create new set" );
04958                 lmap[value] = newSet;
04959                 // set the tag value
04960                 rval = mbImpl->tag_set_data( tags[tag_type], &newSet, 1, &value );MB_CHK_SET_ERR( rval, "can't set tag for new set" );
04961 
04962                 // we also need to add the new created set to the file set, if not null
04963                 if( file_set )
04964                 {
04965                     rval = mbImpl->add_entities( file_set, &newSet, 1 );MB_CHK_SET_ERR( rval, "can't add new set to the file set" );
04966                 }
04967             }
04968             // add the entity to the set pointed to by the map
04969             rval = mbImpl->add_entities( lmap[value], &geh, 1 );MB_CHK_SET_ERR( rval, "can't add ghost ent to the set" );
04970         }
04971     }
04972 
04973     for( int i = 0; i < num_tags; i++ )
04974         delete[] tagVals[i];
04975     delete[] tagVals;
04976     delete[] rangeSets;
04977     delete[] tags;
04978     delete[] localMaps;
04979     return MB_SUCCESS;
04980 }
04981 ErrorCode ParallelComm::create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim )
04982 {
04983     std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
04984 
04985     // Build up the list of shared entities
04986     int procs[MAX_SHARING_PROCS];
04987     EntityHandle handles[MAX_SHARING_PROCS];
04988     ErrorCode result;
04989     int nprocs;
04990     unsigned char pstat;
04991     for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
04992     {
04993         if( shared_dim != -1 && mbImpl->dimension_from_handle( *vit ) > shared_dim ) continue;
04994         result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
04995         std::sort( procs, procs + nprocs );
04996         std::vector< int > tmp_procs( procs, procs + nprocs );
04997         assert( tmp_procs.size() != 2 );
04998         proc_nvecs[tmp_procs].push_back( *vit );
04999     }
05000 
05001     Skinner skinner( mbImpl );
05002     Range skin_ents[4];
05003     result = mbImpl->get_entities_by_dimension( this_set, resolve_dim, skin_ents[resolve_dim] );MB_CHK_SET_ERR( result, "Failed to get skin entities by dimension" );
05004     result =
05005         skinner.find_skin( this_set, skin_ents[resolve_dim], false, skin_ents[resolve_dim - 1], 0, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
05006     if( shared_dim > 1 )
05007     {
05008         result = mbImpl->get_adjacencies( skin_ents[resolve_dim - 1], resolve_dim - 2, true, skin_ents[resolve_dim - 2],
05009                                           Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
05010     }
05011 
05012     result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );
05013 
05014     return create_interface_sets( proc_nvecs );
05015 }
05016 
05017 ErrorCode ParallelComm::create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
05018 {
05019     if( proc_nvecs.empty() ) return MB_SUCCESS;
05020 
05021     int proc_ids[MAX_SHARING_PROCS];
05022     EntityHandle proc_handles[MAX_SHARING_PROCS];
05023     Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
05024     ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in create_interface_sets" );
05025     Range::iterator rit;
05026 
05027     // Create interface sets, tag them, and tag their contents with iface set tag
05028     std::vector< unsigned char > pstatus;
05029     for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator vit = proc_nvecs.begin();
05030          vit != proc_nvecs.end(); ++vit )
05031     {
05032         // Create the set
05033         EntityHandle new_set;
05034         result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
05035         interfaceSets.insert( new_set );
05036 
05037         // Add entities
05038         assert( !vit->second.empty() );
05039         result = mbImpl->add_entities( new_set, &( vit->second )[0], ( vit->second ).size() );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
05040         // Tag set with the proc rank(s)
05041         if( vit->first.size() == 1 )
05042         {
05043             assert( ( vit->first )[0] != (int)procConfig.proc_rank() );
05044             result = mbImpl->tag_set_data( shp_tag, &new_set, 1, &( vit->first )[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
05045             proc_handles[0] = 0;
05046             result          = mbImpl->tag_set_data( shh_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
05047         }
05048         else
05049         {
05050             // Pad tag data out to MAX_SHARING_PROCS with -1
05051             if( vit->first.size() > MAX_SHARING_PROCS )
05052             {
05053                 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_set ) )
05054                           << ' ' << ID_FROM_HANDLE( new_set ) << " on process " << proc_config().proc_rank()
05055                           << std::endl;
05056                 std::cerr.flush();
05057                 MPI_Abort( proc_config().proc_comm(), 66 );
05058             }
05059             // assert(vit->first.size() <= MAX_SHARING_PROCS);
05060             std::copy( vit->first.begin(), vit->first.end(), proc_ids );
05061             std::fill( proc_ids + vit->first.size(), proc_ids + MAX_SHARING_PROCS, -1 );
05062             result = mbImpl->tag_set_data( shps_tag, &new_set, 1, proc_ids );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
05063             unsigned int ind = std::find( proc_ids, proc_ids + vit->first.size(), procConfig.proc_rank() ) - proc_ids;
05064             assert( ind < vit->first.size() );
05065             std::fill( proc_handles, proc_handles + MAX_SHARING_PROCS, 0 );
05066             proc_handles[ind] = new_set;
05067             result            = mbImpl->tag_set_data( shhs_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
05068         }
05069 
05070         // Get the owning proc, then set the pstatus tag on iface set
05071         int min_proc       = ( vit->first )[0];
05072         unsigned char pval = ( PSTATUS_SHARED | PSTATUS_INTERFACE );
05073         if( min_proc < (int)procConfig.proc_rank() ) pval |= PSTATUS_NOT_OWNED;
05074         if( vit->first.size() > 1 ) pval |= PSTATUS_MULTISHARED;
05075         result = mbImpl->tag_set_data( pstat_tag, &new_set, 1, &pval );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
05076 
05077         // Tag the vertices with the same thing
05078         pstatus.clear();
05079         std::vector< EntityHandle > verts;
05080         for( std::vector< EntityHandle >::iterator v2it = ( vit->second ).begin(); v2it != ( vit->second ).end();
05081              ++v2it )
05082             if( mbImpl->type_from_handle( *v2it ) == MBVERTEX ) verts.push_back( *v2it );
05083         pstatus.resize( verts.size(), pval );
05084         if( !verts.empty() )
05085         {
05086             result = mbImpl->tag_set_data( pstat_tag, &verts[0], verts.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set vertices with pstatus" );
05087         }
05088     }
05089 
05090     return MB_SUCCESS;
05091 }
05092 
05093 ErrorCode ParallelComm::create_iface_pc_links()
05094 {
05095     // Now that we've resolved the entities in the iface sets,
05096     // set parent/child links between the iface sets
05097 
05098     // First tag all entities in the iface sets
05099     Tag tmp_iface_tag;
05100     EntityHandle tmp_iface_set = 0;
05101     ErrorCode result           = mbImpl->tag_get_handle( "__tmp_iface", 1, MB_TYPE_HANDLE, tmp_iface_tag,
05102                                                          MB_TAG_DENSE | MB_TAG_CREAT, &tmp_iface_set );MB_CHK_SET_ERR( result, "Failed to create temporary interface set tag" );
05103 
05104     Range iface_ents;
05105     std::vector< EntityHandle > tag_vals;
05106     Range::iterator rit;
05107 
05108     for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
05109     {
05110         // tag entities with interface set
05111         iface_ents.clear();
05112         result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in interface set" );
05113 
05114         if( iface_ents.empty() ) continue;
05115 
05116         tag_vals.resize( iface_ents.size() );
05117         std::fill( tag_vals.begin(), tag_vals.end(), *rit );
05118         result = mbImpl->tag_set_data( tmp_iface_tag, iface_ents, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to tag iface entities with interface set" );
05119     }
05120 
05121     // Now go back through interface sets and add parent/child links
05122     Range tmp_ents2;
05123     for( int d = 2; d >= 0; d-- )
05124     {
05125         for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
05126         {
05127             // Get entities on this interface
05128             iface_ents.clear();
05129             result = mbImpl->get_entities_by_handle( *rit, iface_ents, true );MB_CHK_SET_ERR( result, "Failed to get entities by handle" );
05130             if( iface_ents.empty() || mbImpl->dimension_from_handle( *iface_ents.rbegin() ) != d ) continue;
05131 
05132             // Get higher-dimensional entities and their interface sets
05133             result = mbImpl->get_adjacencies( &( *iface_ents.begin() ), 1, d + 1, false, tmp_ents2 );MB_CHK_SET_ERR( result, "Failed to get adjacencies for interface sets" );
05134             tag_vals.resize( tmp_ents2.size() );
05135             result = mbImpl->tag_get_data( tmp_iface_tag, tmp_ents2, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to get tmp iface tag for interface sets" );
05136 
05137             // Go through and for any on interface make it a parent
05138             EntityHandle last_set = 0;
05139             for( unsigned int i = 0; i < tag_vals.size(); i++ )
05140             {
05141                 if( tag_vals[i] && tag_vals[i] != last_set )
05142                 {
05143                     result = mbImpl->add_parent_child( tag_vals[i], *rit );MB_CHK_SET_ERR( result, "Failed to add parent/child link for interface set" );
05144                     last_set = tag_vals[i];
05145                 }
05146             }
05147         }
05148     }
05149 
05150     // Delete the temporary tag
05151     result = mbImpl->tag_delete( tmp_iface_tag );MB_CHK_SET_ERR( result, "Failed to delete tmp iface tag" );
05152 
05153     return MB_SUCCESS;
05154 }
05155 
05156 ErrorCode ParallelComm::get_proc_nvecs( int resolve_dim,
05157                                         int shared_dim,
05158                                         Range* skin_ents,
05159                                         std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
05160 {
05161     // Set sharing procs tags on other skin ents
05162     ErrorCode result;
05163     const EntityHandle* connect;
05164     int num_connect;
05165     std::set< int > sharing_procs;
05166     std::vector< EntityHandle > dum_connect;
05167     std::vector< int > sp_vec;
05168 
05169     for( int d = 3; d > 0; d-- )
05170     {
05171         if( resolve_dim == d ) continue;
05172 
05173         for( Range::iterator rit = skin_ents[d].begin(); rit != skin_ents[d].end(); ++rit )
05174         {
05175             // Get connectivity
05176             result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect );MB_CHK_SET_ERR( result, "Failed to get connectivity on non-vertex skin entities" );
05177 
05178             int op = ( resolve_dim < shared_dim ? Interface::UNION : Interface::INTERSECT );
05179             result = get_sharing_data( connect, num_connect, sharing_procs, op );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_proc_nvecs" );
05180             if( sharing_procs.empty() ||
05181                 ( sharing_procs.size() == 1 && *sharing_procs.begin() == (int)procConfig.proc_rank() ) )
05182                 continue;
05183 
05184             // Need to specify sharing data correctly for entities or they will
05185             // end up in a different interface set than corresponding vertices
05186             if( sharing_procs.size() == 2 )
05187             {
05188                 std::set< int >::iterator it = sharing_procs.find( proc_config().proc_rank() );
05189                 assert( it != sharing_procs.end() );
05190                 sharing_procs.erase( it );
05191             }
05192 
05193             // Intersection is the owning proc(s) for this skin ent
05194             sp_vec.clear();
05195             std::copy( sharing_procs.begin(), sharing_procs.end(), std::back_inserter( sp_vec ) );
05196             assert( sp_vec.size() != 2 );
05197             proc_nvecs[sp_vec].push_back( *rit );
05198         }
05199     }
05200 
05201 #ifndef NDEBUG
05202     // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
05203     for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
05204          mit != proc_nvecs.end(); ++mit )
05205     {
05206         std::vector< EntityHandle > tmp_vec = ( mit->second );
05207         std::sort( tmp_vec.begin(), tmp_vec.end() );
05208         std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
05209         assert( vit == tmp_vec.end() );
05210     }
05211 #endif
05212 
05213     return MB_SUCCESS;
05214 }
05215 
05216 // Overloaded form of tag_shared_verts
05217 // Tuple coming in is of form (arbitrary value, remoteProc, localHandle, remoteHandle)
05218 // Also will check for doubles in the list if the list is sorted
05219 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
05220                                           std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
05221                                           Range& /*proc_verts*/,
05222                                           unsigned int i_extra )
05223 {
05224     Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
05225     ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
05226 
05227     unsigned int j = 0, i = 0;
05228     std::vector< int > sharing_procs, sharing_procs2, tag_procs;
05229     std::vector< EntityHandle > sharing_handles, sharing_handles2, tag_lhandles, tag_rhandles;
05230     std::vector< unsigned char > pstatus;
05231 
05232     // Were on tuple j/2
05233     if( i_extra ) i += i_extra;
05234     while( j < 2 * shared_ents.get_n() )
05235     {
05236         // Count & accumulate sharing procs
05237         EntityHandle this_ent = shared_ents.vul_rd[j], other_ent = 0;
05238         int other_proc = -1;
05239         while( j < 2 * shared_ents.get_n() && shared_ents.vul_rd[j] == this_ent )
05240         {
05241             j++;
05242             // Shouldn't have same proc
05243             assert( shared_ents.vi_rd[i] != (int)procConfig.proc_rank() );
05244             // Grab the remote data if its not a dublicate
05245             if( shared_ents.vul_rd[j] != other_ent || shared_ents.vi_rd[i] != other_proc )
05246             {
05247                 assert( 0 != shared_ents.vul_rd[j] );
05248                 sharing_procs.push_back( shared_ents.vi_rd[i] );
05249                 sharing_handles.push_back( shared_ents.vul_rd[j] );
05250             }
05251             other_proc = shared_ents.vi_rd[i];
05252             other_ent  = shared_ents.vul_rd[j];
05253             j++;
05254             i += 1 + i_extra;
05255         }
05256 
05257         if( sharing_procs.size() > 1 )
05258         {
05259             // Add current proc/handle to list
05260             sharing_procs.push_back( procConfig.proc_rank() );
05261             sharing_handles.push_back( this_ent );
05262 
05263             // Sort sharing_procs and sharing_handles such that
05264             // sharing_procs is in ascending order. Use temporary
05265             // lists and binary search to re-order sharing_handles.
05266             sharing_procs2 = sharing_procs;
05267             std::sort( sharing_procs2.begin(), sharing_procs2.end() );
05268             sharing_handles2.resize( sharing_handles.size() );
05269             for( size_t k = 0; k < sharing_handles.size(); k++ )
05270             {
05271                 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
05272                              sharing_procs2.begin();
05273                 sharing_handles2[idx] = sharing_handles[k];
05274             }
05275             sharing_procs.swap( sharing_procs2 );
05276             sharing_handles.swap( sharing_handles2 );
05277         }
05278 
05279         assert( sharing_procs.size() != 2 );
05280         proc_nvecs[sharing_procs].push_back( this_ent );
05281 
05282         unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
05283         if( sharing_procs.size() == 1 )
05284         {
05285             tag_procs.push_back( sharing_procs[0] );
05286             tag_lhandles.push_back( this_ent );
05287             tag_rhandles.push_back( sharing_handles[0] );
05288             pstatus.push_back( share_flag );
05289         }
05290         else
05291         {
05292             // Pad lists
05293             // assert(sharing_procs.size() <= MAX_SHARING_PROCS);
05294             if( sharing_procs.size() > MAX_SHARING_PROCS )
05295             {
05296                 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
05297                           << proc_config().proc_rank() << std::endl;
05298                 std::cerr.flush();
05299                 MPI_Abort( proc_config().proc_comm(), 66 );
05300             }
05301             sharing_procs.resize( MAX_SHARING_PROCS, -1 );
05302             sharing_handles.resize( MAX_SHARING_PROCS, 0 );
05303             result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
05304             result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
05305             result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
05306             sharedEnts.insert( this_ent );
05307         }
05308 
05309         // Reset sharing proc(s) tags
05310         sharing_procs.clear();
05311         sharing_handles.clear();
05312     }
05313 
05314     if( !tag_procs.empty() )
05315     {
05316         result = mbImpl->tag_set_data( shp_tag, &tag_lhandles[0], tag_procs.size(), &tag_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
05317         result = mbImpl->tag_set_data( shh_tag, &tag_lhandles[0], tag_procs.size(), &tag_rhandles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
05318         result = mbImpl->tag_set_data( pstat_tag, &tag_lhandles[0], tag_procs.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
05319         for( std::vector< EntityHandle >::iterator vvt = tag_lhandles.begin(); vvt != tag_lhandles.end(); vvt++ )
05320             sharedEnts.insert( *vvt );
05321     }
05322 
05323 #ifndef NDEBUG
05324     // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
05325     for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
05326          mit != proc_nvecs.end(); ++mit )
05327     {
05328         std::vector< EntityHandle > tmp_vec = ( mit->second );
05329         std::sort( tmp_vec.begin(), tmp_vec.end() );
05330         std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
05331         assert( vit == tmp_vec.end() );
05332     }
05333 #endif
05334 
05335     return MB_SUCCESS;
05336 }
05337 
05338 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
05339                                           Range* skin_ents,
05340                                           std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
05341                                           Range& /*proc_verts*/ )
05342 {
05343     Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
05344     ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
05345 
05346     unsigned int j = 0, i = 0;
05347     std::vector< int > sharing_procs, sharing_procs2;
05348     std::vector< EntityHandle > sharing_handles, sharing_handles2, skin_verts( skin_ents[0].size() );
05349     for( Range::iterator rit = skin_ents[0].begin(); rit != skin_ents[0].end(); ++rit, i++ )
05350         skin_verts[i] = *rit;
05351     i = 0;
05352 
05353     while( j < 2 * shared_ents.get_n() )
05354     {
05355         // Count & accumulate sharing procs
05356         int this_idx          = shared_ents.vi_rd[j];
05357         EntityHandle this_ent = skin_verts[this_idx];
05358         while( j < 2 * shared_ents.get_n() && shared_ents.vi_rd[j] == this_idx )
05359         {
05360             j++;
05361             // Shouldn't have same proc
05362             assert( shared_ents.vi_rd[j] != (int)procConfig.proc_rank() );
05363             sharing_procs.push_back( shared_ents.vi_rd[j++] );
05364             sharing_handles.push_back( shared_ents.vul_rd[i++] );
05365         }
05366 
05367         if( sharing_procs.size() > 1 )
05368         {
05369             // Add current proc/handle to list
05370             sharing_procs.push_back( procConfig.proc_rank() );
05371             sharing_handles.push_back( this_ent );
05372         }
05373 
05374         // Sort sharing_procs and sharing_handles such that
05375         // sharing_procs is in ascending order. Use temporary
05376         // lists and binary search to re-order sharing_handles.
05377         sharing_procs2 = sharing_procs;
05378         std::sort( sharing_procs2.begin(), sharing_procs2.end() );
05379         sharing_handles2.resize( sharing_handles.size() );
05380         for( size_t k = 0; k < sharing_handles.size(); k++ )
05381         {
05382             size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
05383                          sharing_procs2.begin();
05384             sharing_handles2[idx] = sharing_handles[k];
05385         }
05386         sharing_procs.swap( sharing_procs2 );
05387         sharing_handles.swap( sharing_handles2 );
05388 
05389         assert( sharing_procs.size() != 2 );
05390         proc_nvecs[sharing_procs].push_back( this_ent );
05391 
05392         unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
05393         if( sharing_procs.size() == 1 )
05394         {
05395             result = mbImpl->tag_set_data( shp_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
05396             result = mbImpl->tag_set_data( shh_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
05397             result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &share_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
05398             sharedEnts.insert( this_ent );
05399         }
05400         else
05401         {
05402             // Pad lists
05403             // assert(sharing_procs.size() <= MAX_SHARING_PROCS);
05404             if( sharing_procs.size() > MAX_SHARING_PROCS )
05405             {
05406                 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
05407                           << proc_config().proc_rank() << std::endl;
05408                 std::cerr.flush();
05409                 MPI_Abort( proc_config().proc_comm(), 66 );
05410             }
05411             sharing_procs.resize( MAX_SHARING_PROCS, -1 );
05412             sharing_handles.resize( MAX_SHARING_PROCS, 0 );
05413             result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
05414             result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
05415             result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
05416             sharedEnts.insert( this_ent );
05417         }
05418 
05419         // Reset sharing proc(s) tags
05420         sharing_procs.clear();
05421         sharing_handles.clear();
05422     }
05423 
05424 #ifndef NDEBUG
05425     // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
05426     for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
05427          mit != proc_nvecs.end(); ++mit )
05428     {
05429         std::vector< EntityHandle > tmp_vec = ( mit->second );
05430         std::sort( tmp_vec.begin(), tmp_vec.end() );
05431         std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
05432         assert( vit == tmp_vec.end() );
05433     }
05434 #endif
05435 
05436     return MB_SUCCESS;
05437 }
05438 
05439 //! Get processors with which this processor communicates; sets are sorted by processor
05440 ErrorCode ParallelComm::get_interface_procs( std::set< unsigned int >& procs_set, bool get_buffs )
05441 {
05442     // Make sure the sharing procs vector is empty
05443     procs_set.clear();
05444 
05445     // Pre-load vector of single-proc tag values
05446     unsigned int i, j;
05447     std::vector< int > iface_proc( interfaceSets.size() );
05448     ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), interfaceSets, &iface_proc[0] );MB_CHK_SET_ERR( result, "Failed to get iface_proc for iface sets" );
05449 
05450     // Get sharing procs either from single-proc vector or by getting
05451     // multi-proc tag value
05452     int tmp_iface_procs[MAX_SHARING_PROCS];
05453     std::fill( tmp_iface_procs, tmp_iface_procs + MAX_SHARING_PROCS, -1 );
05454     Range::iterator rit;
05455     for( rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); ++rit, i++ )
05456     {
05457         if( -1 != iface_proc[i] )
05458         {
05459             assert( iface_proc[i] != (int)procConfig.proc_rank() );
05460             procs_set.insert( (unsigned int)iface_proc[i] );
05461         }
05462         else
05463         {
05464             // Get the sharing_procs tag
05465             result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, tmp_iface_procs );MB_CHK_SET_ERR( result, "Failed to get iface_procs for iface set" );
05466             for( j = 0; j < MAX_SHARING_PROCS; j++ )
05467             {
05468                 if( -1 != tmp_iface_procs[j] && tmp_iface_procs[j] != (int)procConfig.proc_rank() )
05469                     procs_set.insert( (unsigned int)tmp_iface_procs[j] );
05470                 else if( -1 == tmp_iface_procs[j] )
05471                 {
05472                     std::fill( tmp_iface_procs, tmp_iface_procs + j, -1 );
05473                     break;
05474                 }
05475             }
05476         }
05477     }
05478 
05479     if( get_buffs )
05480     {
05481         for( std::set< unsigned int >::iterator sit = procs_set.begin(); sit != procs_set.end(); ++sit )
05482             get_buffers( *sit );
05483     }
05484 
05485     return MB_SUCCESS;
05486 }
05487 
05488 ErrorCode ParallelComm::get_pstatus( EntityHandle entity, unsigned char& pstatus_val )
05489 {
05490     ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstatus_val );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
05491     return result;
05492 }
05493 
05494 ErrorCode ParallelComm::get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents )
05495 {
05496     Range ents;
05497     ErrorCode result;
05498 
05499     if( -1 == dim )
05500     {
05501         result = mbImpl->get_entities_by_handle( 0, ents );MB_CHK_SET_ERR( result, "Failed to get all entities" );
05502     }
05503     else
05504     {
05505         result = mbImpl->get_entities_by_dimension( 0, dim, ents );MB_CHK_SET_ERR( result, "Failed to get entities of dimension " << dim );
05506     }
05507 
05508     std::vector< unsigned char > pstatus( ents.size() );
05509     result = mbImpl->tag_get_data( pstatus_tag(), ents, &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
05510     Range::iterator rit = ents.begin();
05511     int i               = 0;
05512     if( pstatus_val )
05513     {
05514         for( ; rit != ents.end(); i++, ++rit )
05515         {
05516             if( pstatus[i] & pstatus_val && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
05517                 pstatus_ents.insert( *rit );
05518         }
05519     }
05520     else
05521     {
05522         for( ; rit != ents.end(); i++, ++rit )
05523         {
05524             if( !pstatus[i] && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
05525                 pstatus_ents.insert( *rit );
05526         }
05527     }
05528 
05529     return MB_SUCCESS;
05530 }
05531 
05532 ErrorCode ParallelComm::check_global_ids( EntityHandle this_set,
05533                                           const int dimension,
05534                                           const int start_id,
05535                                           const bool largest_dim_only,
05536                                           const bool parallel,
05537                                           const bool owned_only )
05538 {
05539     // Global id tag
05540     Tag gid_tag = mbImpl->globalId_tag();
05541     int def_val = -1;
05542     Range dum_range;
05543 
05544     void* tag_ptr    = &def_val;
05545     ErrorCode result = mbImpl->get_entities_by_type_and_tag( this_set, MBVERTEX, &gid_tag, &tag_ptr, 1, dum_range );MB_CHK_SET_ERR( result, "Failed to get entities by MBVERTEX type and gid tag" );
05546 
05547     if( !dum_range.empty() )
05548     {
05549         // Just created it, so we need global ids
05550         result = assign_global_ids( this_set, dimension, start_id, largest_dim_only, parallel, owned_only );MB_CHK_SET_ERR( result, "Failed assigning global ids" );
05551     }
05552 
05553     return MB_SUCCESS;
05554 }
05555 
05556 bool ParallelComm::is_iface_proc( EntityHandle this_set, int to_proc )
05557 {
05558     int sharing_procs[MAX_SHARING_PROCS];
05559     std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
05560     ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &this_set, 1, sharing_procs );
05561     if( MB_SUCCESS == result && to_proc == sharing_procs[0] ) return true;
05562 
05563     result = mbImpl->tag_get_data( sharedps_tag(), &this_set, 1, sharing_procs );
05564     if( MB_SUCCESS != result ) return false;
05565 
05566     for( int i = 0; i < MAX_SHARING_PROCS; i++ )
05567     {
05568         if( to_proc == sharing_procs[i] )
05569             return true;
05570         else if( -1 == sharing_procs[i] )
05571             return false;
05572     }
05573 
05574     return false;
05575 }
05576 
05577 ErrorCode ParallelComm::filter_pstatus( Range& ents,
05578                                         unsigned char pstat,
05579                                         unsigned char op,
05580                                         int to_proc,
05581                                         Range* returned_ents )
05582 {
05583     Range tmp_ents;
05584 
05585     // assert(!ents.empty());
05586     if( ents.empty() )
05587     {
05588         if( returned_ents ) returned_ents->clear();
05589         return MB_SUCCESS;
05590     }
05591 
05592     // Put into tmp_ents any entities which are not owned locally or
05593     // who are already shared with to_proc
05594     std::vector< unsigned char > shared_flags( ents.size() ), shared_flags2;
05595     ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), ents, &shared_flags[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus flag" );
05596     Range::const_iterator rit, hint = tmp_ents.begin();
05597     ;
05598     int i;
05599     if( op == PSTATUS_OR )
05600     {
05601         for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
05602         {
05603             if( ( ( shared_flags[i] & ~pstat ) ^ shared_flags[i] ) & pstat )
05604             {
05605                 hint = tmp_ents.insert( hint, *rit );
05606                 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
05607             }
05608         }
05609     }
05610     else if( op == PSTATUS_AND )
05611     {
05612         for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
05613         {
05614             if( ( shared_flags[i] & pstat ) == pstat )
05615             {
05616                 hint = tmp_ents.insert( hint, *rit );
05617                 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
05618             }
05619         }
05620     }
05621     else if( op == PSTATUS_NOT )
05622     {
05623         for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
05624         {
05625             if( !( shared_flags[i] & pstat ) )
05626             {
05627                 hint = tmp_ents.insert( hint, *rit );
05628                 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
05629             }
05630         }
05631     }
05632     else
05633     {
05634         assert( false );
05635         return MB_FAILURE;
05636     }
05637 
05638     if( -1 != to_proc )
05639     {
05640         int sharing_procs[MAX_SHARING_PROCS];
05641         std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
05642         Range tmp_ents2;
05643         hint = tmp_ents2.begin();
05644 
05645         for( rit = tmp_ents.begin(), i = 0; rit != tmp_ents.end(); ++rit, i++ )
05646         {
05647             // We need to check sharing procs
05648             if( shared_flags2[i] & PSTATUS_MULTISHARED )
05649             {
05650                 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag" );
05651                 assert( -1 != sharing_procs[0] );
05652                 for( unsigned int j = 0; j < MAX_SHARING_PROCS; j++ )
05653                 {
05654                     // If to_proc shares this entity, add it to list
05655                     if( sharing_procs[j] == to_proc )
05656                     {
05657                         hint = tmp_ents2.insert( hint, *rit );
05658                     }
05659                     else if( -1 == sharing_procs[j] )
05660                         break;
05661 
05662                     sharing_procs[j] = -1;
05663                 }
05664             }
05665             else if( shared_flags2[i] & PSTATUS_SHARED )
05666             {
05667                 result = mbImpl->tag_get_data( sharedp_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag" );
05668                 assert( -1 != sharing_procs[0] );
05669                 if( sharing_procs[0] == to_proc ) hint = tmp_ents2.insert( hint, *rit );
05670                 sharing_procs[0] = -1;
05671             }
05672             else
05673                 assert( "should never get here" && false );
05674         }
05675 
05676         tmp_ents.swap( tmp_ents2 );
05677     }
05678 
05679     if( returned_ents )
05680         returned_ents->swap( tmp_ents );
05681     else
05682         ents.swap( tmp_ents );
05683 
05684     return MB_SUCCESS;
05685 }
05686 
05687 ErrorCode ParallelComm::exchange_ghost_cells( int ghost_dim,
05688                                               int bridge_dim,
05689                                               int num_layers,
05690                                               int addl_ents,
05691                                               bool store_remote_handles,
05692                                               bool wait_all,
05693                                               EntityHandle* file_set )
05694 {
05695 #ifdef MOAB_HAVE_MPE
05696     if( myDebug->get_verbosity() == 2 )
05697     {
05698         if( !num_layers )
05699             MPE_Log_event( IFACE_START, procConfig.proc_rank(), "Starting interface exchange." );
05700         else
05701             MPE_Log_event( GHOST_START, procConfig.proc_rank(), "Starting ghost exchange." );
05702     }
05703 #endif
05704 
05705     myDebug->tprintf( 1, "Entering exchange_ghost_cells with num_layers = %d\n", num_layers );
05706     if( myDebug->get_verbosity() == 4 )
05707     {
05708         msgs.clear();
05709         msgs.reserve( MAX_SHARING_PROCS );
05710     }
05711 
05712     // If we're only finding out about existing ents, we have to be storing
05713     // remote handles too
05714     assert( num_layers > 0 || store_remote_handles );
05715 
05716     const bool is_iface = !num_layers;
05717 
05718     // Get the b-dimensional interface(s) with with_proc, where b = bridge_dim
05719 
05720     int success;
05721     ErrorCode result = MB_SUCCESS;
05722     int incoming1 = 0, incoming2 = 0;
05723 
05724     reset_all_buffers();
05725 
05726     // When this function is called, buffProcs should already have any
05727     // communicating procs
05728 
05729     //===========================================
05730     // Post ghost irecv's for ghost entities from all communicating procs
05731     //===========================================
05732 #ifdef MOAB_HAVE_MPE
05733     if( myDebug->get_verbosity() == 2 )
05734     {
05735         MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." );
05736     }
05737 #endif
05738 
05739     // Index reqs the same as buffer/sharing procs indices
05740     std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ),
05741         recv_remoteh_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
05742     std::vector< unsigned int >::iterator proc_it;
05743     int ind, p;
05744     sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
05745     for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
05746     {
05747         incoming1++;
05748         PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
05749                            MB_MESG_ENTS_SIZE, incoming1 );
05750         success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
05751                              MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
05752         if( success != MPI_SUCCESS )
05753         {
05754             MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
05755         }
05756     }
05757 
05758     //===========================================
05759     // Get entities to be sent to neighbors
05760     //===========================================
05761     Range sent_ents[MAX_SHARING_PROCS], allsent, tmp_range;
05762     TupleList entprocs;
05763     int dum_ack_buff;
05764     result = get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents, allsent, entprocs );MB_CHK_SET_ERR( result, "get_sent_ents failed" );
05765 
05766     // augment file set with the entities to be sent
05767     // we might have created new entities if addl_ents>0, edges and/or faces
05768     if( addl_ents > 0 && file_set && !allsent.empty() )
05769     {
05770         result = mbImpl->add_entities( *file_set, allsent );MB_CHK_SET_ERR( result, "Failed to add new sub-entities to set" );
05771     }
05772     myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
05773                       (unsigned long)allsent.size() );
05774 
05775     //===========================================
05776     // Pack and send ents from this proc to others
05777     //===========================================
05778     for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
05779     {
05780         myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", sent_ents[p].compactness(),
05781                           (unsigned long)sent_ents[p].size() );
05782 
05783         // Reserve space on front for size and for initial buff size
05784         localOwnedBuffs[p]->reset_buffer( sizeof( int ) );
05785 
05786         // Entities
05787         result = pack_entities( sent_ents[p], localOwnedBuffs[p], store_remote_handles, buffProcs[p], is_iface,
05788                                 &entprocs, &allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
05789 
05790         if( myDebug->get_verbosity() == 4 )
05791         {
05792             msgs.resize( msgs.size() + 1 );
05793             msgs.back() = new Buffer( *localOwnedBuffs[p] );
05794         }
05795 
05796         // Send the buffer (size stored in front in send_buffer)
05797         result = send_buffer( *proc_it, localOwnedBuffs[p], MB_MESG_ENTS_SIZE, sendReqs[3 * p],
05798                               recv_ent_reqs[3 * p + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE,
05799                               ( !is_iface && store_remote_handles ?  // this used for ghosting only
05800                                     localOwnedBuffs[p]
05801                                                                   : NULL ),
05802                               &recv_remoteh_reqs[3 * p], &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" );
05803     }
05804 
05805     entprocs.reset();
05806 
05807     //===========================================
05808     // Receive/unpack new entities
05809     //===========================================
05810     // Number of incoming messages for ghosts is the number of procs we
05811     // communicate with; for iface, it's the number of those with lower rank
05812     MPI_Status status;
05813     std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
05814     std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
05815     std::vector< std::vector< int > > L1p( buffProcs.size() );
05816     std::vector< EntityHandle > L2hloc, L2hrem;
05817     std::vector< unsigned int > L2p;
05818     std::vector< EntityHandle > new_ents;
05819 
05820     while( incoming1 )
05821     {
05822         // Wait for all recvs of ghost ents before proceeding to sending remote handles,
05823         // b/c some procs may have sent to a 3rd proc ents owned by me;
05824         PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
05825 
05826         success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status );
05827         if( MPI_SUCCESS != success )
05828         {
05829             MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
05830         }
05831 
05832         PRINT_DEBUG_RECD( status );
05833 
05834         // OK, received something; decrement incoming counter
05835         incoming1--;
05836         bool done = false;
05837 
05838         // In case ind is for ack, we need index of one before it
05839         unsigned int base_ind = 3 * ( ind / 3 );
05840         result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1],
05841                               recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1],
05842                               sendReqs[base_ind + 2], done,
05843                               ( !is_iface && store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ),
05844                               MB_MESG_REMOTEH_SIZE,  // maybe base_ind+1?
05845                               &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
05846 
05847         if( done )
05848         {
05849             if( myDebug->get_verbosity() == 4 )
05850             {
05851                 msgs.resize( msgs.size() + 1 );
05852                 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] );
05853             }
05854 
05855             // Message completely received - process buffer that was sent
05856             remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
05857             result = unpack_entities( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, ind / 3, is_iface,
05858                                       L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );
05859             if( MB_SUCCESS != result )
05860             {
05861                 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
05862                 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false );
05863                 return result;
05864             }
05865 
05866             if( recv_ent_reqs.size() != 3 * buffProcs.size() )
05867             {
05868                 // Post irecv's for remote handles from new proc; shouldn't be iface,
05869                 // since we know about all procs we share with
05870                 assert( !is_iface );
05871                 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
05872                 for( unsigned int i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 )
05873                 {
05874                     localOwnedBuffs[i / 3]->reset_buffer();
05875                     incoming2++;
05876                     PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr,
05877                                        INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
05878                     success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
05879                                          buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
05880                                          &recv_remoteh_reqs[i] );
05881                     if( success != MPI_SUCCESS )
05882                     {
05883                         MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
05884                     }
05885                 }
05886                 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
05887                 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
05888             }
05889         }
05890     }
05891 
05892     // Add requests for any new addl procs
05893     if( recv_ent_reqs.size() != 3 * buffProcs.size() )
05894     {
05895         // Shouldn't get here...
05896         MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in ghost exchange" );
05897     }
05898 
05899 #ifdef MOAB_HAVE_MPE
05900     if( myDebug->get_verbosity() == 2 )
05901     {
05902         MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." );
05903     }
05904 #endif
05905 
05906     if( is_iface )
05907     {
05908         // Need to check over entities I sent and make sure I received
05909         // handles for them from all expected procs; if not, need to clean
05910         // them up
05911         result = check_clean_iface( allsent );
05912         if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
05913 
05914         // Now set the shared/interface tag on non-vertex entities on interface
05915         result = tag_iface_entities();MB_CHK_SET_ERR( result, "Failed to tag iface entities" );
05916 
05917 #ifndef NDEBUG
05918         result = check_sent_ents( allsent );
05919         if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
05920         result = check_all_shared_handles( true );
05921         if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
05922 #endif
05923 
05924 #ifdef MOAB_HAVE_MPE
05925         if( myDebug->get_verbosity() == 2 )
05926         {
05927             MPE_Log_event( IFACE_END, procConfig.proc_rank(), "Ending interface exchange." );
05928         }
05929 #endif
05930 
05931         //===========================================
05932         // Wait if requested
05933         //===========================================
05934         if( wait_all )
05935         {
05936             if( myDebug->get_verbosity() == 5 )
05937             {
05938                 success = MPI_Barrier( procConfig.proc_comm() );
05939             }
05940             else
05941             {
05942                 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
05943                 success = MPI_Waitall( 3 * buffProcs.size(), &recv_ent_reqs[0], mult_status );
05944                 if( MPI_SUCCESS != success )
05945                 {
05946                     MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
05947                 }
05948                 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
05949                 if( MPI_SUCCESS != success )
05950                 {
05951                     MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
05952                 }
05953                 /*success = MPI_Waitall(3*buffProcs.size(), &recv_remoteh_reqs[0], mult_status);
05954                 if (MPI_SUCCESS != success) {
05955                   MB_SET_ERR(MB_FAILURE, "Failed in waitall in ghost exchange");
05956                 }*/
05957             }
05958         }
05959 
05960         myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
05961         myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==true \n" );
05962 
05963         return MB_SUCCESS;
05964     }
05965 
05966     // we still need to wait on sendReqs, if they are not fulfilled yet
05967     if( wait_all )
05968     {
05969         if( myDebug->get_verbosity() == 5 )
05970         {
05971             success = MPI_Barrier( procConfig.proc_comm() );
05972         }
05973         else
05974         {
05975             MPI_Status mult_status[3 * MAX_SHARING_PROCS];
05976             success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
05977             if( MPI_SUCCESS != success )
05978             {
05979                 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
05980             }
05981         }
05982     }
05983     //===========================================
05984     // Send local handles for new ghosts to owner, then add
05985     // those to ghost list for that owner
05986     //===========================================
05987     for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
05988     {
05989 
05990         // Reserve space on front for size and for initial buff size
05991         remoteOwnedBuffs[p]->reset_buffer( sizeof( int ) );
05992 
05993         result = pack_remote_handles( L1hloc[p], L1hrem[p], L1p[p], *proc_it, remoteOwnedBuffs[p] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
05994         remoteOwnedBuffs[p]->set_stored_size();
05995 
05996         if( myDebug->get_verbosity() == 4 )
05997         {
05998             msgs.resize( msgs.size() + 1 );
05999             msgs.back() = new Buffer( *remoteOwnedBuffs[p] );
06000         }
06001         result = send_buffer( buffProcs[p], remoteOwnedBuffs[p], MB_MESG_REMOTEH_SIZE, sendReqs[3 * p],
06002                               recv_remoteh_reqs[3 * p + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
06003     }
06004 
06005     //===========================================
06006     // Process remote handles of my ghosteds
06007     //===========================================
06008     while( incoming2 )
06009     {
06010         PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
06011         success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status );
06012         if( MPI_SUCCESS != success )
06013         {
06014             MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
06015         }
06016 
06017         // OK, received something; decrement incoming counter
06018         incoming2--;
06019 
06020         PRINT_DEBUG_RECD( status );
06021 
06022         bool done             = false;
06023         unsigned int base_ind = 3 * ( ind / 3 );
06024         result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1],
06025                               recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3],
06026                               sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
06027         if( done )
06028         {
06029             // Incoming remote handles
06030             if( myDebug->get_verbosity() == 4 )
06031             {
06032                 msgs.resize( msgs.size() + 1 );
06033                 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] );
06034             }
06035             localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
06036             result =
06037                 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
06038         }
06039     }
06040 
06041 #ifdef MOAB_HAVE_MPE
06042     if( myDebug->get_verbosity() == 2 )
06043     {
06044         MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
06045         MPE_Log_event( GHOST_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." );
06046     }
06047 #endif
06048 
06049     //===========================================
06050     // Wait if requested
06051     //===========================================
06052     if( wait_all )
06053     {
06054         if( myDebug->get_verbosity() == 5 )
06055         {
06056             success = MPI_Barrier( procConfig.proc_comm() );
06057         }
06058         else
06059         {
06060             MPI_Status mult_status[3 * MAX_SHARING_PROCS];
06061             success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status );
06062             if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
06063         }
06064         if( MPI_SUCCESS != success )
06065         {
06066             MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
06067         }
06068     }
06069 
06070 #ifndef NDEBUG
06071     result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" );
06072     result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Failed check on all shared handles" );
06073 #endif
06074 
06075     if( file_set && !new_ents.empty() )
06076     {
06077         result = mbImpl->add_entities( *file_set, &new_ents[0], new_ents.size() );MB_CHK_SET_ERR( result, "Failed to add new entities to set" );
06078     }
06079 
06080     myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
06081     myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==false \n" );
06082 
06083     return MB_SUCCESS;
06084 }
06085 
06086 ErrorCode ParallelComm::send_buffer( const unsigned int to_proc,
06087                                      Buffer* send_buff,
06088                                      int mesg_tag,
06089                                      MPI_Request& send_req,
06090                                      MPI_Request& ack_req,
06091                                      int* ack_buff,
06092                                      int& this_incoming,
06093                                      int next_mesg_tag,
06094                                      Buffer* next_recv_buff,
06095                                      MPI_Request* next_recv_req,
06096                                      int* next_incoming )
06097 {
06098     ErrorCode result = MB_SUCCESS;
06099     int success;
06100 
06101     // If small message, post recv for remote handle message
06102     if( send_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE && next_recv_buff )
06103     {
06104         ( *next_incoming )++;
06105         PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, next_mesg_tag,
06106                            *next_incoming );
06107         success = MPI_Irecv( next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, to_proc, next_mesg_tag,
06108                              procConfig.proc_comm(), next_recv_req );
06109         if( success != MPI_SUCCESS )
06110         {
06111             MB_SET_ERR( MB_FAILURE, "Failed to post irecv for next message in ghost exchange" );
06112         }
06113     }
06114     // If large, we'll need an ack before sending the rest
06115     else if( send_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
06116     {
06117         this_incoming++;
06118         PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, (unsigned char*)ack_buff, sizeof( int ), mesg_tag - 1,
06119                            this_incoming );
06120         success = MPI_Irecv( (void*)ack_buff, sizeof( int ), MPI_UNSIGNED_CHAR, to_proc, mesg_tag - 1,
06121                              procConfig.proc_comm(), &ack_req );
06122         if( success != MPI_SUCCESS )
06123         {
06124             MB_SET_ERR( MB_FAILURE, "Failed to post irecv for entity ack in ghost exchange" );
06125         }
06126     }
06127 
06128     // Send the buffer
06129     PRINT_DEBUG_ISEND( procConfig.proc_rank(), to_proc, send_buff->mem_ptr, mesg_tag,
06130                        std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ) );
06131     assert( 0 <= send_buff->get_stored_size() && send_buff->get_stored_size() <= (int)send_buff->alloc_size );
06132     success = MPI_Isend( send_buff->mem_ptr, std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ),
06133                          MPI_UNSIGNED_CHAR, to_proc, mesg_tag, procConfig.proc_comm(), &send_req );
06134     if( success != MPI_SUCCESS ) return MB_FAILURE;
06135 
06136     return result;
06137 }
06138 
06139 ErrorCode ParallelComm::recv_buffer( int mesg_tag_expected,
06140                                      const MPI_Status& mpi_status,
06141                                      Buffer* recv_buff,
06142                                      MPI_Request& recv_req,
06143                                      MPI_Request& /*ack_recvd_req*/,
06144                                      int& this_incoming,
06145                                      Buffer* send_buff,
06146                                      MPI_Request& send_req,
06147                                      MPI_Request& sent_ack_req,
06148                                      bool& done,
06149                                      Buffer* next_buff,
06150                                      int next_tag,
06151                                      MPI_Request* next_req,
06152                                      int* next_incoming )
06153 {
06154     // Process a received message; if there will be more coming,
06155     // post a receive for 2nd part then send an ack message
06156     int from_proc = mpi_status.MPI_SOURCE;
06157     int success;
06158 
06159     // Set the buff_ptr on the recv_buffer; needs to point beyond any
06160     // valid data already in the buffer
06161     recv_buff->reset_ptr( std::min( recv_buff->get_stored_size(), (int)recv_buff->alloc_size ) );
06162 
06163     if( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
06164     {
06165         // 1st message & large - allocate buffer, post irecv for 2nd message,
06166         // then send ack
06167         recv_buff->reserve( recv_buff->get_stored_size() );
06168         assert( recv_buff->alloc_size > INITIAL_BUFF_SIZE );
06169 
06170         // Will expect a 2nd message
06171         this_incoming++;
06172 
06173         PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr + INITIAL_BUFF_SIZE,
06174                            recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, mesg_tag_expected + 1, this_incoming );
06175         success = MPI_Irecv( recv_buff->mem_ptr + INITIAL_BUFF_SIZE, recv_buff->get_stored_size() - INITIAL_BUFF_SIZE,
06176                              MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &recv_req );
06177         if( success != MPI_SUCCESS )
06178         {
06179             MB_SET_ERR( MB_FAILURE, "Failed to post 2nd iRecv in ghost exchange" );
06180         }
06181 
06182         // Send ack, doesn't matter what data actually is
06183         PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr, mesg_tag_expected - 1,
06184                            sizeof( int ) );
06185         success = MPI_Isend( recv_buff->mem_ptr, sizeof( int ), MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected - 1,
06186                              procConfig.proc_comm(), &sent_ack_req );
06187         if( success != MPI_SUCCESS )
06188         {
06189             MB_SET_ERR( MB_FAILURE, "Failed to send ack in ghost exchange" );
06190         }
06191     }
06192     else if( mpi_status.MPI_TAG == mesg_tag_expected - 1 )
06193     {
06194         // Got an ack back, send the 2nd half of message
06195 
06196         // Should be a large message if we got this
06197         assert( *( (size_t*)send_buff->mem_ptr ) > INITIAL_BUFF_SIZE );
06198 
06199         // Post irecv for next message, then send 2nd message
06200         if( next_buff )
06201         {
06202             // We'll expect a return message
06203             ( *next_incoming )++;
06204             PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, next_buff->mem_ptr, INITIAL_BUFF_SIZE, next_tag,
06205                                *next_incoming );
06206 
06207             success = MPI_Irecv( next_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, next_tag,
06208                                  procConfig.proc_comm(), next_req );
06209             if( success != MPI_SUCCESS )
06210             {
06211                 MB_SET_ERR( MB_FAILURE, "Failed to post next irecv in ghost exchange" );
06212             }
06213         }
06214 
06215         // Send 2nd message
06216         PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, send_buff->mem_ptr + INITIAL_BUFF_SIZE,
06217                            mesg_tag_expected + 1, send_buff->get_stored_size() - INITIAL_BUFF_SIZE );
06218 
06219         assert( send_buff->get_stored_size() - INITIAL_BUFF_SIZE < send_buff->alloc_size &&
06220                 0 <= send_buff->get_stored_size() );
06221         success = MPI_Isend( send_buff->mem_ptr + INITIAL_BUFF_SIZE, send_buff->get_stored_size() - INITIAL_BUFF_SIZE,
06222                              MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &send_req );
06223         if( success != MPI_SUCCESS )
06224         {
06225             MB_SET_ERR( MB_FAILURE, "Failed to send 2nd message in ghost exchange" );
06226         }
06227     }
06228     else if( ( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE ) ||
06229              mpi_status.MPI_TAG == mesg_tag_expected + 1 )
06230     {
06231         // Message completely received - signal that we're done
06232         done = true;
06233     }
06234 
06235     return MB_SUCCESS;
06236 }
06237 
06238 struct ProcList
06239 {
06240     int procs[MAX_SHARING_PROCS];
06241 };
06242 static bool operator<( const ProcList& a, const ProcList& b )
06243 {
06244     for( int i = 0; i < MAX_SHARING_PROCS; i++ )
06245     {
06246         if( a.procs[i] < b.procs[i] )
06247             return true;
06248         else if( b.procs[i] < a.procs[i] )
06249             return false;
06250         else if( a.procs[i] < 0 )
06251             return false;
06252     }
06253     return false;
06254 }
06255 
06256 ErrorCode ParallelComm::check_clean_iface( Range& allsent )
06257 {
06258     // allsent is all entities I think are on interface; go over them, looking
06259     // for zero-valued handles, and fix any I find
06260 
06261     // Keep lists of entities for which teh sharing data changed, grouped
06262     // by set of sharing procs.
06263     typedef std::map< ProcList, Range > procmap_t;
06264     procmap_t old_procs, new_procs;
06265 
06266     ErrorCode result = MB_SUCCESS;
06267     Range::iterator rit;
06268     Range::reverse_iterator rvit;
06269     unsigned char pstatus;
06270     int nump;
06271     ProcList sharedp;
06272     EntityHandle sharedh[MAX_SHARING_PROCS];
06273     for( rvit = allsent.rbegin(); rvit != allsent.rend(); ++rvit )
06274     {
06275         result = get_sharing_data( *rvit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
06276         assert( "Should be shared with at least one other proc" &&
06277                 ( nump > 1 || sharedp.procs[0] != (int)procConfig.proc_rank() ) );
06278         assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
06279 
06280         // Look for first null handle in list
06281         int idx = std::find( sharedh, sharedh + nump, (EntityHandle)0 ) - sharedh;
06282         if( idx == nump ) continue;  // All handles are valid
06283 
06284         ProcList old_list( sharedp );
06285         std::sort( old_list.procs, old_list.procs + nump );
06286         old_procs[old_list].insert( *rvit );
06287 
06288         // Remove null handles and corresponding proc ranks from lists
06289         int new_nump       = idx;
06290         bool removed_owner = !idx;
06291         for( ++idx; idx < nump; ++idx )
06292         {
06293             if( sharedh[idx] )
06294             {
06295                 sharedh[new_nump]       = sharedh[idx];
06296                 sharedp.procs[new_nump] = sharedp.procs[idx];
06297                 ++new_nump;
06298             }
06299         }
06300         sharedp.procs[new_nump] = -1;
06301 
06302         if( removed_owner && new_nump > 1 )
06303         {
06304             // The proc that we choose as the entity owner isn't sharing the
06305             // entity (doesn't have a copy of it). We need to pick a different
06306             // owner. Choose the proc with lowest rank.
06307             idx = std::min_element( sharedp.procs, sharedp.procs + new_nump ) - sharedp.procs;
06308             std::swap( sharedp.procs[0], sharedp.procs[idx] );
06309             std::swap( sharedh[0], sharedh[idx] );
06310             if( sharedp.procs[0] == (int)proc_config().proc_rank() ) pstatus &= ~PSTATUS_NOT_OWNED;
06311         }
06312 
06313         result = set_sharing_data( *rvit, pstatus, nump, new_nump, sharedp.procs, sharedh );MB_CHK_SET_ERR( result, "Failed to set sharing data in check_clean_iface" );
06314 
06315         if( new_nump > 1 )
06316         {
06317             if( new_nump == 2 )
06318             {
06319                 if( sharedp.procs[1] != (int)proc_config().proc_rank() )
06320                 {
06321                     assert( sharedp.procs[0] == (int)proc_config().proc_rank() );
06322                     sharedp.procs[0] = sharedp.procs[1];
06323                 }
06324                 sharedp.procs[1] = -1;
06325             }
06326             else
06327             {
06328                 std::sort( sharedp.procs, sharedp.procs + new_nump );
06329             }
06330             new_procs[sharedp].insert( *rvit );
06331         }
06332     }
06333 
06334     if( old_procs.empty() )
06335     {
06336         assert( new_procs.empty() );
06337         return MB_SUCCESS;
06338     }
06339 
06340     // Update interface sets
06341     procmap_t::iterator pmit;
06342     // std::vector<unsigned char> pstatus_list;
06343     rit = interface_sets().begin();
06344     while( rit != interface_sets().end() )
06345     {
06346         result = get_sharing_data( *rit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data for interface set" );
06347         assert( nump != 2 );
06348         std::sort( sharedp.procs, sharedp.procs + nump );
06349         assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
06350 
06351         pmit = old_procs.find( sharedp );
06352         if( pmit != old_procs.end() )
06353         {
06354             result = mbImpl->remove_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
06355         }
06356 
06357         pmit = new_procs.find( sharedp );
06358         if( pmit == new_procs.end() )
06359         {
06360             int count;
06361             result = mbImpl->get_number_entities_by_handle( *rit, count );MB_CHK_SET_ERR( result, "Failed to get number of entities in interface set" );
06362             if( !count )
06363             {
06364                 result = mbImpl->delete_entities( &*rit, 1 );MB_CHK_SET_ERR( result, "Failed to delete entities from interface set" );
06365                 rit = interface_sets().erase( rit );
06366             }
06367             else
06368             {
06369                 ++rit;
06370             }
06371         }
06372         else
06373         {
06374             result = mbImpl->add_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
06375 
06376             // Remove those that we've processed so that we know which ones
06377             // are new.
06378             new_procs.erase( pmit );
06379             ++rit;
06380         }
06381     }
06382 
06383     // Create interface sets for new proc id combinations
06384     std::fill( sharedh, sharedh + MAX_SHARING_PROCS, 0 );
06385     for( pmit = new_procs.begin(); pmit != new_procs.end(); ++pmit )
06386     {
06387         EntityHandle new_set;
06388         result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
06389         interfaceSets.insert( new_set );
06390 
06391         // Add entities
06392         result = mbImpl->add_entities( new_set, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
06393         // Tag set with the proc rank(s)
06394         assert( pmit->first.procs[0] >= 0 );
06395         pstatus = PSTATUS_SHARED | PSTATUS_INTERFACE;
06396         if( pmit->first.procs[1] == -1 )
06397         {
06398             int other = pmit->first.procs[0];
06399             assert( other != (int)procConfig.proc_rank() );
06400             result = mbImpl->tag_set_data( sharedp_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
06401             sharedh[0] = 0;
06402             result     = mbImpl->tag_set_data( sharedh_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
06403             if( other < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
06404         }
06405         else
06406         {
06407             result = mbImpl->tag_set_data( sharedps_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
06408             result = mbImpl->tag_set_data( sharedhs_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
06409             pstatus |= PSTATUS_MULTISHARED;
06410             if( pmit->first.procs[0] < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
06411         }
06412 
06413         result = mbImpl->tag_set_data( pstatus_tag(), &new_set, 1, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
06414 
06415         // Set pstatus on all interface entities in set
06416         result = mbImpl->tag_clear_data( pstatus_tag(), pmit->second, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface entities with pstatus" );
06417     }
06418 
06419     return MB_SUCCESS;
06420 }
06421 
06422 ErrorCode ParallelComm::set_sharing_data( EntityHandle ent,
06423                                           unsigned char pstatus,
06424                                           int old_nump,
06425                                           int new_nump,
06426                                           int* ps,
06427                                           EntityHandle* hs )
06428 {
06429     // If new nump is less than 3, the entity is no longer mutishared
06430     if( old_nump > 2 && ( pstatus & PSTATUS_MULTISHARED ) && new_nump < 3 )
06431     {
06432         // Unset multishared flag
06433         pstatus ^= PSTATUS_MULTISHARED;
06434     }
06435 
06436     // Check for consistency in input data
06437     // DBG
06438     /*  bool con1 = ((new_nump == 2 && pstatus&PSTATUS_SHARED && !(pstatus&PSTATUS_MULTISHARED)) ||
06439       (new_nump > 2 && pstatus&PSTATUS_SHARED && pstatus&PSTATUS_MULTISHARED)); bool con2 =
06440       (!(pstatus&PSTATUS_GHOST) || pstatus&PSTATUS_SHARED); bool con3 = (new_nump < 3 ||
06441       (pstatus&PSTATUS_NOT_OWNED && ps[0] != (int)rank()) || (!(pstatus&PSTATUS_NOT_OWNED) && ps[0]
06442       == (int)rank())); std::cout<<"current rank = "<<rank()<<std::endl; std::cout<<"condition
06443       1::"<<con1<<std::endl; std::cout<<"condition 2::"<<con2<<std::endl; std::cout<<"condition
06444       3::"<<con3<<std::endl;*/
06445 
06446     // DBG
06447 
06448     assert( new_nump > 1 &&
06449             ( ( new_nump == 2 && pstatus & PSTATUS_SHARED &&
06450                 !( pstatus & PSTATUS_MULTISHARED ) ) ||  // If <= 2 must not be multishared
06451               ( new_nump > 2 && pstatus & PSTATUS_SHARED &&
06452                 pstatus & PSTATUS_MULTISHARED ) ) &&                         // If > 2 procs, must be multishared
06453             ( !( pstatus & PSTATUS_GHOST ) || pstatus & PSTATUS_SHARED ) &&  // If ghost, it must also be shared
06454             ( new_nump < 3 ||
06455               ( pstatus & PSTATUS_NOT_OWNED && ps[0] != (int)rank() ) ||      // I'm not owner and first proc not me
06456               ( !( pstatus & PSTATUS_NOT_OWNED ) && ps[0] == (int)rank() ) )  // I'm owner and first proc is me
06457     );
06458 
06459 #ifndef NDEBUG
06460     {
06461         // Check for duplicates in proc list
06462         std::set< unsigned int > dumprocs;
06463         int dp = 0;
06464         for( ; dp < old_nump && -1 != ps[dp]; dp++ )
06465             dumprocs.insert( ps[dp] );
06466         assert( dp == (int)dumprocs.size() );
06467     }
06468 #endif
06469 
06470     ErrorCode result;
06471     // Reset any old data that needs to be
06472     if( old_nump > 2 && new_nump < 3 )
06473     {
06474         // Need to remove multishared tags
06475         result = mbImpl->tag_delete_data( sharedps_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:1" );
06476         result = mbImpl->tag_delete_data( sharedhs_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:2" );
06477         //    if (new_nump < 2)
06478         //      pstatus = 0x0;
06479         //    else if (ps[0] != (int)proc_config().proc_rank())
06480         //      pstatus |= PSTATUS_NOT_OWNED;
06481     }
06482     else if( ( old_nump < 3 && new_nump > 2 ) || ( old_nump > 1 && new_nump == 1 ) )
06483     {
06484         // Reset sharedp and sharedh tags
06485         int tmp_p          = -1;
06486         EntityHandle tmp_h = 0;
06487         result             = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, &tmp_p );MB_CHK_SET_ERR( result, "set_sharing_data:3" );
06488         result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, &tmp_h );MB_CHK_SET_ERR( result, "set_sharing_data:4" );
06489     }
06490 
06491     assert( "check for multishared/owner I'm first proc" &&
06492             ( !( pstatus & PSTATUS_MULTISHARED ) || ( pstatus & ( PSTATUS_NOT_OWNED | PSTATUS_GHOST ) ) ||
06493               ( ps[0] == (int)rank() ) ) &&
06494             "interface entities should have > 1 proc" && ( !( pstatus & PSTATUS_INTERFACE ) || new_nump > 1 ) &&
06495             "ghost entities should have > 1 proc" && ( !( pstatus & PSTATUS_GHOST ) || new_nump > 1 ) );
06496 
06497     // Now set new data
06498     if( new_nump > 2 )
06499     {
06500         result = mbImpl->tag_set_data( sharedps_tag(), &ent, 1, ps );MB_CHK_SET_ERR( result, "set_sharing_data:5" );
06501         result = mbImpl->tag_set_data( sharedhs_tag(), &ent, 1, hs );MB_CHK_SET_ERR( result, "set_sharing_data:6" );
06502     }
06503     else
06504     {
06505         unsigned int j = ( ps[0] == (int)procConfig.proc_rank() ? 1 : 0 );
06506         assert( -1 != ps[j] );
06507         result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, ps + j );MB_CHK_SET_ERR( result, "set_sharing_data:7" );
06508         result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, hs + j );MB_CHK_SET_ERR( result, "set_sharing_data:8" );
06509     }
06510 
06511     result = mbImpl->tag_set_data( pstatus_tag(), &ent, 1, &pstatus );MB_CHK_SET_ERR( result, "set_sharing_data:9" );
06512 
06513     if( old_nump > 1 && new_nump < 2 ) sharedEnts.erase( ent );
06514 
06515     return result;
06516 }
06517 
06518 ErrorCode ParallelComm::get_sent_ents( const bool is_iface,
06519                                        const int bridge_dim,
06520                                        const int ghost_dim,
06521                                        const int num_layers,
06522                                        const int addl_ents,
06523                                        Range* sent_ents,
06524                                        Range& allsent,
06525                                        TupleList& entprocs )
06526 {
06527     ErrorCode result;
06528     unsigned int ind;
06529     std::vector< unsigned int >::iterator proc_it;
06530     Range tmp_range;
06531 
06532     // Done in a separate loop over procs because sometimes later procs
06533     // need to add info to earlier procs' messages
06534     for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
06535     {
06536         if( !is_iface )
06537         {
06538             result =
06539                 get_ghosted_entities( bridge_dim, ghost_dim, buffProcs[ind], num_layers, addl_ents, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get ghost layers" );
06540         }
06541         else
06542         {
06543             result = get_iface_entities( buffProcs[ind], -1, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get interface layers" );
06544         }
06545 
06546         // Filter out entities already shared with destination
06547         tmp_range.clear();
06548         result = filter_pstatus( sent_ents[ind], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
06549         if( !tmp_range.empty() ) sent_ents[ind] = subtract( sent_ents[ind], tmp_range );
06550 
06551         allsent.merge( sent_ents[ind] );
06552     }
06553 
06554     //===========================================
06555     // Need to get procs each entity is sent to
06556     //===========================================
06557 
06558     // Get the total # of proc/handle pairs
06559     int npairs = 0;
06560     for( ind = 0; ind < buffProcs.size(); ind++ )
06561         npairs += sent_ents[ind].size();
06562 
06563     // Allocate a TupleList of that size
06564     entprocs.initialize( 1, 0, 1, 0, npairs );
06565     entprocs.enableWriteAccess();
06566 
06567     // Put the proc/handle pairs in the list
06568     for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
06569     {
06570         for( Range::iterator rit = sent_ents[ind].begin(); rit != sent_ents[ind].end(); ++rit )
06571         {
06572             entprocs.vi_wr[entprocs.get_n()]  = *proc_it;
06573             entprocs.vul_wr[entprocs.get_n()] = *rit;
06574             entprocs.inc_n();
06575         }
06576     }
06577     // Sort by handle
06578     moab::TupleList::buffer sort_buffer;
06579     sort_buffer.buffer_init( npairs );
06580     entprocs.sort( 1, &sort_buffer );
06581 
06582     entprocs.disableWriteAccess();
06583     sort_buffer.reset();
06584 
06585     return MB_SUCCESS;
06586 }
06587 
06588 ErrorCode ParallelComm::exchange_ghost_cells( ParallelComm** pcs,
06589                                               unsigned int num_procs,
06590                                               int ghost_dim,
06591                                               int bridge_dim,
06592                                               int num_layers,
06593                                               int addl_ents,
06594                                               bool store_remote_handles,
06595                                               EntityHandle* file_sets )
06596 {
06597     // Static version of function, exchanging info through buffers rather
06598     // than through messages
06599 
06600     // If we're only finding out about existing ents, we have to be storing
06601     // remote handles too
06602     assert( num_layers > 0 || store_remote_handles );
06603 
06604     const bool is_iface = !num_layers;
06605 
06606     unsigned int ind;
06607     ParallelComm* pc;
06608     ErrorCode result = MB_SUCCESS;
06609 
06610     std::vector< Error* > ehs( num_procs );
06611     for( unsigned int i = 0; i < num_procs; i++ )
06612     {
06613         result = pcs[i]->get_moab()->query_interface( ehs[i] );
06614         assert( MB_SUCCESS == result );
06615     }
06616 
06617     // When this function is called, buffProcs should already have any
06618     // communicating procs
06619 
06620     //===========================================
06621     // Get entities to be sent to neighbors
06622     //===========================================
06623 
06624     // Done in a separate loop over procs because sometimes later procs
06625     // need to add info to earlier procs' messages
06626     Range sent_ents[MAX_SHARING_PROCS][MAX_SHARING_PROCS], allsent[MAX_SHARING_PROCS];
06627 
06628     //===========================================
06629     // Get entities to be sent to neighbors
06630     //===========================================
06631     TupleList entprocs[MAX_SHARING_PROCS];
06632     for( unsigned int p = 0; p < num_procs; p++ )
06633     {
06634         pc     = pcs[p];
06635         result = pc->get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents[p], allsent[p],
06636                                     entprocs[p] );MB_CHK_SET_ERR( result, "p = " << p << ", get_sent_ents failed" );
06637 
06638         //===========================================
06639         // Pack entities into buffers
06640         //===========================================
06641         for( ind = 0; ind < pc->buffProcs.size(); ind++ )
06642         {
06643             // Entities
06644             pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
06645             result = pc->pack_entities( sent_ents[p][ind], pc->localOwnedBuffs[ind], store_remote_handles,
06646                                         pc->buffProcs[ind], is_iface, &entprocs[p], &allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", packing entities failed" );
06647         }
06648 
06649         entprocs[p].reset();
06650     }
06651 
06652     //===========================================
06653     // Receive/unpack new entities
06654     //===========================================
06655     // Number of incoming messages for ghosts is the number of procs we
06656     // communicate with; for iface, it's the number of those with lower rank
06657     std::vector< std::vector< EntityHandle > > L1hloc[MAX_SHARING_PROCS], L1hrem[MAX_SHARING_PROCS];
06658     std::vector< std::vector< int > > L1p[MAX_SHARING_PROCS];
06659     std::vector< EntityHandle > L2hloc[MAX_SHARING_PROCS], L2hrem[MAX_SHARING_PROCS];
06660     std::vector< unsigned int > L2p[MAX_SHARING_PROCS];
06661     std::vector< EntityHandle > new_ents[MAX_SHARING_PROCS];
06662 
06663     for( unsigned int p = 0; p < num_procs; p++ )
06664     {
06665         L1hloc[p].resize( pcs[p]->buffProcs.size() );
06666         L1hrem[p].resize( pcs[p]->buffProcs.size() );
06667         L1p[p].resize( pcs[p]->buffProcs.size() );
06668     }
06669 
06670     for( unsigned int p = 0; p < num_procs; p++ )
06671     {
06672         pc = pcs[p];
06673 
06674         for( ind = 0; ind < pc->buffProcs.size(); ind++ )
06675         {
06676             // Incoming ghost entities; unpack; returns entities received
06677             // both from sending proc and from owning proc (which may be different)
06678 
06679             // Buffer could be empty, which means there isn't any message to
06680             // unpack (due to this comm proc getting added as a result of indirect
06681             // communication); just skip this unpack
06682             if( pc->localOwnedBuffs[ind]->get_stored_size() == 0 ) continue;
06683 
06684             unsigned int to_p = pc->buffProcs[ind];
06685             pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
06686             result = pcs[to_p]->unpack_entities( pc->localOwnedBuffs[ind]->buff_ptr, store_remote_handles, ind,
06687                                                  is_iface, L1hloc[to_p], L1hrem[to_p], L1p[to_p], L2hloc[to_p],
06688                                                  L2hrem[to_p], L2p[to_p], new_ents[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack entities" );
06689         }
06690     }
06691 
06692     if( is_iface )
06693     {
06694         // Need to check over entities I sent and make sure I received
06695         // handles for them from all expected procs; if not, need to clean
06696         // them up
06697         for( unsigned int p = 0; p < num_procs; p++ )
06698         {
06699             result = pcs[p]->check_clean_iface( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
06700         }
06701 
06702 #ifndef NDEBUG
06703         for( unsigned int p = 0; p < num_procs; p++ )
06704         {
06705             result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
06706         }
06707         result = check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
06708 #endif
06709         return MB_SUCCESS;
06710     }
06711 
06712     //===========================================
06713     // Send local handles for new ghosts to owner, then add
06714     // those to ghost list for that owner
06715     //===========================================
06716     std::vector< unsigned int >::iterator proc_it;
06717     for( unsigned int p = 0; p < num_procs; p++ )
06718     {
06719         pc = pcs[p];
06720 
06721         for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
06722         {
06723             // Skip if iface layer and higher-rank proc
06724             pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
06725             result = pc->pack_remote_handles( L1hloc[p][ind], L1hrem[p][ind], L1p[p][ind], *proc_it,
06726                                               pc->localOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to pack remote handles" );
06727         }
06728     }
06729 
06730     //===========================================
06731     // Process remote handles of my ghosteds
06732     //===========================================
06733     for( unsigned int p = 0; p < num_procs; p++ )
06734     {
06735         pc = pcs[p];
06736 
06737         for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
06738         {
06739             // Incoming remote handles
06740             unsigned int to_p = pc->buffProcs[ind];
06741             pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
06742             result = pcs[to_p]->unpack_remote_handles( p, pc->localOwnedBuffs[ind]->buff_ptr, L2hloc[to_p],
06743                                                        L2hrem[to_p], L2p[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack remote handles" );
06744         }
06745     }
06746 
06747 #ifndef NDEBUG
06748     for( unsigned int p = 0; p < num_procs; p++ )
06749     {
06750         result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
06751     }
06752 
06753     result = ParallelComm::check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
06754 #endif
06755 
06756     if( file_sets )
06757     {
06758         for( unsigned int p = 0; p < num_procs; p++ )
06759         {
06760             if( new_ents[p].empty() ) continue;
06761             result = pcs[p]->get_moab()->add_entities( file_sets[p], &new_ents[p][0], new_ents[p].size() );MB_CHK_SET_ERR( result, "p = " << p << ", failed to add new entities to set" );
06762         }
06763     }
06764 
06765     return MB_SUCCESS;
06766 }
06767 
06768 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& exchange_procs )
06769 {
06770     // Set buffers
06771     int n_proc = exchange_procs.size();
06772     for( int i = 0; i < n_proc; i++ )
06773         get_buffers( exchange_procs[i] );
06774     reset_all_buffers();
06775 
06776     // Post ghost irecv's for entities from all communicating procs
06777     // Index requests the same as buffer/sharing procs indices
06778     int success;
06779     recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06780     recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06781     sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06782 
06783     int incoming = 0;
06784     for( int i = 0; i < n_proc; i++ )
06785     {
06786         int ind = get_buffers( exchange_procs[i] );
06787         incoming++;
06788         PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
06789                            MB_MESG_ENTS_SIZE, incoming );
06790         success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
06791                              MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
06792         if( success != MPI_SUCCESS )
06793         {
06794             MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
06795         }
06796     }
06797 
06798     return MB_SUCCESS;
06799 }
06800 
06801 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs )
06802 {
06803     // Set buffers
06804     int num = shared_procs.size();
06805     for( int i = 0; i < num; i++ )
06806         get_buffers( shared_procs[i] );
06807     reset_all_buffers();
06808     num = remoteOwnedBuffs.size();
06809     for( int i = 0; i < num; i++ )
06810         remoteOwnedBuffs[i]->set_stored_size();
06811     num = localOwnedBuffs.size();
06812     for( int i = 0; i < num; i++ )
06813         localOwnedBuffs[i]->set_stored_size();
06814 
06815     // Post ghost irecv's for entities from all communicating procs
06816     // Index requests the same as buffer/sharing procs indices
06817     int success;
06818     recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06819     recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06820     sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
06821 
06822     int incoming                           = 0;
06823     std::set< unsigned int >::iterator it  = recv_procs.begin();
06824     std::set< unsigned int >::iterator eit = recv_procs.end();
06825     for( ; it != eit; ++it )
06826     {
06827         int ind = get_buffers( *it );
06828         incoming++;
06829         PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
06830                            MB_MESG_ENTS_SIZE, incoming );
06831         success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
06832                              MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
06833         if( success != MPI_SUCCESS )
06834         {
06835             MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
06836         }
06837     }
06838 
06839     return MB_SUCCESS;
06840 }
06841 
06842 ErrorCode ParallelComm::exchange_owned_meshs( std::vector< unsigned int >& exchange_procs,
06843                                               std::vector< Range* >& exchange_ents,
06844                                               std::vector< MPI_Request >& recv_ent_reqs,
06845                                               std::vector< MPI_Request >& recv_remoteh_reqs,
06846                                               bool store_remote_handles,
06847                                               bool wait_all,
06848                                               bool migrate,
06849                                               int dim )
06850 {
06851     // Filter out entities already shared with destination
06852     // Exchange twice for entities and sets
06853     ErrorCode result;
06854     std::vector< unsigned int > exchange_procs_sets;
06855     std::vector< Range* > exchange_sets;
06856     int n_proc = exchange_procs.size();
06857     for( int i = 0; i < n_proc; i++ )
06858     {
06859         Range set_range   = exchange_ents[i]->subset_by_type( MBENTITYSET );
06860         *exchange_ents[i] = subtract( *exchange_ents[i], set_range );
06861         Range* tmp_range  = new Range( set_range );
06862         exchange_sets.push_back( tmp_range );
06863         exchange_procs_sets.push_back( exchange_procs[i] );
06864     }
06865 
06866     if( dim == 2 )
06867     {
06868         // Exchange entities first
06869         result = exchange_owned_mesh( exchange_procs, exchange_ents, recvReqs, recvRemotehReqs, true,
06870                                       store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" );
06871