MOAB: Mesh Oriented datABase
(version 5.4.1)
|
00001 #include "moab/Interface.hpp" 00002 #include "moab/ParallelComm.hpp" 00003 #include "moab/WriteUtilIface.hpp" 00004 #include "moab/ReadUtilIface.hpp" 00005 #include "SequenceManager.hpp" 00006 #include "moab/Error.hpp" 00007 #include "EntitySequence.hpp" 00008 #include "MBTagConventions.hpp" 00009 #include "moab/Skinner.hpp" 00010 #include "MBParallelConventions.h" 00011 #include "moab/Core.hpp" 00012 #include "ElementSequence.hpp" 00013 #include "moab/CN.hpp" 00014 #include "moab/RangeMap.hpp" 00015 #include "moab/MeshTopoUtil.hpp" 00016 #include "TagInfo.hpp" 00017 #include "DebugOutput.hpp" 00018 #include "SharedSetData.hpp" 00019 #include "moab/ScdInterface.hpp" 00020 #include "moab/TupleList.hpp" 00021 #include "moab/gs.hpp" 00022 00023 #include <iostream> 00024 #include <sstream> 00025 #include <algorithm> 00026 #include <functional> 00027 #include <numeric> 00028 00029 #include <cmath> 00030 #include <cstdlib> 00031 #include <cassert> 00032 00033 #ifdef MOAB_HAVE_MPI 00034 #include "moab_mpi.h" 00035 #endif 00036 #ifdef MOAB_HAVE_MPE 00037 #include "mpe.h" 00038 int IFACE_START, IFACE_END; 00039 int GHOST_START, GHOST_END; 00040 int SHAREDV_START, SHAREDV_END; 00041 int RESOLVE_START, RESOLVE_END; 00042 int ENTITIES_START, ENTITIES_END; 00043 int RHANDLES_START, RHANDLES_END; 00044 int OWNED_START, OWNED_END; 00045 #endif 00046 00047 namespace moab 00048 { 00049 00050 const unsigned int ParallelComm::INITIAL_BUFF_SIZE = 1024; 00051 00052 const int MAX_BCAST_SIZE = ( 1 << 28 ); 00053 00054 std::vector< ParallelComm::Buffer* > msgs; 00055 unsigned int __PACK_num = 0, __UNPACK_num = 0, __PACK_count = 0, __UNPACK_count = 0; 00056 std::string __PACK_string, __UNPACK_string; 00057 00058 #ifdef DEBUG_PACKING_TIMES 00059 #define PC( n, m ) \ 00060 { \ 00061 if( __PACK_num == (unsigned int)n && __PACK_string == m ) \ 00062 __PACK_count++; \ 00063 else \ 00064 { \ 00065 if( __PACK_count > 1 ) std::cerr << " (" << __PACK_count << "x)"; \ 00066 __PACK_count = 1; \ 00067 __PACK_string = m; \ 00068 __PACK_num = n; \ 00069 std::cerr << std::endl << "PACK: " << n << m; \ 00070 } \ 00071 } 00072 #define UPC( n, m ) \ 00073 { \ 00074 if( __UNPACK_num == (unsigned int)n && __UNPACK_string == m ) \ 00075 __UNPACK_count++; \ 00076 else \ 00077 { \ 00078 if( __UNPACK_count > 1 ) std::cerr << "(" << __UNPACK_count << "x)"; \ 00079 __UNPACK_count = 1; \ 00080 __UNPACK_string = m; \ 00081 __UNPACK_num = n; \ 00082 std::cerr << std::endl << "UNPACK: " << n << m; \ 00083 } \ 00084 } 00085 #else 00086 #define PC( n, m ) 00087 #define UPC( n, m ) 00088 #endif 00089 00090 template < typename T > 00091 static inline void UNPACK( unsigned char*& buff, T* val, size_t count ) 00092 { 00093 memcpy( val, buff, count * sizeof( T ) ); 00094 buff += count * sizeof( T ); 00095 } 00096 00097 template < typename T > 00098 static inline void PACK( unsigned char*& buff, const T* val, size_t count ) 00099 { 00100 memcpy( buff, val, count * sizeof( T ) ); 00101 buff += count * sizeof( T ); 00102 } 00103 00104 static inline void PACK_INTS( unsigned char*& buff, const int* int_val, size_t num ) 00105 { 00106 PACK( buff, int_val, num ); 00107 PC( num, " ints" ); 00108 } 00109 00110 static inline void PACK_INT( unsigned char*& buff, int int_val ) 00111 { 00112 PACK_INTS( buff, &int_val, 1 ); 00113 } 00114 00115 static inline void PACK_DBLS( unsigned char*& buff, const double* dbl_val, size_t num ) 00116 { 00117 PACK( buff, dbl_val, num ); 00118 PC( num, " doubles" ); 00119 } 00120 00121 // static inline 00122 // void PACK_DBL(unsigned char*& buff, const double dbl_val) 00123 //{ PACK_DBLS(buff, &dbl_val, 1); } 00124 00125 static inline void PACK_EH( unsigned char*& buff, const EntityHandle* eh_val, size_t num ) 00126 { 00127 PACK( buff, eh_val, num ); 00128 PC( num, " handles" ); 00129 } 00130 00131 // static inline 00132 // void PACK_CHAR_64(unsigned char*& buff, const char* str) 00133 //{ 00134 // memcpy(buff, str, 64); 00135 // buff += 64; 00136 // PC(64, " chars"); 00137 //} 00138 00139 static inline void PACK_VOID( unsigned char*& buff, const void* val, size_t num ) 00140 { 00141 PACK( buff, reinterpret_cast< const unsigned char* >( val ), num ); 00142 PC( num, " void" ); 00143 } 00144 00145 static inline void PACK_BYTES( unsigned char*& buff, const void* val, int num ) 00146 { 00147 PACK_INT( buff, num ); 00148 PACK_VOID( buff, val, num ); 00149 } 00150 00151 static inline void PACK_RANGE( unsigned char*& buff, const Range& rng ) 00152 { 00153 PACK_INT( buff, rng.psize() ); 00154 Range::const_pair_iterator cit; 00155 for( cit = rng.const_pair_begin(); cit != rng.const_pair_end(); ++cit ) 00156 { 00157 EntityHandle eh[2] = { cit->first, cit->second }; 00158 PACK_EH( buff, eh, 2 ); 00159 } 00160 PC( rng.psize(), "-subranged range" ); 00161 } 00162 00163 static inline void UNPACK_INTS( unsigned char*& buff, int* int_val, size_t num ) 00164 { 00165 UNPACK( buff, int_val, num ); 00166 UPC( num, " ints" ); 00167 } 00168 00169 static inline void UNPACK_INT( unsigned char*& buff, int& int_val ) 00170 { 00171 UNPACK_INTS( buff, &int_val, 1 ); 00172 } 00173 00174 static inline void UNPACK_DBLS( unsigned char*& buff, double* dbl_val, size_t num ) 00175 { 00176 UNPACK( buff, dbl_val, num ); 00177 UPC( num, " doubles" ); 00178 } 00179 00180 static inline void UNPACK_DBL( unsigned char*& buff, double& dbl_val ) 00181 { 00182 UNPACK_DBLS( buff, &dbl_val, 1 ); 00183 } 00184 00185 static inline void UNPACK_EH( unsigned char*& buff, EntityHandle* eh_val, size_t num ) 00186 { 00187 UNPACK( buff, eh_val, num ); 00188 UPC( num, " handles" ); 00189 } 00190 00191 // static inline 00192 // void UNPACK_CHAR_64(unsigned char*& buff, char* char_val) 00193 //{ 00194 // memcpy(buff, char_val, 64); 00195 // buff += 64; 00196 // UPC(64, " chars"); 00197 //} 00198 00199 static inline void UNPACK_VOID( unsigned char*& buff, void* val, size_t num ) 00200 { 00201 UNPACK( buff, reinterpret_cast< unsigned char* >( val ), num ); 00202 UPC( num, " void" ); 00203 } 00204 00205 static inline void UNPACK_TYPE( unsigned char*& buff, EntityType& type ) 00206 { 00207 int int_type = MBMAXTYPE; 00208 UNPACK_INT( buff, int_type ); 00209 type = static_cast< EntityType >( int_type ); 00210 assert( type >= MBVERTEX && type <= MBMAXTYPE ); 00211 } 00212 00213 static inline void UNPACK_RANGE( unsigned char*& buff, Range& rng ) 00214 { 00215 int num_subs; 00216 EntityHandle eh[2]; 00217 UNPACK_INT( buff, num_subs ); 00218 for( int i = 0; i < num_subs; i++ ) 00219 { 00220 UPC( num_subs, "-subranged range" ); 00221 UNPACK_EH( buff, eh, 2 ); 00222 rng.insert( eh[0], eh[1] ); 00223 } 00224 } 00225 00226 enum MBMessageTag 00227 { 00228 MB_MESG_ANY = MPI_ANY_TAG, 00229 MB_MESG_ENTS_ACK, 00230 MB_MESG_ENTS_SIZE, 00231 MB_MESG_ENTS_LARGE, 00232 MB_MESG_REMOTEH_ACK, 00233 MB_MESG_REMOTEH_SIZE, 00234 MB_MESG_REMOTEH_LARGE, 00235 MB_MESG_TAGS_ACK, 00236 MB_MESG_TAGS_SIZE, 00237 MB_MESG_TAGS_LARGE 00238 }; 00239 00240 static inline size_t RANGE_SIZE( const Range& rng ) 00241 { 00242 return 2 * sizeof( EntityHandle ) * rng.psize() + sizeof( int ); 00243 } 00244 00245 #define PRINT_DEBUG_ISEND( A, B, C, D, E ) print_debug_isend( ( A ), ( B ), ( C ), ( D ), ( E ) ) 00246 #define PRINT_DEBUG_IRECV( A, B, C, D, E, F ) print_debug_irecv( ( A ), ( B ), ( C ), ( D ), ( E ), ( F ) ) 00247 #define PRINT_DEBUG_RECD( A ) print_debug_recd( ( A ) ) 00248 #define PRINT_DEBUG_WAITANY( A, B, C ) print_debug_waitany( ( A ), ( B ), ( C ) ) 00249 00250 void ParallelComm::print_debug_isend( int from, int to, unsigned char* buff, int tag, int sz ) 00251 { 00252 myDebug->tprintf( 3, "Isend, %d->%d, buffer ptr = %p, tag=%d, size=%d\n", from, to, (void*)buff, tag, sz ); 00253 } 00254 00255 void ParallelComm::print_debug_irecv( int to, int from, unsigned char* buff, int sz, int tag, int incoming ) 00256 { 00257 myDebug->tprintf( 3, "Irecv, %d<-%d, buffer ptr = %p, tag=%d, size=%d", to, from, (void*)buff, tag, sz ); 00258 if( tag < MB_MESG_REMOTEH_ACK ) 00259 myDebug->printf( 3, ", incoming1=%d\n", incoming ); 00260 else if( tag < MB_MESG_TAGS_ACK ) 00261 myDebug->printf( 3, ", incoming2=%d\n", incoming ); 00262 else 00263 myDebug->printf( 3, ", incoming=%d\n", incoming ); 00264 } 00265 00266 void ParallelComm::print_debug_recd( MPI_Status status ) 00267 { 00268 if( myDebug->get_verbosity() == 3 ) 00269 { 00270 int this_count; 00271 int success = MPI_Get_count( &status, MPI_UNSIGNED_CHAR, &this_count ); 00272 if( MPI_SUCCESS != success ) this_count = -1; 00273 myDebug->tprintf( 3, "Received from %d, count = %d, tag = %d\n", status.MPI_SOURCE, this_count, 00274 status.MPI_TAG ); 00275 } 00276 } 00277 00278 void ParallelComm::print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc ) 00279 { 00280 if( myDebug->get_verbosity() == 3 ) 00281 { 00282 myDebug->tprintf( 3, "Waitany, p=%d, ", proc ); 00283 if( tag < MB_MESG_REMOTEH_ACK ) 00284 myDebug->print( 3, ", recv_ent_reqs=" ); 00285 else if( tag < MB_MESG_TAGS_ACK ) 00286 myDebug->print( 3, ", recv_remoteh_reqs=" ); 00287 else 00288 myDebug->print( 3, ", recv_tag_reqs=" ); 00289 for( unsigned int i = 0; i < reqs.size(); i++ ) 00290 myDebug->printf( 3, " %p", (void*)(intptr_t)reqs[i] ); 00291 myDebug->print( 3, "\n" ); 00292 } 00293 } 00294 00295 /** Name of tag used to store ParallelComm Index on mesh paritioning sets */ 00296 const char* PARTITIONING_PCOMM_TAG_NAME = "__PRTN_PCOMM"; 00297 00298 /** \brief Tag storing parallel communication objects 00299 * 00300 * This tag stores pointers to ParallelComm communication 00301 * objects; one of these is allocated for each different 00302 * communicator used to read mesh. ParallelComm stores 00303 * partition and interface sets corresponding to its parallel mesh. 00304 * By default, a parallel read uses the first ParallelComm object 00305 * on the interface instance; if instantiated with one, ReadParallel 00306 * adds this object to the interface instance too. 00307 * 00308 * Tag type: opaque 00309 * Tag size: MAX_SHARING_PROCS*sizeof(ParallelComm*) 00310 */ 00311 #define PARALLEL_COMM_TAG_NAME "__PARALLEL_COMM" 00312 00313 ParallelComm::ParallelComm( Interface* impl, MPI_Comm cm, int* id ) 00314 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ), 00315 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ), 00316 myDebug( NULL ) 00317 { 00318 initialize(); 00319 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() ); 00320 if( id ) *id = pcommID; 00321 } 00322 00323 ParallelComm::ParallelComm( Interface* impl, std::vector< unsigned char >& /*tmp_buff*/, MPI_Comm cm, int* id ) 00324 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ), 00325 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ), 00326 myDebug( NULL ) 00327 { 00328 initialize(); 00329 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() ); 00330 if( id ) *id = pcommID; 00331 } 00332 00333 ParallelComm::~ParallelComm() 00334 { 00335 remove_pcomm( this ); 00336 delete_all_buffers(); 00337 delete myDebug; 00338 delete sharedSetData; 00339 } 00340 00341 void ParallelComm::initialize() 00342 { 00343 Core* core = dynamic_cast< Core* >( mbImpl ); 00344 sequenceManager = core->sequence_manager(); 00345 mbImpl->query_interface( errorHandler ); 00346 00347 // Initialize MPI, if necessary 00348 int flag = 1; 00349 int retval = MPI_Initialized( &flag ); 00350 if( MPI_SUCCESS != retval || !flag ) 00351 { 00352 int argc = 0; 00353 char** argv = NULL; 00354 00355 // mpi not initialized yet - initialize here 00356 retval = MPI_Init( &argc, &argv ); 00357 assert( MPI_SUCCESS == retval ); 00358 } 00359 00360 // Reserve space for vectors 00361 buffProcs.reserve( MAX_SHARING_PROCS ); 00362 localOwnedBuffs.reserve( MAX_SHARING_PROCS ); 00363 remoteOwnedBuffs.reserve( MAX_SHARING_PROCS ); 00364 00365 pcommID = add_pcomm( this ); 00366 00367 if( !myDebug ) 00368 { 00369 myDebug = new DebugOutput( "ParallelComm", std::cerr ); 00370 myDebug->set_rank( procConfig.proc_rank() ); 00371 } 00372 } 00373 00374 int ParallelComm::add_pcomm( ParallelComm* pc ) 00375 { 00376 // Add this pcomm to instance tag 00377 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS, (ParallelComm*)NULL ); 00378 Tag pc_tag = pcomm_tag( mbImpl, true ); 00379 assert( 0 != pc_tag ); 00380 00381 const EntityHandle root = 0; 00382 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00383 if( MB_SUCCESS != result && MB_TAG_NOT_FOUND != result ) return -1; 00384 int index = 0; 00385 while( index < MAX_SHARING_PROCS && pc_array[index] ) 00386 index++; 00387 if( index == MAX_SHARING_PROCS ) 00388 { 00389 index = -1; 00390 assert( false ); 00391 } 00392 else 00393 { 00394 pc_array[index] = pc; 00395 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00396 } 00397 return index; 00398 } 00399 00400 void ParallelComm::remove_pcomm( ParallelComm* pc ) 00401 { 00402 // Remove this pcomm from instance tag 00403 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS ); 00404 Tag pc_tag = pcomm_tag( mbImpl, true ); 00405 00406 const EntityHandle root = 0; 00407 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00408 std::vector< ParallelComm* >::iterator pc_it = std::find( pc_array.begin(), pc_array.end(), pc ); 00409 assert( MB_SUCCESS == result && pc_it != pc_array.end() ); 00410 // Empty if test to get around compiler warning about unused var 00411 if( MB_SUCCESS == result ) 00412 { 00413 } 00414 00415 *pc_it = NULL; 00416 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00417 } 00418 00419 //! Assign a global id space, for largest-dimension or all entities (and 00420 //! in either case for vertices too) 00421 ErrorCode ParallelComm::assign_global_ids( EntityHandle this_set, 00422 const int dimension, 00423 const int start_id, 00424 const bool largest_dim_only, 00425 const bool parallel, 00426 const bool owned_only ) 00427 { 00428 Range entities[4]; 00429 ErrorCode result; 00430 std::vector< unsigned char > pstatus; 00431 for( int dim = 0; dim <= dimension; dim++ ) 00432 { 00433 if( dim == 0 || !largest_dim_only || dim == dimension ) 00434 { 00435 result = mbImpl->get_entities_by_dimension( this_set, dim, entities[dim] );MB_CHK_SET_ERR( result, "Failed to get vertices in assign_global_ids" ); 00436 } 00437 00438 // Need to filter out non-locally-owned entities!!! 00439 pstatus.resize( entities[dim].size() ); 00440 result = mbImpl->tag_get_data( pstatus_tag(), entities[dim], &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus in assign_global_ids" ); 00441 00442 Range dum_range; 00443 Range::iterator rit; 00444 unsigned int i; 00445 for( rit = entities[dim].begin(), i = 0; rit != entities[dim].end(); ++rit, i++ ) 00446 if( pstatus[i] & PSTATUS_NOT_OWNED ) dum_range.insert( *rit ); 00447 entities[dim] = subtract( entities[dim], dum_range ); 00448 } 00449 00450 return assign_global_ids( entities, dimension, start_id, parallel, owned_only ); 00451 } 00452 00453 //! Assign a global id space, for largest-dimension or all entities (and 00454 //! in either case for vertices too) 00455 ErrorCode ParallelComm::assign_global_ids( Range entities[], 00456 const int dimension, 00457 const int start_id, 00458 const bool parallel, 00459 const bool owned_only ) 00460 { 00461 int local_num_elements[4]; 00462 ErrorCode result; 00463 for( int dim = 0; dim <= dimension; dim++ ) 00464 { 00465 local_num_elements[dim] = entities[dim].size(); 00466 } 00467 00468 // Communicate numbers 00469 std::vector< int > num_elements( procConfig.proc_size() * 4 ); 00470 #ifdef MOAB_HAVE_MPI 00471 if( procConfig.proc_size() > 1 && parallel ) 00472 { 00473 int retval = 00474 MPI_Allgather( local_num_elements, 4, MPI_INT, &num_elements[0], 4, MPI_INT, procConfig.proc_comm() ); 00475 if( 0 != retval ) return MB_FAILURE; 00476 } 00477 else 00478 #endif 00479 for( int dim = 0; dim < 4; dim++ ) 00480 num_elements[dim] = local_num_elements[dim]; 00481 00482 // My entities start at one greater than total_elems[d] 00483 int total_elems[4] = { start_id, start_id, start_id, start_id }; 00484 00485 for( unsigned int proc = 0; proc < procConfig.proc_rank(); proc++ ) 00486 { 00487 for( int dim = 0; dim < 4; dim++ ) 00488 total_elems[dim] += num_elements[4 * proc + dim]; 00489 } 00490 00491 // Assign global ids now 00492 Tag gid_tag = mbImpl->globalId_tag(); 00493 00494 for( int dim = 0; dim < 4; dim++ ) 00495 { 00496 if( entities[dim].empty() ) continue; 00497 num_elements.resize( entities[dim].size() ); 00498 int i = 0; 00499 for( Range::iterator rit = entities[dim].begin(); rit != entities[dim].end(); ++rit ) 00500 num_elements[i++] = total_elems[dim]++; 00501 00502 result = mbImpl->tag_set_data( gid_tag, entities[dim], &num_elements[0] );MB_CHK_SET_ERR( result, "Failed to set global id tag in assign_global_ids" ); 00503 } 00504 00505 if( owned_only ) return MB_SUCCESS; 00506 00507 // Exchange tags 00508 for( int dim = 1; dim < 4; dim++ ) 00509 entities[0].merge( entities[dim] ); 00510 00511 return exchange_tags( gid_tag, entities[0] ); 00512 } 00513 00514 int ParallelComm::get_buffers( int to_proc, bool* is_new ) 00515 { 00516 int ind = -1; 00517 std::vector< unsigned int >::iterator vit = std::find( buffProcs.begin(), buffProcs.end(), to_proc ); 00518 if( vit == buffProcs.end() ) 00519 { 00520 assert( "shouldn't need buffer to myself" && to_proc != (int)procConfig.proc_rank() ); 00521 ind = buffProcs.size(); 00522 buffProcs.push_back( (unsigned int)to_proc ); 00523 localOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) ); 00524 remoteOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) ); 00525 if( is_new ) *is_new = true; 00526 } 00527 else 00528 { 00529 ind = vit - buffProcs.begin(); 00530 if( is_new ) *is_new = false; 00531 } 00532 assert( ind < MAX_SHARING_PROCS ); 00533 return ind; 00534 } 00535 00536 ErrorCode ParallelComm::broadcast_entities( const int from_proc, 00537 Range& entities, 00538 const bool adjacencies, 00539 const bool tags ) 00540 { 00541 #ifndef MOAB_HAVE_MPI 00542 return MB_FAILURE; 00543 #else 00544 00545 ErrorCode result = MB_SUCCESS; 00546 int success; 00547 int buff_size; 00548 00549 Buffer buff( INITIAL_BUFF_SIZE ); 00550 buff.reset_ptr( sizeof( int ) ); 00551 if( (int)procConfig.proc_rank() == from_proc ) 00552 { 00553 result = add_verts( entities );MB_CHK_SET_ERR( result, "Failed to add adj vertices" ); 00554 00555 buff.reset_ptr( sizeof( int ) ); 00556 result = pack_buffer( entities, adjacencies, tags, false, -1, &buff );MB_CHK_SET_ERR( result, "Failed to compute buffer size in broadcast_entities" ); 00557 buff.set_stored_size(); 00558 buff_size = buff.buff_ptr - buff.mem_ptr; 00559 } 00560 00561 success = MPI_Bcast( &buff_size, 1, MPI_INT, from_proc, procConfig.proc_comm() ); 00562 if( MPI_SUCCESS != success ) 00563 { 00564 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" ); 00565 } 00566 00567 if( !buff_size ) // No data 00568 return MB_SUCCESS; 00569 00570 if( (int)procConfig.proc_rank() != from_proc ) buff.reserve( buff_size ); 00571 00572 size_t offset = 0; 00573 while( buff_size ) 00574 { 00575 int sz = std::min( buff_size, MAX_BCAST_SIZE ); 00576 success = MPI_Bcast( buff.mem_ptr + offset, sz, MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() ); 00577 if( MPI_SUCCESS != success ) 00578 { 00579 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer failed" ); 00580 } 00581 00582 offset += sz; 00583 buff_size -= sz; 00584 } 00585 00586 if( (int)procConfig.proc_rank() != from_proc ) 00587 { 00588 std::vector< std::vector< EntityHandle > > dum1a, dum1b; 00589 std::vector< std::vector< int > > dum1p; 00590 std::vector< EntityHandle > dum2, dum4; 00591 std::vector< unsigned int > dum3; 00592 buff.reset_ptr( sizeof( int ) ); 00593 result = unpack_buffer( buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );MB_CHK_SET_ERR( result, "Failed to unpack buffer in broadcast_entities" ); 00594 std::copy( dum4.begin(), dum4.end(), range_inserter( entities ) ); 00595 } 00596 00597 return MB_SUCCESS; 00598 #endif 00599 } 00600 00601 ErrorCode ParallelComm::scatter_entities( const int from_proc, 00602 std::vector< Range >& entities, 00603 const bool adjacencies, 00604 const bool tags ) 00605 { 00606 #ifndef MOAB_HAVE_MPI 00607 return MB_FAILURE; 00608 #else 00609 ErrorCode result = MB_SUCCESS; 00610 int i, success, buff_size, prev_size; 00611 int nProcs = (int)procConfig.proc_size(); 00612 int* sendCounts = new int[nProcs]; 00613 int* displacements = new int[nProcs]; 00614 sendCounts[0] = sizeof( int ); 00615 displacements[0] = 0; 00616 Buffer buff( INITIAL_BUFF_SIZE ); 00617 buff.reset_ptr( sizeof( int ) ); 00618 buff.set_stored_size(); 00619 unsigned int my_proc = procConfig.proc_rank(); 00620 00621 // Get buffer size array for each remote processor 00622 if( my_proc == (unsigned int)from_proc ) 00623 { 00624 for( i = 1; i < nProcs; i++ ) 00625 { 00626 prev_size = buff.buff_ptr - buff.mem_ptr; 00627 buff.reset_ptr( prev_size + sizeof( int ) ); 00628 result = add_verts( entities[i] );MB_CHK_SET_ERR( result, "Failed to add verts" ); 00629 00630 result = pack_buffer( entities[i], adjacencies, tags, false, -1, &buff ); 00631 if( MB_SUCCESS != result ) 00632 { 00633 delete[] sendCounts; 00634 delete[] displacements; 00635 MB_SET_ERR( result, "Failed to pack buffer in scatter_entities" ); 00636 } 00637 00638 buff_size = buff.buff_ptr - buff.mem_ptr - prev_size; 00639 *( (int*)( buff.mem_ptr + prev_size ) ) = buff_size; 00640 sendCounts[i] = buff_size; 00641 } 00642 } 00643 00644 // Broadcast buffer size array 00645 success = MPI_Bcast( sendCounts, nProcs, MPI_INT, from_proc, procConfig.proc_comm() ); 00646 if( MPI_SUCCESS != success ) 00647 { 00648 delete[] sendCounts; 00649 delete[] displacements; 00650 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" ); 00651 } 00652 00653 for( i = 1; i < nProcs; i++ ) 00654 { 00655 displacements[i] = displacements[i - 1] + sendCounts[i - 1]; 00656 } 00657 00658 Buffer rec_buff; 00659 rec_buff.reserve( sendCounts[my_proc] ); 00660 00661 // Scatter actual geometry 00662 success = MPI_Scatterv( buff.mem_ptr, sendCounts, displacements, MPI_UNSIGNED_CHAR, rec_buff.mem_ptr, 00663 sendCounts[my_proc], MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() ); 00664 00665 if( MPI_SUCCESS != success ) 00666 { 00667 delete[] sendCounts; 00668 delete[] displacements; 00669 MB_SET_ERR( MB_FAILURE, "MPI_Scatterv of buffer failed" ); 00670 } 00671 00672 // Unpack in remote processors 00673 if( my_proc != (unsigned int)from_proc ) 00674 { 00675 std::vector< std::vector< EntityHandle > > dum1a, dum1b; 00676 std::vector< std::vector< int > > dum1p; 00677 std::vector< EntityHandle > dum2, dum4; 00678 std::vector< unsigned int > dum3; 00679 rec_buff.reset_ptr( sizeof( int ) ); 00680 result = unpack_buffer( rec_buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 ); 00681 if( MB_SUCCESS != result ) 00682 { 00683 delete[] sendCounts; 00684 delete[] displacements; 00685 MB_SET_ERR( result, "Failed to unpack buffer in scatter_entities" ); 00686 } 00687 00688 std::copy( dum4.begin(), dum4.end(), range_inserter( entities[my_proc] ) ); 00689 } 00690 00691 delete[] sendCounts; 00692 delete[] displacements; 00693 00694 return MB_SUCCESS; 00695 #endif 00696 } 00697 00698 ErrorCode ParallelComm::send_entities( const int to_proc, 00699 Range& orig_ents, 00700 const bool adjs, 00701 const bool tags, 00702 const bool store_remote_handles, 00703 const bool is_iface, 00704 Range& /*final_ents*/, 00705 int& incoming1, 00706 int& incoming2, 00707 TupleList& entprocs, 00708 std::vector< MPI_Request >& recv_remoteh_reqs, 00709 bool /*wait_all*/ ) 00710 { 00711 #ifndef MOAB_HAVE_MPI 00712 return MB_FAILURE; 00713 #else 00714 // Pack entities to local buffer 00715 int ind = get_buffers( to_proc ); 00716 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 00717 00718 // Add vertices 00719 ErrorCode result = add_verts( orig_ents );MB_CHK_SET_ERR( result, "Failed to add verts in send_entities" ); 00720 00721 // Filter out entities already shared with destination 00722 Range tmp_range; 00723 result = filter_pstatus( orig_ents, PSTATUS_SHARED, PSTATUS_AND, to_proc, &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 00724 if( !tmp_range.empty() ) 00725 { 00726 orig_ents = subtract( orig_ents, tmp_range ); 00727 } 00728 00729 result = pack_buffer( orig_ents, adjs, tags, store_remote_handles, to_proc, localOwnedBuffs[ind], &entprocs );MB_CHK_SET_ERR( result, "Failed to pack buffer in send_entities" ); 00730 00731 // Send buffer 00732 result = send_buffer( to_proc, localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], recvReqs[2 * ind + 1], 00733 (int*)( remoteOwnedBuffs[ind]->mem_ptr ), 00734 //&ackbuff, 00735 incoming1, MB_MESG_REMOTEH_SIZE, 00736 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind] : NULL ), 00737 &recv_remoteh_reqs[2 * ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to send buffer" ); 00738 00739 return MB_SUCCESS; 00740 #endif 00741 } 00742 00743 ErrorCode ParallelComm::send_entities( std::vector< unsigned int >& send_procs, 00744 std::vector< Range* >& send_ents, 00745 int& incoming1, 00746 int& incoming2, 00747 const bool store_remote_handles ) 00748 { 00749 #ifdef MOAB_HAVE_MPE 00750 if( myDebug->get_verbosity() == 2 ) 00751 { 00752 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_entities." ); 00753 } 00754 #endif 00755 myDebug->tprintf( 1, "Entering send_entities\n" ); 00756 if( myDebug->get_verbosity() == 4 ) 00757 { 00758 msgs.clear(); 00759 msgs.reserve( MAX_SHARING_PROCS ); 00760 } 00761 00762 unsigned int i; 00763 int ind; 00764 ErrorCode result = MB_SUCCESS; 00765 00766 // Set buffProcs with communicating procs 00767 unsigned int n_proc = send_procs.size(); 00768 for( i = 0; i < n_proc; i++ ) 00769 { 00770 ind = get_buffers( send_procs[i] ); 00771 result = add_verts( *send_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" ); 00772 00773 // Filter out entities already shared with destination 00774 Range tmp_range; 00775 result = filter_pstatus( *send_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 00776 if( !tmp_range.empty() ) 00777 { 00778 *send_ents[i] = subtract( *send_ents[i], tmp_range ); 00779 } 00780 } 00781 00782 //=========================================== 00783 // Get entities to be sent to neighbors 00784 // Need to get procs each entity is sent to 00785 //=========================================== 00786 Range allsent, tmp_range; 00787 int npairs = 0; 00788 TupleList entprocs; 00789 for( i = 0; i < n_proc; i++ ) 00790 { 00791 int n_ents = send_ents[i]->size(); 00792 if( n_ents > 0 ) 00793 { 00794 npairs += n_ents; // Get the total # of proc/handle pairs 00795 allsent.merge( *send_ents[i] ); 00796 } 00797 } 00798 00799 // Allocate a TupleList of that size 00800 entprocs.initialize( 1, 0, 1, 0, npairs ); 00801 entprocs.enableWriteAccess(); 00802 00803 // Put the proc/handle pairs in the list 00804 for( i = 0; i < n_proc; i++ ) 00805 { 00806 for( Range::iterator rit = send_ents[i]->begin(); rit != send_ents[i]->end(); ++rit ) 00807 { 00808 entprocs.vi_wr[entprocs.get_n()] = send_procs[i]; 00809 entprocs.vul_wr[entprocs.get_n()] = *rit; 00810 entprocs.inc_n(); 00811 } 00812 } 00813 00814 // Sort by handle 00815 moab::TupleList::buffer sort_buffer; 00816 sort_buffer.buffer_init( npairs ); 00817 entprocs.sort( 1, &sort_buffer ); 00818 entprocs.disableWriteAccess(); 00819 sort_buffer.reset(); 00820 00821 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(), 00822 (unsigned long)allsent.size() ); 00823 00824 //=========================================== 00825 // Pack and send ents from this proc to others 00826 //=========================================== 00827 for( i = 0; i < n_proc; i++ ) 00828 { 00829 if( send_ents[i]->size() > 0 ) 00830 { 00831 ind = get_buffers( send_procs[i] ); 00832 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", send_ents[i]->compactness(), 00833 (unsigned long)send_ents[i]->size() ); 00834 // Reserve space on front for size and for initial buff size 00835 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 00836 result = pack_buffer( *send_ents[i], false, true, store_remote_handles, buffProcs[ind], 00837 localOwnedBuffs[ind], &entprocs, &allsent ); 00838 00839 if( myDebug->get_verbosity() == 4 ) 00840 { 00841 msgs.resize( msgs.size() + 1 ); 00842 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 00843 } 00844 00845 // Send the buffer (size stored in front in send_buffer) 00846 result = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], 00847 recvReqs[2 * ind + 1], &ackbuff, incoming1, MB_MESG_REMOTEH_SIZE, 00848 ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recvRemotehReqs[2 * ind], 00849 &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost send" ); 00850 } 00851 } 00852 entprocs.reset(); 00853 00854 #ifdef MOAB_HAVE_MPE 00855 if( myDebug->get_verbosity() == 2 ) 00856 { 00857 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_entities." ); 00858 } 00859 #endif 00860 00861 return MB_SUCCESS; 00862 } 00863 00864 ///////////////////////////////////////////////////////////////////////////////// 00865 // Send and Receive routines for a sequence of entities: use case UMR 00866 ///////////////////////////////////////////////////////////////////////////////// 00867 void print_buff( unsigned char* ch, int size ) 00868 { 00869 for( int i = 0; i < size; i++ ) 00870 std::cout << ch[i]; 00871 std::cout << "\n"; 00872 } 00873 ErrorCode ParallelComm::send_recv_entities( std::vector< int >& send_procs, 00874 std::vector< std::vector< int > >& msgsizes, 00875 std::vector< std::vector< EntityHandle > >& senddata, 00876 std::vector< std::vector< EntityHandle > >& recvdata ) 00877 { 00878 #ifdef USE_MPE 00879 if( myDebug->get_verbosity() == 2 ) 00880 { 00881 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_recv_entities." ); 00882 } 00883 #endif 00884 myDebug->tprintf( 1, "Entering send_recv_entities\n" ); 00885 if( myDebug->get_verbosity() == 4 ) 00886 { 00887 msgs.clear(); 00888 msgs.reserve( MAX_SHARING_PROCS ); 00889 } 00890 00891 // unsigned int i; 00892 int i, ind, success; 00893 ErrorCode error = MB_SUCCESS; 00894 00895 //=========================================== 00896 // Pack and send ents from this proc to others 00897 //=========================================== 00898 00899 // std::cout<<"resetting all buffers"<<std::endl; 00900 00901 reset_all_buffers(); 00902 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 00903 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 00904 int ack_buff; 00905 int incoming = 0; 00906 00907 std::vector< unsigned int >::iterator sit; 00908 00909 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 00910 { 00911 incoming++; 00912 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 00913 MB_MESG_ENTS_SIZE, incoming ); 00914 00915 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit, 00916 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] ); 00917 if( success != MPI_SUCCESS ) 00918 { 00919 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in send_recv_entities" ); 00920 } 00921 } 00922 00923 // std::set<unsigned int>::iterator it; 00924 for( i = 0; i < (int)send_procs.size(); i++ ) 00925 { 00926 // Get index of the shared processor in the local buffer 00927 ind = get_buffers( send_procs[i] ); 00928 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 00929 00930 int buff_size = msgsizes[i].size() * sizeof( int ) + senddata[i].size() * sizeof( EntityHandle ); 00931 localOwnedBuffs[ind]->check_space( buff_size ); 00932 00933 // Pack entities 00934 std::vector< int > msg; 00935 msg.insert( msg.end(), msgsizes[i].begin(), msgsizes[i].end() ); 00936 PACK_INTS( localOwnedBuffs[ind]->buff_ptr, &msg[0], msg.size() ); 00937 00938 std::vector< EntityHandle > entities; 00939 entities.insert( entities.end(), senddata[i].begin(), senddata[i].end() ); 00940 PACK_EH( localOwnedBuffs[ind]->buff_ptr, &entities[0], entities.size() ); 00941 localOwnedBuffs[ind]->set_stored_size(); 00942 00943 if( myDebug->get_verbosity() == 4 ) 00944 { 00945 msgs.resize( msgs.size() + 1 ); 00946 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 00947 } 00948 00949 // Send the buffer (size stored in front in send_buffer) 00950 error = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind], 00951 recv_ent_reqs[3 * ind + 2], &ack_buff, incoming );MB_CHK_SET_ERR( error, "Failed to Isend in send_recv_entities" ); 00952 } 00953 00954 //=========================================== 00955 // Receive and unpack ents from received data 00956 //=========================================== 00957 00958 while( incoming ) 00959 { 00960 00961 MPI_Status status; 00962 int index_in_recv_requests; 00963 00964 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 00965 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &index_in_recv_requests, &status ); 00966 if( MPI_SUCCESS != success ) 00967 { 00968 MB_SET_ERR( MB_FAILURE, "Failed in waitany in send_recv_entities" ); 00969 } 00970 00971 // Processor index in the list is divided by 3 00972 ind = index_in_recv_requests / 3; 00973 00974 PRINT_DEBUG_RECD( status ); 00975 00976 // OK, received something; decrement incoming counter 00977 incoming--; 00978 00979 bool done = false; 00980 00981 error = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind], 00982 recv_ent_reqs[3 * ind + 1], // This is for receiving the second message 00983 recv_ent_reqs[3 * ind + 2], // This would be for ack, but it is not 00984 // used; consider removing it 00985 incoming, localOwnedBuffs[ind], 00986 sendReqs[3 * ind + 1], // Send request for sending the second message 00987 sendReqs[3 * ind + 2], // This is for sending the ack 00988 done );MB_CHK_SET_ERR( error, "Failed to resize recv buffer" ); 00989 00990 if( done ) 00991 { 00992 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 00993 00994 int from_proc = status.MPI_SOURCE; 00995 int idx = std::find( send_procs.begin(), send_procs.end(), from_proc ) - send_procs.begin(); 00996 00997 int msg = msgsizes[idx].size(); 00998 std::vector< int > recvmsg( msg ); 00999 int ndata = senddata[idx].size(); 01000 std::vector< EntityHandle > dum_vec( ndata ); 01001 01002 UNPACK_INTS( remoteOwnedBuffs[ind]->buff_ptr, &recvmsg[0], msg ); 01003 UNPACK_EH( remoteOwnedBuffs[ind]->buff_ptr, &dum_vec[0], ndata ); 01004 01005 recvdata[idx].insert( recvdata[idx].end(), dum_vec.begin(), dum_vec.end() ); 01006 } 01007 } 01008 01009 #ifdef USE_MPE 01010 if( myDebug->get_verbosity() == 2 ) 01011 { 01012 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_recv_entities." ); 01013 } 01014 #endif 01015 01016 return MB_SUCCESS; 01017 } 01018 01019 ErrorCode ParallelComm::update_remote_data( EntityHandle entity, 01020 std::vector< int >& procs, 01021 std::vector< EntityHandle >& handles ) 01022 { 01023 ErrorCode error; 01024 unsigned char pstatus = PSTATUS_INTERFACE; 01025 01026 int procmin = *std::min_element( procs.begin(), procs.end() ); 01027 01028 if( (int)rank() > procmin ) 01029 pstatus |= PSTATUS_NOT_OWNED; 01030 else 01031 procmin = rank(); 01032 01033 // DBG 01034 // std::cout<<"entity = "<<entity<<std::endl; 01035 // for (int j=0; j<procs.size(); j++) 01036 // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl; 01037 // DBG 01038 01039 if( (int)procs.size() > 1 ) 01040 { 01041 procs.push_back( rank() ); 01042 handles.push_back( entity ); 01043 01044 int idx = std::find( procs.begin(), procs.end(), procmin ) - procs.begin(); 01045 01046 std::iter_swap( procs.begin(), procs.begin() + idx ); 01047 std::iter_swap( handles.begin(), handles.begin() + idx ); 01048 01049 // DBG 01050 // std::cout<<"entity = "<<entity<<std::endl; 01051 // for (int j=0; j<procs.size(); j++) 01052 // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl; 01053 // DBG 01054 } 01055 01056 // if ((entity == 10388) && (rank()==1)) 01057 // std::cout<<"Here"<<std::endl; 01058 01059 error = update_remote_data( entity, &procs[0], &handles[0], procs.size(), pstatus );MB_CHK_ERR( error ); 01060 01061 return MB_SUCCESS; 01062 } 01063 01064 ErrorCode ParallelComm::get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc ) 01065 { 01066 ErrorCode error; 01067 std::vector< EntityHandle > newents; 01068 error = get_remote_handles( true, local_vec, rem_vec, num_ents, to_proc, newents );MB_CHK_ERR( error ); 01069 01070 return MB_SUCCESS; 01071 } 01072 01073 ////////////////////////////////////////////////////////////////// 01074 01075 ErrorCode ParallelComm::recv_entities( const int from_proc, 01076 const bool store_remote_handles, 01077 const bool is_iface, 01078 Range& final_ents, 01079 int& incoming1, 01080 int& incoming2, 01081 std::vector< std::vector< EntityHandle > >& L1hloc, 01082 std::vector< std::vector< EntityHandle > >& L1hrem, 01083 std::vector< std::vector< int > >& L1p, 01084 std::vector< EntityHandle >& L2hloc, 01085 std::vector< EntityHandle >& L2hrem, 01086 std::vector< unsigned int >& L2p, 01087 std::vector< MPI_Request >& recv_remoteh_reqs, 01088 bool /*wait_all*/ ) 01089 { 01090 #ifndef MOAB_HAVE_MPI 01091 return MB_FAILURE; 01092 #else 01093 // Non-blocking receive for the first message (having size info) 01094 int ind1 = get_buffers( from_proc ); 01095 incoming1++; 01096 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, 01097 MB_MESG_ENTS_SIZE, incoming1 ); 01098 int success = MPI_Irecv( remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, 01099 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind1] ); 01100 if( success != MPI_SUCCESS ) 01101 { 01102 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 01103 } 01104 01105 // Receive messages in while loop 01106 return recv_messages( from_proc, store_remote_handles, is_iface, final_ents, incoming1, incoming2, L1hloc, L1hrem, 01107 L1p, L2hloc, L2hrem, L2p, recv_remoteh_reqs ); 01108 #endif 01109 } 01110 01111 ErrorCode ParallelComm::recv_entities( std::set< unsigned int >& recv_procs, 01112 int incoming1, 01113 int incoming2, 01114 const bool store_remote_handles, 01115 const bool migrate ) 01116 { 01117 //=========================================== 01118 // Receive/unpack new entities 01119 //=========================================== 01120 // Number of incoming messages is the number of procs we communicate with 01121 int success, ind, i; 01122 ErrorCode result; 01123 MPI_Status status; 01124 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() ); 01125 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() ); 01126 std::vector< std::vector< int > > L1p( buffProcs.size() ); 01127 std::vector< EntityHandle > L2hloc, L2hrem; 01128 std::vector< unsigned int > L2p; 01129 std::vector< EntityHandle > new_ents; 01130 01131 while( incoming1 ) 01132 { 01133 // Wait for all recvs of ents before proceeding to sending remote handles, 01134 // b/c some procs may have sent to a 3rd proc ents owned by me; 01135 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 01136 01137 success = MPI_Waitany( 2 * buffProcs.size(), &recvReqs[0], &ind, &status ); 01138 if( MPI_SUCCESS != success ) 01139 { 01140 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 01141 } 01142 01143 PRINT_DEBUG_RECD( status ); 01144 01145 // OK, received something; decrement incoming counter 01146 incoming1--; 01147 bool done = false; 01148 01149 // In case ind is for ack, we need index of one before it 01150 unsigned int base_ind = 2 * ( ind / 2 ); 01151 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 2], recvReqs[ind], recvReqs[ind + 1], 01152 incoming1, localOwnedBuffs[ind / 2], sendReqs[base_ind], sendReqs[base_ind + 1], done, 01153 ( store_remote_handles ? localOwnedBuffs[ind / 2] : NULL ), MB_MESG_REMOTEH_SIZE, 01154 &recvRemotehReqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 01155 01156 if( done ) 01157 { 01158 if( myDebug->get_verbosity() == 4 ) 01159 { 01160 msgs.resize( msgs.size() + 1 ); 01161 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 2] ); 01162 } 01163 01164 // Message completely received - process buffer that was sent 01165 remoteOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) ); 01166 result = unpack_buffer( remoteOwnedBuffs[ind / 2]->buff_ptr, store_remote_handles, buffProcs[ind / 2], 01167 ind / 2, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true ); 01168 if( MB_SUCCESS != result ) 01169 { 01170 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl; 01171 print_buffer( remoteOwnedBuffs[ind / 2]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 2], false ); 01172 return result; 01173 } 01174 01175 if( recvReqs.size() != 2 * buffProcs.size() ) 01176 { 01177 // Post irecv's for remote handles from new proc 01178 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01179 for( i = recvReqs.size(); i < (int)( 2 * buffProcs.size() ); i += 2 ) 01180 { 01181 localOwnedBuffs[i / 2]->reset_buffer(); 01182 incoming2++; 01183 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 2], localOwnedBuffs[i / 2]->mem_ptr, 01184 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 ); 01185 success = MPI_Irecv( localOwnedBuffs[i / 2]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, 01186 buffProcs[i / 2], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(), 01187 &recvRemotehReqs[i] ); 01188 if( success != MPI_SUCCESS ) 01189 { 01190 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" ); 01191 } 01192 } 01193 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01194 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01195 } 01196 } 01197 } 01198 01199 // Assign and remove newly created elements from/to receive processor 01200 result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" ); 01201 if( migrate ) 01202 { 01203 // result = remove_entities_part(allsent, procConfig.proc_rank());MB_CHK_SET_ERR(ressult, 01204 // "Failed to remove entities to part"); 01205 } 01206 01207 // Add requests for any new addl procs 01208 if( recvReqs.size() != 2 * buffProcs.size() ) 01209 { 01210 // Shouldn't get here... 01211 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" ); 01212 } 01213 01214 #ifdef MOAB_HAVE_MPE 01215 if( myDebug->get_verbosity() == 2 ) 01216 { 01217 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending recv entities." ); 01218 } 01219 #endif 01220 01221 //=========================================== 01222 // Send local handles for new entity to owner 01223 //=========================================== 01224 std::set< unsigned int >::iterator it = recv_procs.begin(); 01225 std::set< unsigned int >::iterator eit = recv_procs.end(); 01226 for( ; it != eit; ++it ) 01227 { 01228 ind = get_buffers( *it ); 01229 // Reserve space on front for size and for initial buff size 01230 remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 01231 01232 result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 01233 remoteOwnedBuffs[ind]->set_stored_size(); 01234 01235 if( myDebug->get_verbosity() == 4 ) 01236 { 01237 msgs.resize( msgs.size() + 1 ); 01238 msgs.back() = new Buffer( *remoteOwnedBuffs[ind] ); 01239 } 01240 result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[2 * ind], 01241 recvRemotehReqs[2 * ind + 1], &ackbuff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 01242 } 01243 01244 //=========================================== 01245 // Process remote handles of my ghosteds 01246 //=========================================== 01247 while( incoming2 ) 01248 { 01249 PRINT_DEBUG_WAITANY( recvRemotehReqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 01250 success = MPI_Waitany( 2 * buffProcs.size(), &recvRemotehReqs[0], &ind, &status ); 01251 if( MPI_SUCCESS != success ) 01252 { 01253 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 01254 } 01255 01256 // OK, received something; decrement incoming counter 01257 incoming2--; 01258 01259 PRINT_DEBUG_RECD( status ); 01260 bool done = false; 01261 unsigned int base_ind = 2 * ( ind / 2 ); 01262 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 2], recvRemotehReqs[ind], 01263 recvRemotehReqs[ind + 1], incoming2, remoteOwnedBuffs[ind / 2], sendReqs[base_ind], 01264 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 01265 if( done ) 01266 { 01267 // Incoming remote handles 01268 if( myDebug->get_verbosity() == 4 ) 01269 { 01270 msgs.resize( msgs.size() + 1 ); 01271 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 01272 } 01273 01274 localOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) ); 01275 result = 01276 unpack_remote_handles( buffProcs[ind / 2], localOwnedBuffs[ind / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 01277 } 01278 } 01279 01280 #ifdef MOAB_HAVE_MPE 01281 if( myDebug->get_verbosity() == 2 ) 01282 { 01283 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." ); 01284 MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending recv entities (still doing checks)." ); 01285 } 01286 #endif 01287 myDebug->tprintf( 1, "Exiting recv_entities.\n" ); 01288 01289 return MB_SUCCESS; 01290 } 01291 01292 ErrorCode ParallelComm::recv_messages( const int from_proc, 01293 const bool store_remote_handles, 01294 const bool is_iface, 01295 Range& final_ents, 01296 int& incoming1, 01297 int& incoming2, 01298 std::vector< std::vector< EntityHandle > >& L1hloc, 01299 std::vector< std::vector< EntityHandle > >& L1hrem, 01300 std::vector< std::vector< int > >& L1p, 01301 std::vector< EntityHandle >& L2hloc, 01302 std::vector< EntityHandle >& L2hrem, 01303 std::vector< unsigned int >& L2p, 01304 std::vector< MPI_Request >& recv_remoteh_reqs ) 01305 { 01306 #ifndef MOAB_HAVE_MPI 01307 return MB_FAILURE; 01308 #else 01309 MPI_Status status; 01310 ErrorCode result; 01311 int ind1 = get_buffers( from_proc ); 01312 int success, ind2; 01313 std::vector< EntityHandle > new_ents; 01314 01315 // Wait and receive messages 01316 while( incoming1 ) 01317 { 01318 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() ); 01319 success = MPI_Waitany( 2, &recvReqs[2 * ind1], &ind2, &status ); 01320 if( MPI_SUCCESS != success ) 01321 { 01322 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_messages" ); 01323 } 01324 01325 PRINT_DEBUG_RECD( status ); 01326 01327 // OK, received something; decrement incoming counter 01328 incoming1--; 01329 bool done = false; 01330 01331 // In case ind is for ack, we need index of one before it 01332 ind2 += 2 * ind1; 01333 unsigned int base_ind = 2 * ( ind2 / 2 ); 01334 01335 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind2 / 2], 01336 // recvbuff, 01337 recvReqs[ind2], recvReqs[ind2 + 1], incoming1, localOwnedBuffs[ind2 / 2], 01338 sendReqs[base_ind], sendReqs[base_ind + 1], done, 01339 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind2 / 2] : NULL ), 01340 MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 01341 01342 if( done ) 01343 { 01344 // If it is done, unpack buffer 01345 remoteOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) ); 01346 result = unpack_buffer( remoteOwnedBuffs[ind2 / 2]->buff_ptr, store_remote_handles, from_proc, ind2 / 2, 01347 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );MB_CHK_SET_ERR( result, "Failed to unpack buffer in recev_messages" ); 01348 01349 std::copy( new_ents.begin(), new_ents.end(), range_inserter( final_ents ) ); 01350 01351 // Send local handles for new elements to owner 01352 // Reserve space on front for size and for initial buff size 01353 remoteOwnedBuffs[ind2 / 2]->reset_buffer( sizeof( int ) ); 01354 01355 result = pack_remote_handles( L1hloc[ind2 / 2], L1hrem[ind2 / 2], L1p[ind2 / 2], from_proc, 01356 remoteOwnedBuffs[ind2 / 2] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 01357 remoteOwnedBuffs[ind2 / 2]->set_stored_size(); 01358 01359 result = send_buffer( buffProcs[ind2 / 2], remoteOwnedBuffs[ind2 / 2], MB_MESG_REMOTEH_SIZE, sendReqs[ind2], 01360 recv_remoteh_reqs[ind2 + 1], (int*)( localOwnedBuffs[ind2 / 2]->mem_ptr ), 01361 //&ackbuff, 01362 incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 01363 } 01364 } 01365 01366 return MB_SUCCESS; 01367 #endif 01368 } 01369 01370 ErrorCode ParallelComm::recv_remote_handle_messages( const int from_proc, 01371 int& incoming2, 01372 std::vector< EntityHandle >& L2hloc, 01373 std::vector< EntityHandle >& L2hrem, 01374 std::vector< unsigned int >& L2p, 01375 std::vector< MPI_Request >& recv_remoteh_reqs ) 01376 { 01377 #ifndef MOAB_HAVE_MPI 01378 return MB_FAILURE; 01379 #else 01380 MPI_Status status; 01381 ErrorCode result; 01382 int ind1 = get_buffers( from_proc ); 01383 int success, ind2; 01384 01385 while( incoming2 ) 01386 { 01387 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 01388 success = MPI_Waitany( 2, &recv_remoteh_reqs[2 * ind1], &ind2, &status ); 01389 if( MPI_SUCCESS != success ) 01390 { 01391 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_remote_handle_messages" ); 01392 } 01393 01394 // OK, received something; decrement incoming counter 01395 incoming2--; 01396 01397 PRINT_DEBUG_RECD( status ); 01398 01399 bool done = false; 01400 ind2 += 2 * ind1; 01401 unsigned int base_ind = 2 * ( ind2 / 2 ); 01402 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind2 / 2], recv_remoteh_reqs[ind2], 01403 recv_remoteh_reqs[ind2 + 1], incoming2, remoteOwnedBuffs[ind2 / 2], sendReqs[base_ind], 01404 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 01405 if( done ) 01406 { 01407 // Incoming remote handles 01408 localOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) ); 01409 result = 01410 unpack_remote_handles( buffProcs[ind2 / 2], localOwnedBuffs[ind2 / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 01411 } 01412 } 01413 01414 return MB_SUCCESS; 01415 #endif 01416 } 01417 01418 ErrorCode ParallelComm::pack_buffer( Range& orig_ents, 01419 const bool /*adjacencies*/, 01420 const bool tags, 01421 const bool store_remote_handles, 01422 const int to_proc, 01423 Buffer* buff, 01424 TupleList* entprocs, 01425 Range* allsent ) 01426 { 01427 // Pack the buffer with the entity ranges, adjacencies, and tags sections 01428 // 01429 // Note: new entities used in subsequent connectivity lists, sets, or tags, 01430 // are referred to as (MBMAXTYPE + index), where index is into vector 01431 // of new entities, 0-based 01432 ErrorCode result; 01433 01434 Range set_range; 01435 std::vector< Tag > all_tags; 01436 std::vector< Range > tag_ranges; 01437 01438 Range::const_iterator rit; 01439 01440 // Entities 01441 result = pack_entities( orig_ents, buff, store_remote_handles, to_proc, false, entprocs, allsent );MB_CHK_SET_ERR( result, "Packing entities failed" ); 01442 01443 // Sets 01444 result = pack_sets( orig_ents, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing sets (count) failed" ); 01445 01446 // Tags 01447 Range final_ents; 01448 if( tags ) 01449 { 01450 result = get_tag_send_list( orig_ents, all_tags, tag_ranges );MB_CHK_SET_ERR( result, "Failed to get tagged entities" ); 01451 result = pack_tags( orig_ents, all_tags, all_tags, tag_ranges, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing tags (count) failed" ); 01452 } 01453 else 01454 { // Set tag size to 0 01455 buff->check_space( sizeof( int ) ); 01456 PACK_INT( buff->buff_ptr, 0 ); 01457 buff->set_stored_size(); 01458 } 01459 01460 return result; 01461 } 01462 01463 ErrorCode ParallelComm::unpack_buffer( unsigned char* buff_ptr, 01464 const bool store_remote_handles, 01465 const int from_proc, 01466 const int ind, 01467 std::vector< std::vector< EntityHandle > >& L1hloc, 01468 std::vector< std::vector< EntityHandle > >& L1hrem, 01469 std::vector< std::vector< int > >& L1p, 01470 std::vector< EntityHandle >& L2hloc, 01471 std::vector< EntityHandle >& L2hrem, 01472 std::vector< unsigned int >& L2p, 01473 std::vector< EntityHandle >& new_ents, 01474 const bool created_iface ) 01475 { 01476 unsigned char* tmp_buff = buff_ptr; 01477 ErrorCode result; 01478 result = unpack_entities( buff_ptr, store_remote_handles, ind, false, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, 01479 new_ents, created_iface );MB_CHK_SET_ERR( result, "Unpacking entities failed" ); 01480 if( myDebug->get_verbosity() == 3 ) 01481 { 01482 myDebug->tprintf( 4, "unpack_entities buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01483 tmp_buff = buff_ptr; 01484 } 01485 result = unpack_sets( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking sets failed" ); 01486 if( myDebug->get_verbosity() == 3 ) 01487 { 01488 myDebug->tprintf( 4, "unpack_sets buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01489 tmp_buff = buff_ptr; 01490 } 01491 result = unpack_tags( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking tags failed" ); 01492 if( myDebug->get_verbosity() == 3 ) 01493 { 01494 myDebug->tprintf( 4, "unpack_tags buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01495 // tmp_buff = buff_ptr; 01496 } 01497 01498 if( myDebug->get_verbosity() == 3 ) myDebug->print( 4, "\n" ); 01499 01500 return MB_SUCCESS; 01501 } 01502 01503 int ParallelComm::estimate_ents_buffer_size( Range& entities, const bool store_remote_handles ) 01504 { 01505 int buff_size = 0; 01506 std::vector< EntityHandle > dum_connect_vec; 01507 const EntityHandle* connect; 01508 int num_connect; 01509 01510 int num_verts = entities.num_of_type( MBVERTEX ); 01511 // # verts + coords + handles 01512 buff_size += 2 * sizeof( int ) + 3 * sizeof( double ) * num_verts; 01513 if( store_remote_handles ) buff_size += sizeof( EntityHandle ) * num_verts; 01514 01515 // Do a rough count by looking at first entity of each type 01516 for( EntityType t = MBEDGE; t < MBENTITYSET; t++ ) 01517 { 01518 const Range::iterator rit = entities.lower_bound( t ); 01519 if( TYPE_FROM_HANDLE( *rit ) != t ) continue; 01520 01521 ErrorCode result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect_vec );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get connectivity to estimate buffer size", -1 ); 01522 01523 // Number, type, nodes per entity 01524 buff_size += 3 * sizeof( int ); 01525 int num_ents = entities.num_of_type( t ); 01526 // Connectivity, handle for each ent 01527 buff_size += ( num_connect + 1 ) * sizeof( EntityHandle ) * num_ents; 01528 } 01529 01530 // Extra entity type at end, passed as int 01531 buff_size += sizeof( int ); 01532 01533 return buff_size; 01534 } 01535 01536 int ParallelComm::estimate_sets_buffer_size( Range& entities, const bool /*store_remote_handles*/ ) 01537 { 01538 // Number of sets 01539 int buff_size = sizeof( int ); 01540 01541 // Do a rough count by looking at first entity of each type 01542 Range::iterator rit = entities.lower_bound( MBENTITYSET ); 01543 ErrorCode result; 01544 01545 for( ; rit != entities.end(); ++rit ) 01546 { 01547 unsigned int options; 01548 result = mbImpl->get_meshset_options( *rit, options );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get meshset options", -1 ); 01549 01550 buff_size += sizeof( int ); 01551 01552 Range set_range; 01553 if( options & MESHSET_SET ) 01554 { 01555 // Range-based set; count the subranges 01556 result = mbImpl->get_entities_by_handle( *rit, set_range );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get set entities", -1 ); 01557 01558 // Set range 01559 buff_size += RANGE_SIZE( set_range ); 01560 } 01561 else if( options & MESHSET_ORDERED ) 01562 { 01563 // Just get the number of entities in the set 01564 int num_ents; 01565 result = mbImpl->get_number_entities_by_handle( *rit, num_ents );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get number entities in ordered set", -1 ); 01566 01567 // Set vec 01568 buff_size += sizeof( EntityHandle ) * num_ents + sizeof( int ); 01569 } 01570 01571 // Get numbers of parents/children 01572 int num_par, num_ch; 01573 result = mbImpl->num_child_meshsets( *rit, &num_ch );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num children", -1 ); 01574 result = mbImpl->num_parent_meshsets( *rit, &num_par );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num parents", -1 ); 01575 01576 buff_size += ( num_ch + num_par ) * sizeof( EntityHandle ) + 2 * sizeof( int ); 01577 } 01578 01579 return buff_size; 01580 } 01581 01582 ErrorCode ParallelComm::pack_entities( Range& entities, 01583 Buffer* buff, 01584 const bool store_remote_handles, 01585 const int to_proc, 01586 const bool /*is_iface*/, 01587 TupleList* entprocs, 01588 Range* /*allsent*/ ) 01589 { 01590 // Packed information: 01591 // 1. # entities = E 01592 // 2. for e in E 01593 // a. # procs sharing e, incl. sender and receiver = P 01594 // b. for p in P (procs sharing e) 01595 // c. for p in P (handle for e on p) (Note1) 01596 // 3. vertex/entity info 01597 01598 // Get an estimate of the buffer size & pre-allocate buffer size 01599 int buff_size = estimate_ents_buffer_size( entities, store_remote_handles ); 01600 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate ents buffer size" ); 01601 buff->check_space( buff_size ); 01602 myDebug->tprintf( 3, "estimate buffer size for %d entities: %d \n", (int)entities.size(), buff_size ); 01603 01604 unsigned int num_ents; 01605 ErrorCode result; 01606 01607 std::vector< EntityHandle > entities_vec( entities.size() ); 01608 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 01609 01610 // First pack procs/handles sharing this ent, not including this dest but including 01611 // others (with zero handles) 01612 if( store_remote_handles ) 01613 { 01614 // Buff space is at least proc + handle for each entity; use avg of 4 other procs 01615 // to estimate buff size, but check later 01616 buff->check_space( sizeof( int ) + ( 5 * sizeof( int ) + sizeof( EntityHandle ) ) * entities.size() ); 01617 01618 // 1. # entities = E 01619 PACK_INT( buff->buff_ptr, entities.size() ); 01620 01621 Range::iterator rit; 01622 01623 // Pre-fetch sharedp and pstatus 01624 std::vector< int > sharedp_vals( entities.size() ); 01625 result = mbImpl->tag_get_data( sharedp_tag(), entities, &sharedp_vals[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 01626 std::vector< char > pstatus_vals( entities.size() ); 01627 result = mbImpl->tag_get_data( pstatus_tag(), entities, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 01628 01629 unsigned int i; 01630 int tmp_procs[MAX_SHARING_PROCS]; 01631 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 01632 std::set< unsigned int > dumprocs; 01633 01634 // 2. for e in E 01635 for( rit = entities.begin(), i = 0; rit != entities.end(); ++rit, i++ ) 01636 { 01637 unsigned int ind = 01638 std::lower_bound( entprocs->vul_rd, entprocs->vul_rd + entprocs->get_n(), *rit ) - entprocs->vul_rd; 01639 assert( ind < entprocs->get_n() ); 01640 01641 while( ind < entprocs->get_n() && entprocs->vul_rd[ind] == *rit ) 01642 dumprocs.insert( entprocs->vi_rd[ind++] ); 01643 01644 result = build_sharedhps_list( *rit, pstatus_vals[i], sharedp_vals[i], dumprocs, num_ents, tmp_procs, 01645 tmp_handles );MB_CHK_SET_ERR( result, "Failed to build sharedhps" ); 01646 01647 dumprocs.clear(); 01648 01649 // Now pack them 01650 buff->check_space( ( num_ents + 1 ) * sizeof( int ) + num_ents * sizeof( EntityHandle ) ); 01651 PACK_INT( buff->buff_ptr, num_ents ); 01652 PACK_INTS( buff->buff_ptr, tmp_procs, num_ents ); 01653 PACK_EH( buff->buff_ptr, tmp_handles, num_ents ); 01654 01655 #ifndef NDEBUG 01656 // Check for duplicates in proc list 01657 unsigned int dp = 0; 01658 for( ; dp < MAX_SHARING_PROCS && -1 != tmp_procs[dp]; dp++ ) 01659 dumprocs.insert( tmp_procs[dp] ); 01660 assert( dumprocs.size() == dp ); 01661 dumprocs.clear(); 01662 #endif 01663 } 01664 } 01665 01666 // Pack vertices 01667 Range these_ents = entities.subset_by_type( MBVERTEX ); 01668 num_ents = these_ents.size(); 01669 01670 if( num_ents ) 01671 { 01672 buff_size = 2 * sizeof( int ) + 3 * num_ents * sizeof( double ); 01673 buff->check_space( buff_size ); 01674 01675 // Type, # ents 01676 PACK_INT( buff->buff_ptr, ( (int)MBVERTEX ) ); 01677 PACK_INT( buff->buff_ptr, ( (int)num_ents ) ); 01678 01679 std::vector< double > tmp_coords( 3 * num_ents ); 01680 result = mbImpl->get_coords( these_ents, &tmp_coords[0] );MB_CHK_SET_ERR( result, "Failed to get vertex coordinates" ); 01681 PACK_DBLS( buff->buff_ptr, &tmp_coords[0], 3 * num_ents ); 01682 01683 myDebug->tprintf( 4, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(), 01684 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01685 } 01686 01687 // Now entities; go through range, packing by type and equal # verts per element 01688 Range::iterator start_rit = entities.find( *these_ents.rbegin() ); 01689 ++start_rit; 01690 int last_nodes = -1; 01691 EntityType last_type = MBMAXTYPE; 01692 these_ents.clear(); 01693 Range::iterator end_rit = start_rit; 01694 EntitySequence* seq; 01695 ElementSequence* eseq; 01696 01697 while( start_rit != entities.end() || !these_ents.empty() ) 01698 { 01699 // Cases: 01700 // A: !end, last_type == MBMAXTYPE, seq: save contig sequence in these_ents 01701 // B: !end, last type & nodes same, seq: save contig sequence in these_ents 01702 // C: !end, last type & nodes different: pack these_ents, then save contig sequence in 01703 // these_ents D: end: pack these_ents 01704 01705 // Find the sequence holding current start entity, if we're not at end 01706 eseq = NULL; 01707 if( start_rit != entities.end() ) 01708 { 01709 result = sequenceManager->find( *start_rit, seq );MB_CHK_SET_ERR( result, "Failed to find entity sequence" ); 01710 if( NULL == seq ) return MB_FAILURE; 01711 eseq = dynamic_cast< ElementSequence* >( seq ); 01712 } 01713 01714 // Pack the last batch if at end or next one is different 01715 if( !these_ents.empty() && 01716 ( !eseq || eseq->type() != last_type || last_nodes != (int)eseq->nodes_per_element() ) ) 01717 { 01718 result = pack_entity_seq( last_nodes, store_remote_handles, to_proc, these_ents, entities_vec, buff );MB_CHK_SET_ERR( result, "Failed to pack entities from a sequence" ); 01719 these_ents.clear(); 01720 } 01721 01722 if( eseq ) 01723 { 01724 // Continuation of current range, just save these entities 01725 // Get position in entities list one past end of this sequence 01726 end_rit = entities.lower_bound( start_rit, entities.end(), eseq->end_handle() + 1 ); 01727 01728 // Put these entities in the range 01729 std::copy( start_rit, end_rit, range_inserter( these_ents ) ); 01730 01731 last_type = eseq->type(); 01732 last_nodes = eseq->nodes_per_element(); 01733 } 01734 else if( start_rit != entities.end() && TYPE_FROM_HANDLE( *start_rit ) == MBENTITYSET ) 01735 break; 01736 01737 start_rit = end_rit; 01738 } 01739 01740 // Pack MBMAXTYPE to indicate end of ranges 01741 buff->check_space( sizeof( int ) ); 01742 PACK_INT( buff->buff_ptr, ( (int)MBMAXTYPE ) ); 01743 01744 buff->set_stored_size(); 01745 return MB_SUCCESS; 01746 } 01747 01748 ErrorCode ParallelComm::build_sharedhps_list( const EntityHandle entity, 01749 const unsigned char pstatus, 01750 const int 01751 #ifndef NDEBUG 01752 sharedp 01753 #endif 01754 , 01755 const std::set< unsigned int >& procs, 01756 unsigned int& num_ents, 01757 int* tmp_procs, 01758 EntityHandle* tmp_handles ) 01759 { 01760 num_ents = 0; 01761 unsigned char pstat; 01762 ErrorCode result = get_sharing_data( entity, tmp_procs, tmp_handles, pstat, num_ents );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 01763 assert( pstat == pstatus ); 01764 01765 // Build shared proc/handle lists 01766 // Start with multi-shared, since if it is the owner will be first 01767 if( pstatus & PSTATUS_MULTISHARED ) 01768 { 01769 } 01770 else if( pstatus & PSTATUS_NOT_OWNED ) 01771 { 01772 // If not multishared and not owned, other sharing proc is owner, put that 01773 // one first 01774 assert( "If not owned, I should be shared too" && pstatus & PSTATUS_SHARED && 1 == num_ents ); 01775 tmp_procs[1] = procConfig.proc_rank(); 01776 tmp_handles[1] = entity; 01777 num_ents = 2; 01778 } 01779 else if( pstatus & PSTATUS_SHARED ) 01780 { 01781 // If not multishared and owned, I'm owner 01782 assert( "shared and owned, should be only 1 sharing proc" && 1 == num_ents ); 01783 tmp_procs[1] = tmp_procs[0]; 01784 tmp_procs[0] = procConfig.proc_rank(); 01785 tmp_handles[1] = tmp_handles[0]; 01786 tmp_handles[0] = entity; 01787 num_ents = 2; 01788 } 01789 else 01790 { 01791 // Not shared yet, just add owner (me) 01792 tmp_procs[0] = procConfig.proc_rank(); 01793 tmp_handles[0] = entity; 01794 num_ents = 1; 01795 } 01796 01797 #ifndef NDEBUG 01798 int tmp_ps = num_ents; 01799 #endif 01800 01801 // Now add others, with zero handle for now 01802 for( std::set< unsigned int >::iterator sit = procs.begin(); sit != procs.end(); ++sit ) 01803 { 01804 #ifndef NDEBUG 01805 if( tmp_ps && std::find( tmp_procs, tmp_procs + tmp_ps, *sit ) != tmp_procs + tmp_ps ) 01806 { 01807 std::cerr << "Trouble with something already in shared list on proc " << procConfig.proc_rank() 01808 << ". Entity:" << std::endl; 01809 list_entities( &entity, 1 ); 01810 std::cerr << "pstatus = " << (int)pstatus << ", sharedp = " << sharedp << std::endl; 01811 std::cerr << "tmp_ps = "; 01812 for( int i = 0; i < tmp_ps; i++ ) 01813 std::cerr << tmp_procs[i] << " "; 01814 std::cerr << std::endl; 01815 std::cerr << "procs = "; 01816 for( std::set< unsigned int >::iterator sit2 = procs.begin(); sit2 != procs.end(); ++sit2 ) 01817 std::cerr << *sit2 << " "; 01818 assert( false ); 01819 } 01820 #endif 01821 tmp_procs[num_ents] = *sit; 01822 tmp_handles[num_ents] = 0; 01823 num_ents++; 01824 } 01825 01826 // Put -1 after procs and 0 after handles 01827 if( MAX_SHARING_PROCS > num_ents ) 01828 { 01829 tmp_procs[num_ents] = -1; 01830 tmp_handles[num_ents] = 0; 01831 } 01832 01833 return MB_SUCCESS; 01834 } 01835 01836 ErrorCode ParallelComm::pack_entity_seq( const int nodes_per_entity, 01837 const bool store_remote_handles, 01838 const int to_proc, 01839 Range& these_ents, 01840 std::vector< EntityHandle >& entities_vec, 01841 Buffer* buff ) 01842 { 01843 int tmp_space = 3 * sizeof( int ) + nodes_per_entity * these_ents.size() * sizeof( EntityHandle ); 01844 buff->check_space( tmp_space ); 01845 01846 // Pack the entity type 01847 PACK_INT( buff->buff_ptr, ( (int)TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01848 01849 // Pack # ents 01850 PACK_INT( buff->buff_ptr, these_ents.size() ); 01851 01852 // Pack the nodes per entity 01853 PACK_INT( buff->buff_ptr, nodes_per_entity ); 01854 myDebug->tprintf( 3, "after some pack int %d \n", buff->get_current_size() ); 01855 01856 // Pack the connectivity 01857 std::vector< EntityHandle > connect; 01858 ErrorCode result = MB_SUCCESS; 01859 for( Range::const_iterator rit = these_ents.begin(); rit != these_ents.end(); ++rit ) 01860 { 01861 connect.clear(); 01862 result = mbImpl->get_connectivity( &( *rit ), 1, connect, false );MB_CHK_SET_ERR( result, "Failed to get connectivity" ); 01863 assert( (int)connect.size() == nodes_per_entity ); 01864 result = 01865 get_remote_handles( store_remote_handles, &connect[0], &connect[0], connect.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" ); 01866 PACK_EH( buff->buff_ptr, &connect[0], connect.size() ); 01867 } 01868 01869 myDebug->tprintf( 3, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(), 01870 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01871 01872 return result; 01873 } 01874 01875 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 01876 EntityHandle* from_vec, 01877 EntityHandle* to_vec_tmp, 01878 int num_ents, 01879 int to_proc, 01880 const std::vector< EntityHandle >& new_ents ) 01881 { 01882 // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE RANGE-BASED VERSION, NO REUSE 01883 // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE 01884 // OTHER VERSION TOO!!! 01885 if( 0 == num_ents ) return MB_SUCCESS; 01886 01887 // Use a local destination ptr in case we're doing an in-place copy 01888 std::vector< EntityHandle > tmp_vector; 01889 EntityHandle* to_vec = to_vec_tmp; 01890 if( to_vec == from_vec ) 01891 { 01892 tmp_vector.resize( num_ents ); 01893 to_vec = &tmp_vector[0]; 01894 } 01895 01896 if( !store_remote_handles ) 01897 { 01898 int err; 01899 // In this case, substitute position in new_ents list 01900 for( int i = 0; i < num_ents; i++ ) 01901 { 01902 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin(); 01903 assert( new_ents[ind] == from_vec[i] ); 01904 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err ); 01905 assert( to_vec[i] != 0 && !err && -1 != ind ); 01906 } 01907 } 01908 else 01909 { 01910 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 01911 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 01912 01913 // Get single-proc destination handles and shared procs 01914 std::vector< int > sharing_procs( num_ents ); 01915 result = mbImpl->tag_get_data( shh_tag, from_vec, num_ents, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" ); 01916 result = mbImpl->tag_get_data( shp_tag, from_vec, num_ents, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" ); 01917 for( int j = 0; j < num_ents; j++ ) 01918 { 01919 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0; 01920 } 01921 01922 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 01923 int tmp_procs[MAX_SHARING_PROCS]; 01924 int i; 01925 // Go through results, and for 0-valued ones, look for multiple shared proc 01926 for( i = 0; i < num_ents; i++ ) 01927 { 01928 if( !to_vec[i] ) 01929 { 01930 result = mbImpl->tag_get_data( shps_tag, from_vec + i, 1, tmp_procs ); 01931 if( MB_SUCCESS == result ) 01932 { 01933 for( int j = 0; j < MAX_SHARING_PROCS; j++ ) 01934 { 01935 if( -1 == tmp_procs[j] ) 01936 break; 01937 else if( tmp_procs[j] == to_proc ) 01938 { 01939 result = mbImpl->tag_get_data( shhs_tag, from_vec + i, 1, tmp_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 01940 to_vec[i] = tmp_handles[j]; 01941 assert( to_vec[i] ); 01942 break; 01943 } 01944 } 01945 } 01946 if( !to_vec[i] ) 01947 { 01948 int j = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin(); 01949 if( (int)new_ents.size() == j ) 01950 { 01951 std::cout << "Failed to find new entity in send list, proc " << procConfig.proc_rank() 01952 << std::endl; 01953 for( int k = 0; k <= num_ents; k++ ) 01954 std::cout << k << ": " << from_vec[k] << " " << to_vec[k] << std::endl; 01955 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" ); 01956 } 01957 int err; 01958 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err ); 01959 if( err ) 01960 { 01961 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" ); 01962 } 01963 } 01964 } 01965 } 01966 } 01967 01968 // memcpy over results if from_vec and to_vec are the same 01969 if( to_vec_tmp == from_vec ) memcpy( from_vec, to_vec, num_ents * sizeof( EntityHandle ) ); 01970 01971 return MB_SUCCESS; 01972 } 01973 01974 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 01975 const Range& from_range, 01976 EntityHandle* to_vec, 01977 int to_proc, 01978 const std::vector< EntityHandle >& new_ents ) 01979 { 01980 // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE VECTOR-BASED VERSION, NO REUSE 01981 // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE 01982 // OTHER VERSION TOO!!! 01983 if( from_range.empty() ) return MB_SUCCESS; 01984 01985 if( !store_remote_handles ) 01986 { 01987 int err; 01988 // In this case, substitute position in new_ents list 01989 Range::iterator rit; 01990 unsigned int i; 01991 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ ) 01992 { 01993 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin(); 01994 assert( new_ents[ind] == *rit ); 01995 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err ); 01996 assert( to_vec[i] != 0 && !err && -1 != ind ); 01997 } 01998 } 01999 else 02000 { 02001 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 02002 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 02003 02004 // Get single-proc destination handles and shared procs 02005 std::vector< int > sharing_procs( from_range.size() ); 02006 result = mbImpl->tag_get_data( shh_tag, from_range, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" ); 02007 result = mbImpl->tag_get_data( shp_tag, from_range, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" ); 02008 for( unsigned int j = 0; j < from_range.size(); j++ ) 02009 { 02010 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0; 02011 } 02012 02013 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 02014 int tmp_procs[MAX_SHARING_PROCS]; 02015 // Go through results, and for 0-valued ones, look for multiple shared proc 02016 Range::iterator rit; 02017 unsigned int i; 02018 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ ) 02019 { 02020 if( !to_vec[i] ) 02021 { 02022 result = mbImpl->tag_get_data( shhs_tag, &( *rit ), 1, tmp_handles ); 02023 if( MB_SUCCESS == result ) 02024 { 02025 result = mbImpl->tag_get_data( shps_tag, &( *rit ), 1, tmp_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" ); 02026 for( int j = 0; j < MAX_SHARING_PROCS; j++ ) 02027 if( tmp_procs[j] == to_proc ) 02028 { 02029 to_vec[i] = tmp_handles[j]; 02030 break; 02031 } 02032 } 02033 02034 if( !to_vec[i] ) 02035 { 02036 int j = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin(); 02037 if( (int)new_ents.size() == j ) 02038 { 02039 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" ); 02040 } 02041 int err; 02042 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err ); 02043 if( err ) 02044 { 02045 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" ); 02046 } 02047 } 02048 } 02049 } 02050 } 02051 02052 return MB_SUCCESS; 02053 } 02054 02055 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 02056 const Range& from_range, 02057 Range& to_range, 02058 int to_proc, 02059 const std::vector< EntityHandle >& new_ents ) 02060 { 02061 std::vector< EntityHandle > to_vector( from_range.size() ); 02062 02063 ErrorCode result = get_remote_handles( store_remote_handles, from_range, &to_vector[0], to_proc, new_ents );MB_CHK_SET_ERR( result, "Failed to get remote handles" ); 02064 std::copy( to_vector.begin(), to_vector.end(), range_inserter( to_range ) ); 02065 return result; 02066 } 02067 02068 ErrorCode ParallelComm::unpack_entities( unsigned char*& buff_ptr, 02069 const bool store_remote_handles, 02070 const int /*from_ind*/, 02071 const bool is_iface, 02072 std::vector< std::vector< EntityHandle > >& L1hloc, 02073 std::vector< std::vector< EntityHandle > >& L1hrem, 02074 std::vector< std::vector< int > >& L1p, 02075 std::vector< EntityHandle >& L2hloc, 02076 std::vector< EntityHandle >& L2hrem, 02077 std::vector< unsigned int >& L2p, 02078 std::vector< EntityHandle >& new_ents, 02079 const bool created_iface ) 02080 { 02081 // General algorithm: 02082 // - unpack # entities 02083 // - save start of remote handle info, then scan forward to entity definition data 02084 // - for all vertices or entities w/ same # verts: 02085 // . get entity type, num ents, and (if !vert) # verts 02086 // . for each ent: 02087 // o get # procs/handles in remote handle info 02088 // o if # procs/handles > 2, check for already-created entity: 02089 // x get index of owner proc (1st in proc list), resize L1 list if nec 02090 // x look for already-arrived entity in L2 by owner handle 02091 // o if no existing entity: 02092 // x if iface, look for existing entity with same connect & type 02093 // x if none found, create vertex or element 02094 // x if !iface & multi-shared, save on L2 02095 // x if !iface, put new entity on new_ents list 02096 // o update proc/handle, pstatus tags, adjusting to put owner first if iface 02097 // o if !iface, save new handle on L1 for all sharing procs 02098 02099 // Lists of handles/procs to return to sending/other procs 02100 // L1hloc[p], L1hrem[p]: handle pairs [h, h'], where h is the local proc handle 02101 // and h' is either the remote proc handle (if that is known) or 02102 // the owner proc handle (otherwise); 02103 // L1p[p]: indicates whether h is remote handle (= -1) or owner (rank of owner) 02104 // L2hloc, L2hrem: local/remote handles for entities shared by > 2 procs; 02105 // remote handles are on owning proc 02106 // L2p: owning procs for handles in L2hrem 02107 02108 ErrorCode result; 02109 bool done = false; 02110 ReadUtilIface* ru = NULL; 02111 02112 result = mbImpl->query_interface( ru );MB_CHK_SET_ERR( result, "Failed to get ReadUtilIface" ); 02113 02114 // 1. # entities = E 02115 int num_ents = 0; 02116 unsigned char* buff_save = buff_ptr; 02117 int i, j; 02118 02119 if( store_remote_handles ) 02120 { 02121 UNPACK_INT( buff_ptr, num_ents ); 02122 02123 buff_save = buff_ptr; 02124 02125 // Save place where remote handle info starts, then scan forward to ents 02126 for( i = 0; i < num_ents; i++ ) 02127 { 02128 UNPACK_INT( buff_ptr, j ); 02129 if( j < 0 ) 02130 { 02131 std::cout << "Should be non-negative # proc/handles."; 02132 return MB_FAILURE; 02133 } 02134 02135 buff_ptr += j * ( sizeof( int ) + sizeof( EntityHandle ) ); 02136 } 02137 } 02138 02139 std::vector< EntityHandle > msg_ents; 02140 02141 while( !done ) 02142 { 02143 EntityType this_type = MBMAXTYPE; 02144 UNPACK_TYPE( buff_ptr, this_type ); 02145 assert( this_type != MBENTITYSET ); 02146 02147 // MBMAXTYPE signifies end of entities data 02148 if( MBMAXTYPE == this_type ) break; 02149 02150 // Get the number of ents 02151 int num_ents2, verts_per_entity = 0; 02152 UNPACK_INT( buff_ptr, num_ents2 ); 02153 02154 // Unpack the nodes per entity 02155 if( MBVERTEX != this_type && num_ents2 ) 02156 { 02157 UNPACK_INT( buff_ptr, verts_per_entity ); 02158 } 02159 02160 std::vector< int > ps( MAX_SHARING_PROCS, -1 ); 02161 std::vector< EntityHandle > hs( MAX_SHARING_PROCS, 0 ); 02162 for( int e = 0; e < num_ents2; e++ ) 02163 { 02164 // Check for existing entity, otherwise make new one 02165 EntityHandle new_h = 0; 02166 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT]; 02167 double coords[3]; 02168 int num_ps = -1; 02169 02170 //======================================= 02171 // Unpack all the data at once, to make sure the buffer pointers 02172 // are tracked correctly 02173 //======================================= 02174 if( store_remote_handles ) 02175 { 02176 // Pointers to other procs/handles 02177 UNPACK_INT( buff_save, num_ps ); 02178 if( 0 >= num_ps ) 02179 { 02180 std::cout << "Shouldn't ever be fewer than 1 procs here." << std::endl; 02181 return MB_FAILURE; 02182 } 02183 02184 UNPACK_INTS( buff_save, &ps[0], num_ps ); 02185 UNPACK_EH( buff_save, &hs[0], num_ps ); 02186 } 02187 02188 if( MBVERTEX == this_type ) 02189 { 02190 UNPACK_DBLS( buff_ptr, coords, 3 ); 02191 } 02192 else 02193 { 02194 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT ); 02195 UNPACK_EH( buff_ptr, connect, verts_per_entity ); 02196 02197 // Update connectivity to local handles 02198 result = get_local_handles( connect, verts_per_entity, msg_ents );MB_CHK_SET_ERR( result, "Failed to get local handles" ); 02199 } 02200 02201 //======================================= 02202 // Now, process that data; begin by finding an identical 02203 // entity, if there is one 02204 //======================================= 02205 if( store_remote_handles ) 02206 { 02207 result = find_existing_entity( is_iface, ps[0], hs[0], num_ps, connect, verts_per_entity, this_type, 02208 L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Failed to get existing entity" ); 02209 } 02210 02211 //======================================= 02212 // If we didn't find one, we'll have to create one 02213 //======================================= 02214 bool created_here = false; 02215 if( !new_h && !is_iface ) 02216 { 02217 if( MBVERTEX == this_type ) 02218 { 02219 // Create a vertex 02220 result = mbImpl->create_vertex( coords, new_h );MB_CHK_SET_ERR( result, "Failed to make new vertex" ); 02221 } 02222 else 02223 { 02224 // Create the element 02225 result = mbImpl->create_element( this_type, connect, verts_per_entity, new_h );MB_CHK_SET_ERR( result, "Failed to make new element" ); 02226 02227 // Update adjacencies 02228 result = ru->update_adjacencies( new_h, 1, verts_per_entity, connect );MB_CHK_SET_ERR( result, "Failed to update adjacencies" ); 02229 } 02230 02231 // Should have a new handle now 02232 assert( new_h ); 02233 02234 created_here = true; 02235 } 02236 02237 //======================================= 02238 // Take care of sharing data 02239 //======================================= 02240 02241 // Need to save entities found in order, for interpretation of 02242 // later parts of this message 02243 if( !is_iface ) 02244 { 02245 assert( new_h ); 02246 msg_ents.push_back( new_h ); 02247 } 02248 02249 if( created_here ) new_ents.push_back( new_h ); 02250 02251 if( new_h && store_remote_handles ) 02252 { 02253 unsigned char new_pstat = 0x0; 02254 if( is_iface ) 02255 { 02256 new_pstat = PSTATUS_INTERFACE; 02257 // Here, lowest rank proc should be first 02258 int idx = std::min_element( &ps[0], &ps[0] + num_ps ) - &ps[0]; 02259 if( idx ) 02260 { 02261 std::swap( ps[0], ps[idx] ); 02262 std::swap( hs[0], hs[idx] ); 02263 } 02264 // Set ownership based on lowest rank; can't be in update_remote_data, because 02265 // there we don't know whether it resulted from ghosting or not 02266 if( ( num_ps > 1 && ps[0] != (int)rank() ) ) new_pstat |= PSTATUS_NOT_OWNED; 02267 } 02268 else if( created_here ) 02269 { 02270 if( created_iface ) 02271 new_pstat = PSTATUS_NOT_OWNED; 02272 else 02273 new_pstat = PSTATUS_GHOST | PSTATUS_NOT_OWNED; 02274 } 02275 02276 // Update sharing data and pstatus, adjusting order if iface 02277 result = update_remote_data( new_h, &ps[0], &hs[0], num_ps, new_pstat );MB_CHK_SET_ERR( result, "unpack_entities" ); 02278 02279 // If a new multi-shared entity, save owner for subsequent lookup in L2 lists 02280 if( store_remote_handles && !is_iface && num_ps > 2 ) 02281 { 02282 L2hrem.push_back( hs[0] ); 02283 L2hloc.push_back( new_h ); 02284 L2p.push_back( ps[0] ); 02285 } 02286 02287 // Need to send this new handle to all sharing procs 02288 if( !is_iface ) 02289 { 02290 for( j = 0; j < num_ps; j++ ) 02291 { 02292 if( ps[j] == (int)procConfig.proc_rank() ) continue; 02293 int idx = get_buffers( ps[j] ); 02294 if( idx == (int)L1hloc.size() ) 02295 { 02296 L1hloc.resize( idx + 1 ); 02297 L1hrem.resize( idx + 1 ); 02298 L1p.resize( idx + 1 ); 02299 } 02300 02301 // Don't bother adding if it's already in the list 02302 std::vector< EntityHandle >::iterator vit = 02303 std::find( L1hloc[idx].begin(), L1hloc[idx].end(), new_h ); 02304 if( vit != L1hloc[idx].end() ) 02305 { 02306 // If it's in the list but remote handle isn't known but we know 02307 // it, replace in the list 02308 if( L1p[idx][vit - L1hloc[idx].begin()] != -1 && hs[j] ) 02309 { 02310 L1hrem[idx][vit - L1hloc[idx].begin()] = hs[j]; 02311 L1p[idx][vit - L1hloc[idx].begin()] = -1; 02312 } 02313 else 02314 continue; 02315 } 02316 else 02317 { 02318 if( !hs[j] ) 02319 { 02320 assert( -1 != ps[0] && num_ps > 2 ); 02321 L1p[idx].push_back( ps[0] ); 02322 L1hrem[idx].push_back( hs[0] ); 02323 } 02324 else 02325 { 02326 assert( 02327 "either this remote handle isn't in the remote list, or " 02328 "it's for another proc" && 02329 ( std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) == L1hrem[idx].end() || 02330 L1p[idx][std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) - 02331 L1hrem[idx].begin()] != -1 ) ); 02332 L1p[idx].push_back( -1 ); 02333 L1hrem[idx].push_back( hs[j] ); 02334 } 02335 L1hloc[idx].push_back( new_h ); 02336 } 02337 } 02338 } 02339 02340 assert( "Shouldn't be here for non-shared entities" && -1 != num_ps ); 02341 std::fill( &ps[0], &ps[num_ps], -1 ); 02342 std::fill( &hs[0], &hs[num_ps], 0 ); 02343 } 02344 } 02345 02346 myDebug->tprintf( 4, "Unpacked %d ents of type %s", num_ents2, CN::EntityTypeName( this_type ) ); 02347 } 02348 02349 myDebug->tprintf( 4, "Done unpacking entities.\n" ); 02350 02351 // Need to sort here, to enable searching 02352 std::sort( new_ents.begin(), new_ents.end() ); 02353 02354 return MB_SUCCESS; 02355 } 02356 02357 ErrorCode ParallelComm::print_buffer( unsigned char* buff_ptr, int mesg_tag, int from_proc, bool sent ) 02358 { 02359 std::cerr << procConfig.proc_rank(); 02360 if( sent ) 02361 std::cerr << " sent"; 02362 else 02363 std::cerr << " received"; 02364 std::cerr << " message type " << mesg_tag << " to/from proc " << from_proc << "; contents:" << std::endl; 02365 02366 int msg_length, num_ents; 02367 unsigned char* orig_ptr = buff_ptr; 02368 UNPACK_INT( buff_ptr, msg_length ); 02369 std::cerr << msg_length << " bytes..." << std::endl; 02370 02371 if( MB_MESG_ENTS_SIZE == mesg_tag || MB_MESG_ENTS_LARGE == mesg_tag ) 02372 { 02373 // 1. # entities = E 02374 int i, j, k; 02375 std::vector< int > ps; 02376 std::vector< EntityHandle > hs; 02377 02378 UNPACK_INT( buff_ptr, num_ents ); 02379 std::cerr << num_ents << " entities..." << std::endl; 02380 02381 // Save place where remote handle info starts, then scan forward to ents 02382 for( i = 0; i < num_ents; i++ ) 02383 { 02384 UNPACK_INT( buff_ptr, j ); 02385 if( 0 > j ) return MB_FAILURE; 02386 ps.resize( j ); 02387 hs.resize( j ); 02388 std::cerr << "Entity " << i << ", # procs = " << j << std::endl; 02389 UNPACK_INTS( buff_ptr, &ps[0], j ); 02390 UNPACK_EH( buff_ptr, &hs[0], j ); 02391 std::cerr << " Procs: "; 02392 for( k = 0; k < j; k++ ) 02393 std::cerr << ps[k] << " "; 02394 std::cerr << std::endl; 02395 std::cerr << " Handles: "; 02396 for( k = 0; k < j; k++ ) 02397 std::cerr << hs[k] << " "; 02398 std::cerr << std::endl; 02399 02400 if( buff_ptr - orig_ptr > msg_length ) 02401 { 02402 std::cerr << "End of buffer..." << std::endl; 02403 std::cerr.flush(); 02404 return MB_FAILURE; 02405 } 02406 } 02407 02408 while( true ) 02409 { 02410 EntityType this_type = MBMAXTYPE; 02411 UNPACK_TYPE( buff_ptr, this_type ); 02412 assert( this_type != MBENTITYSET ); 02413 02414 // MBMAXTYPE signifies end of entities data 02415 if( MBMAXTYPE == this_type ) break; 02416 02417 // Get the number of ents 02418 int num_ents2, verts_per_entity = 0; 02419 UNPACK_INT( buff_ptr, num_ents2 ); 02420 02421 // Unpack the nodes per entity 02422 if( MBVERTEX != this_type && num_ents2 ) 02423 { 02424 UNPACK_INT( buff_ptr, verts_per_entity ); 02425 } 02426 02427 std::cerr << "Type: " << CN::EntityTypeName( this_type ) << "; num_ents = " << num_ents2; 02428 if( MBVERTEX != this_type ) std::cerr << "; verts_per_ent = " << verts_per_entity; 02429 std::cerr << std::endl; 02430 if( num_ents2 < 0 || num_ents2 > msg_length ) 02431 { 02432 std::cerr << "Wrong number of entities, returning." << std::endl; 02433 return MB_FAILURE; 02434 } 02435 02436 for( int e = 0; e < num_ents2; e++ ) 02437 { 02438 // Check for existing entity, otherwise make new one 02439 if( MBVERTEX == this_type ) 02440 { 02441 double coords[3]; 02442 UNPACK_DBLS( buff_ptr, coords, 3 ); 02443 std::cerr << "xyz = " << coords[0] << ", " << coords[1] << ", " << coords[2] << std::endl; 02444 } 02445 else 02446 { 02447 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT]; 02448 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT ); 02449 UNPACK_EH( buff_ptr, connect, verts_per_entity ); 02450 02451 // Update connectivity to local handles 02452 std::cerr << "Connectivity: "; 02453 for( k = 0; k < verts_per_entity; k++ ) 02454 std::cerr << connect[k] << " "; 02455 std::cerr << std::endl; 02456 } 02457 02458 if( buff_ptr - orig_ptr > msg_length ) 02459 { 02460 std::cerr << "End of buffer..." << std::endl; 02461 std::cerr.flush(); 02462 return MB_FAILURE; 02463 } 02464 } 02465 } 02466 } 02467 else if( MB_MESG_REMOTEH_SIZE == mesg_tag || MB_MESG_REMOTEH_LARGE == mesg_tag ) 02468 { 02469 UNPACK_INT( buff_ptr, num_ents ); 02470 std::cerr << num_ents << " entities..." << std::endl; 02471 if( 0 > num_ents || num_ents > msg_length ) 02472 { 02473 std::cerr << "Wrong number of entities, returning." << std::endl; 02474 return MB_FAILURE; 02475 } 02476 std::vector< EntityHandle > L1hloc( num_ents ), L1hrem( num_ents ); 02477 std::vector< int > L1p( num_ents ); 02478 UNPACK_INTS( buff_ptr, &L1p[0], num_ents ); 02479 UNPACK_EH( buff_ptr, &L1hrem[0], num_ents ); 02480 UNPACK_EH( buff_ptr, &L1hloc[0], num_ents ); 02481 std::cerr << num_ents << " Entity pairs; hremote/hlocal/proc: " << std::endl; 02482 for( int i = 0; i < num_ents; i++ ) 02483 { 02484 EntityType etype = TYPE_FROM_HANDLE( L1hloc[i] ); 02485 std::cerr << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hrem[i] ) << ", " 02486 << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hloc[i] ) << ", " << L1p[i] << std::endl; 02487 } 02488 02489 if( buff_ptr - orig_ptr > msg_length ) 02490 { 02491 std::cerr << "End of buffer..." << std::endl; 02492 std::cerr.flush(); 02493 return MB_FAILURE; 02494 } 02495 } 02496 else if( mesg_tag == MB_MESG_TAGS_SIZE || mesg_tag == MB_MESG_TAGS_LARGE ) 02497 { 02498 int num_tags, dum1, data_type, tag_size; 02499 UNPACK_INT( buff_ptr, num_tags ); 02500 std::cerr << "Number of tags = " << num_tags << std::endl; 02501 for( int i = 0; i < num_tags; i++ ) 02502 { 02503 std::cerr << "Tag " << i << ":" << std::endl; 02504 UNPACK_INT( buff_ptr, tag_size ); 02505 UNPACK_INT( buff_ptr, dum1 ); 02506 UNPACK_INT( buff_ptr, data_type ); 02507 std::cerr << "Tag size, type, data type = " << tag_size << ", " << dum1 << ", " << data_type << std::endl; 02508 UNPACK_INT( buff_ptr, dum1 ); 02509 std::cerr << "Default value size = " << dum1 << std::endl; 02510 buff_ptr += dum1; 02511 UNPACK_INT( buff_ptr, dum1 ); 02512 std::string name( (char*)buff_ptr, dum1 ); 02513 std::cerr << "Tag name = " << name.c_str() << std::endl; 02514 buff_ptr += dum1; 02515 UNPACK_INT( buff_ptr, num_ents ); 02516 std::cerr << "Number of ents = " << num_ents << std::endl; 02517 std::vector< EntityHandle > tmp_buff( num_ents ); 02518 UNPACK_EH( buff_ptr, &tmp_buff[0], num_ents ); 02519 int tot_length = 0; 02520 for( int j = 0; j < num_ents; j++ ) 02521 { 02522 EntityType etype = TYPE_FROM_HANDLE( tmp_buff[j] ); 02523 std::cerr << CN::EntityTypeName( etype ) << " " << ID_FROM_HANDLE( tmp_buff[j] ) << ", tag = "; 02524 if( tag_size == MB_VARIABLE_LENGTH ) 02525 { 02526 UNPACK_INT( buff_ptr, dum1 ); 02527 tot_length += dum1; 02528 std::cerr << "(variable, length = " << dum1 << ")" << std::endl; 02529 } 02530 else if( data_type == MB_TYPE_DOUBLE ) 02531 { 02532 double dum_dbl; 02533 UNPACK_DBL( buff_ptr, dum_dbl ); 02534 std::cerr << dum_dbl << std::endl; 02535 } 02536 else if( data_type == MB_TYPE_INTEGER ) 02537 { 02538 int dum_int; 02539 UNPACK_INT( buff_ptr, dum_int ); 02540 std::cerr << dum_int << std::endl; 02541 } 02542 else if( data_type == MB_TYPE_OPAQUE ) 02543 { 02544 std::cerr << "(opaque)" << std::endl; 02545 buff_ptr += tag_size; 02546 } 02547 else if( data_type == MB_TYPE_HANDLE ) 02548 { 02549 EntityHandle dum_eh; 02550 UNPACK_EH( buff_ptr, &dum_eh, 1 ); 02551 std::cerr << dum_eh << std::endl; 02552 } 02553 else if( data_type == MB_TYPE_BIT ) 02554 { 02555 std::cerr << "(bit)" << std::endl; 02556 buff_ptr += tag_size; 02557 } 02558 } 02559 if( tag_size == MB_VARIABLE_LENGTH ) buff_ptr += tot_length; 02560 } 02561 } 02562 else 02563 { 02564 assert( false ); 02565 return MB_FAILURE; 02566 } 02567 02568 std::cerr.flush(); 02569 02570 return MB_SUCCESS; 02571 } 02572 02573 ErrorCode ParallelComm::list_entities( const EntityHandle* ents, int num_ents ) 02574 { 02575 if( NULL == ents ) 02576 { 02577 Range shared_ents; 02578 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) ); 02579 shared_ents.print( "Shared entities:\n" ); 02580 return MB_SUCCESS; 02581 } 02582 02583 unsigned char pstat; 02584 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 02585 int tmp_procs[MAX_SHARING_PROCS]; 02586 unsigned int num_ps; 02587 ErrorCode result; 02588 02589 for( int i = 0; i < num_ents; i++ ) 02590 { 02591 result = mbImpl->list_entities( ents + i, 1 );MB_CHK_ERR( result ); 02592 double coords[3]; 02593 result = mbImpl->get_coords( ents + i, 1, coords ); 02594 std::cout << " coords: " << coords[0] << " " << coords[1] << " " << coords[2] << "\n"; 02595 02596 result = get_sharing_data( ents[i], tmp_procs, tmp_handles, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 02597 02598 std::cout << "Pstatus: "; 02599 if( !num_ps ) 02600 std::cout << "local " << std::endl; 02601 else 02602 { 02603 if( pstat & PSTATUS_NOT_OWNED ) std::cout << "NOT_OWNED; "; 02604 if( pstat & PSTATUS_SHARED ) std::cout << "SHARED; "; 02605 if( pstat & PSTATUS_MULTISHARED ) std::cout << "MULTISHARED; "; 02606 if( pstat & PSTATUS_INTERFACE ) std::cout << "INTERFACE; "; 02607 if( pstat & PSTATUS_GHOST ) std::cout << "GHOST; "; 02608 std::cout << std::endl; 02609 for( unsigned int j = 0; j < num_ps; j++ ) 02610 { 02611 std::cout << " proc " << tmp_procs[j] << " id (handle) " << mbImpl->id_from_handle( tmp_handles[j] ) 02612 << "(" << tmp_handles[j] << ")" << std::endl; 02613 } 02614 } 02615 std::cout << std::endl; 02616 } 02617 02618 return MB_SUCCESS; 02619 } 02620 02621 ErrorCode ParallelComm::list_entities( const Range& ents ) 02622 { 02623 for( Range::iterator rit = ents.begin(); rit != ents.end(); ++rit ) 02624 list_entities( &( *rit ), 1 ); 02625 02626 return MB_SUCCESS; 02627 } 02628 02629 ErrorCode ParallelComm::update_remote_data( Range& local_range, 02630 Range& remote_range, 02631 int other_proc, 02632 const unsigned char add_pstat ) 02633 { 02634 Range::iterator rit, rit2; 02635 ErrorCode result = MB_SUCCESS; 02636 02637 // For each pair of local/remote handles: 02638 for( rit = local_range.begin(), rit2 = remote_range.begin(); rit != local_range.end(); ++rit, ++rit2 ) 02639 { 02640 result = update_remote_data( *rit, &other_proc, &( *rit2 ), 1, add_pstat );MB_CHK_ERR( result ); 02641 } 02642 02643 return MB_SUCCESS; 02644 } 02645 02646 ErrorCode ParallelComm::update_remote_data( const EntityHandle new_h, 02647 const int* ps, 02648 const EntityHandle* hs, 02649 const int num_ps, 02650 const unsigned char add_pstat 02651 // The following lines left in for future debugging, at least until I trust 02652 // this function; tjt, 10/4/2013 02653 // , int *new_ps, 02654 // EntityHandle *new_hs, 02655 // int &new_numps, 02656 // unsigned char &new_pstat 02657 ) 02658 { 02659 // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0 02660 // in this function, so no need to initialize; sharing data does not include 02661 // this proc if shared with only one other 02662 02663 // Following variables declared here to avoid compiler errors 02664 int new_numps; 02665 unsigned char new_pstat; 02666 std::vector< int > new_ps( MAX_SHARING_PROCS, -1 ); 02667 std::vector< EntityHandle > new_hs( MAX_SHARING_PROCS, 0 ); 02668 02669 new_numps = 0; 02670 ErrorCode result = get_sharing_data( new_h, &new_ps[0], &new_hs[0], new_pstat, new_numps );MB_CHK_SET_ERR( result, "Failed to get sharing data in update_remote_data" ); 02671 int num_exist = new_numps; 02672 02673 // Add new pstat info to the flag 02674 new_pstat |= add_pstat; 02675 02676 /* 02677 #define plist(str, lst, siz) \ 02678 std::cout << str << "("; \ 02679 for (int i = 0; i < (int)siz; i++) std::cout << lst[i] << " "; \ 02680 std::cout << ") "; \ 02681 02682 std::cout << "update_remote_data: rank = " << rank() << ", new_h = " << new_h << std::endl; 02683 std::string ostr; 02684 plist("ps", ps, num_ps); 02685 plist("hs", hs, num_ps); 02686 print_pstatus(add_pstat, ostr); 02687 std::cout << ", add_pstat = " << ostr.c_str() << std::endl; 02688 plist("tag_ps", new_ps, new_numps); 02689 plist("tag_hs", new_hs, new_numps); 02690 assert(new_numps <= size()); 02691 print_pstatus(new_pstat, ostr); 02692 std::cout << ", tag_pstat=" << ostr.c_str() << std::endl; 02693 */ 02694 02695 #ifndef NDEBUG 02696 { 02697 // Check for duplicates in proc list 02698 std::set< unsigned int > dumprocs; 02699 unsigned int dp = 0; 02700 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ ) 02701 dumprocs.insert( ps[dp] ); 02702 assert( dp == dumprocs.size() ); 02703 } 02704 #endif 02705 02706 // If only one sharer and I'm the owner, insert myself in the list; 02707 // otherwise, my data is checked at the end 02708 if( 1 == new_numps && !( new_pstat & PSTATUS_NOT_OWNED ) ) 02709 { 02710 new_hs[1] = new_hs[0]; 02711 new_ps[1] = new_ps[0]; 02712 new_hs[0] = new_h; 02713 new_ps[0] = rank(); 02714 new_numps = 2; 02715 } 02716 02717 // Now put passed-in data onto lists 02718 int idx; 02719 for( int i = 0; i < num_ps; i++ ) 02720 { 02721 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, ps[i] ) - &new_ps[0]; 02722 if( idx < new_numps ) 02723 { 02724 if( !new_hs[idx] && hs[i] ) 02725 // h on list is 0 and passed-in h is non-zero, replace it 02726 new_hs[idx] = hs[i]; 02727 else 02728 assert( !hs[i] || new_hs[idx] == hs[i] ); 02729 } 02730 else 02731 { 02732 if( new_numps + 1 == MAX_SHARING_PROCS ) 02733 { 02734 MB_SET_ERR( MB_FAILURE, "Exceeded MAX_SHARING_PROCS for " 02735 << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) << ' ' 02736 << ID_FROM_HANDLE( new_h ) << " in process " << rank() ); 02737 } 02738 new_ps[new_numps] = ps[i]; 02739 new_hs[new_numps] = hs[i]; 02740 new_numps++; 02741 } 02742 } 02743 02744 // Add myself, if it isn't there already 02745 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, rank() ) - &new_ps[0]; 02746 if( idx == new_numps ) 02747 { 02748 new_ps[new_numps] = rank(); 02749 new_hs[new_numps] = new_h; 02750 new_numps++; 02751 } 02752 else if( !new_hs[idx] && new_numps > 2 ) 02753 new_hs[idx] = new_h; 02754 02755 // Proc list is complete; update for shared, multishared 02756 if( new_numps > 1 ) 02757 { 02758 if( new_numps > 2 ) new_pstat |= PSTATUS_MULTISHARED; 02759 new_pstat |= PSTATUS_SHARED; 02760 } 02761 02762 /* 02763 plist("new_ps", new_ps, new_numps); 02764 plist("new_hs", new_hs, new_numps); 02765 print_pstatus(new_pstat, ostr); 02766 std::cout << ", new_pstat=" << ostr.c_str() << std::endl; 02767 std::cout << std::endl; 02768 */ 02769 02770 result = set_sharing_data( new_h, new_pstat, num_exist, new_numps, &new_ps[0], &new_hs[0] );MB_CHK_SET_ERR( result, "Failed to set sharing data in update_remote_data" ); 02771 02772 if( new_pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h ); 02773 02774 return MB_SUCCESS; 02775 } 02776 02777 ErrorCode ParallelComm::update_remote_data_old( const EntityHandle new_h, 02778 const int* ps, 02779 const EntityHandle* hs, 02780 const int num_ps, 02781 const unsigned char add_pstat ) 02782 { 02783 EntityHandle tag_hs[MAX_SHARING_PROCS]; 02784 int tag_ps[MAX_SHARING_PROCS]; 02785 unsigned char pstat; 02786 // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0 02787 // in this function, so no need to initialize 02788 unsigned int num_exist; 02789 ErrorCode result = get_sharing_data( new_h, tag_ps, tag_hs, pstat, num_exist );MB_CHK_ERR( result ); 02790 02791 #ifndef NDEBUG 02792 { 02793 // Check for duplicates in proc list 02794 std::set< unsigned int > dumprocs; 02795 unsigned int dp = 0; 02796 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ ) 02797 dumprocs.insert( ps[dp] ); 02798 assert( dp == dumprocs.size() ); 02799 } 02800 #endif 02801 02802 // Add any new sharing data 02803 bool changed = false; 02804 int idx; 02805 if( !num_exist ) 02806 { 02807 // Just take what caller passed 02808 memcpy( tag_ps, ps, num_ps * sizeof( int ) ); 02809 memcpy( tag_hs, hs, num_ps * sizeof( EntityHandle ) ); 02810 num_exist = num_ps; 02811 // If it's only one, hopefully I'm not there yet... 02812 assert( "I shouldn't be the only proc there." && ( 1 != num_exist || ps[0] != (int)procConfig.proc_rank() ) ); 02813 changed = true; 02814 } 02815 else 02816 { 02817 for( int i = 0; i < num_ps; i++ ) 02818 { 02819 idx = std::find( tag_ps, tag_ps + num_exist, ps[i] ) - tag_ps; 02820 if( idx == (int)num_exist ) 02821 { 02822 if( num_exist == MAX_SHARING_PROCS ) 02823 { 02824 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) 02825 << ' ' << ID_FROM_HANDLE( new_h ) << " in process " << proc_config().proc_rank() 02826 << std::endl; 02827 std::cerr.flush(); 02828 MPI_Abort( proc_config().proc_comm(), 66 ); 02829 } 02830 02831 // If there's only 1 sharing proc, and it's not me, then 02832 // we'll end up with 3; add me to the front 02833 if( !i && num_ps == 1 && num_exist == 1 && ps[0] != (int)procConfig.proc_rank() ) 02834 { 02835 int j = 1; 02836 // If I own this entity, put me at front, otherwise after first 02837 if( !( pstat & PSTATUS_NOT_OWNED ) ) 02838 { 02839 tag_ps[1] = tag_ps[0]; 02840 tag_hs[1] = tag_hs[0]; 02841 j = 0; 02842 } 02843 tag_ps[j] = procConfig.proc_rank(); 02844 tag_hs[j] = new_h; 02845 num_exist++; 02846 } 02847 02848 tag_ps[num_exist] = ps[i]; 02849 tag_hs[num_exist] = hs[i]; 02850 num_exist++; 02851 changed = true; 02852 } 02853 else if( 0 == tag_hs[idx] ) 02854 { 02855 tag_hs[idx] = hs[i]; 02856 changed = true; 02857 } 02858 else if( 0 != hs[i] ) 02859 { 02860 assert( hs[i] == tag_hs[idx] ); 02861 } 02862 } 02863 } 02864 02865 // Adjust for interface layer if necessary 02866 if( add_pstat & PSTATUS_INTERFACE ) 02867 { 02868 idx = std::min_element( tag_ps, tag_ps + num_exist ) - tag_ps; 02869 if( idx ) 02870 { 02871 int tag_proc = tag_ps[idx]; 02872 tag_ps[idx] = tag_ps[0]; 02873 tag_ps[0] = tag_proc; 02874 EntityHandle tag_h = tag_hs[idx]; 02875 tag_hs[idx] = tag_hs[0]; 02876 tag_hs[0] = tag_h; 02877 changed = true; 02878 if( tag_ps[0] != (int)procConfig.proc_rank() ) pstat |= PSTATUS_NOT_OWNED; 02879 } 02880 } 02881 02882 if( !changed ) return MB_SUCCESS; 02883 02884 assert( "interface entities should have > 1 proc" && ( !( add_pstat & PSTATUS_INTERFACE ) || num_exist > 1 ) ); 02885 assert( "ghost entities should have > 1 proc" && ( !( add_pstat & PSTATUS_GHOST ) || num_exist > 1 ) ); 02886 02887 // If it's multi-shared and we created the entity in this unpack, 02888 // local handle probably isn't in handle list yet 02889 if( num_exist > 2 ) 02890 { 02891 idx = std::find( tag_ps, tag_ps + num_exist, procConfig.proc_rank() ) - tag_ps; 02892 assert( idx < (int)num_exist ); 02893 if( !tag_hs[idx] ) tag_hs[idx] = new_h; 02894 } 02895 02896 int tag_p; 02897 EntityHandle tag_h; 02898 02899 // Update pstat 02900 pstat |= add_pstat; 02901 02902 if( num_exist > 2 ) 02903 pstat |= ( PSTATUS_MULTISHARED | PSTATUS_SHARED ); 02904 else if( num_exist > 0 ) 02905 pstat |= PSTATUS_SHARED; 02906 02907 // compare_remote_data(new_h, num_ps, hs, ps, add_pstat, 02908 // num_exist, tag_hs, tag_ps, pstat); 02909 02910 // Reset single shared proc/handle if was shared and moving to multi-shared 02911 if( num_exist > 2 && !( pstat & PSTATUS_MULTISHARED ) && ( pstat & PSTATUS_SHARED ) ) 02912 { 02913 // Must remove sharedp/h first, which really means set to default value 02914 tag_p = -1; 02915 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, &tag_p );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" ); 02916 tag_h = 0; 02917 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, &tag_h );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" ); 02918 } 02919 02920 // Set sharing tags 02921 if( num_exist > 2 ) 02922 { 02923 std::fill( tag_ps + num_exist, tag_ps + MAX_SHARING_PROCS, -1 ); 02924 std::fill( tag_hs + num_exist, tag_hs + MAX_SHARING_PROCS, 0 ); 02925 result = mbImpl->tag_set_data( sharedps_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" ); 02926 result = mbImpl->tag_set_data( sharedhs_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" ); 02927 02928 #ifndef NDEBUG 02929 { 02930 // Check for duplicates in proc list 02931 std::set< unsigned int > dumprocs; 02932 unsigned int dp = 0; 02933 for( ; dp < num_exist && -1 != tag_ps[dp]; dp++ ) 02934 dumprocs.insert( tag_ps[dp] ); 02935 assert( dp == dumprocs.size() ); 02936 } 02937 #endif 02938 } 02939 else if( num_exist == 2 || num_exist == 1 ) 02940 { 02941 if( tag_ps[0] == (int)procConfig.proc_rank() ) 02942 { 02943 assert( 2 == num_exist && tag_ps[1] != (int)procConfig.proc_rank() ); 02944 tag_ps[0] = tag_ps[1]; 02945 tag_hs[0] = tag_hs[1]; 02946 } 02947 assert( tag_ps[0] != -1 && tag_hs[0] != 0 ); 02948 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" ); 02949 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" ); 02950 } 02951 02952 // Now set new pstatus 02953 result = mbImpl->tag_set_data( pstatus_tag(), &new_h, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 02954 02955 if( pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h ); 02956 02957 return MB_SUCCESS; 02958 } 02959 02960 ErrorCode ParallelComm::get_sharing_data( const Range& entities, std::set< int >& procs, int operation ) 02961 { 02962 // Get the union or intersection of sharing data for multiple entities 02963 ErrorCode result; 02964 int sp2[MAX_SHARING_PROCS]; 02965 int num_ps; 02966 unsigned char pstat; 02967 std::set< int > tmp_procs; 02968 procs.clear(); 02969 02970 for( Range::const_iterator rit = entities.begin(); rit != entities.end(); ++rit ) 02971 { 02972 // Get sharing procs 02973 result = get_sharing_data( *rit, sp2, NULL, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_sharing_data" ); 02974 if( !( pstat & PSTATUS_SHARED ) && Interface::INTERSECT == operation ) 02975 { 02976 procs.clear(); 02977 return MB_SUCCESS; 02978 } 02979 02980 if( rit == entities.begin() ) 02981 { 02982 std::copy( sp2, sp2 + num_ps, std::inserter( procs, procs.begin() ) ); 02983 } 02984 else 02985 { 02986 std::sort( sp2, sp2 + num_ps ); 02987 tmp_procs.clear(); 02988 if( Interface::UNION == operation ) 02989 std::set_union( procs.begin(), procs.end(), sp2, sp2 + num_ps, 02990 std::inserter( tmp_procs, tmp_procs.end() ) ); 02991 else if( Interface::INTERSECT == operation ) 02992 std::set_intersection( procs.begin(), procs.end(), sp2, sp2 + num_ps, 02993 std::inserter( tmp_procs, tmp_procs.end() ) ); 02994 else 02995 { 02996 assert( "Unknown operation." && false ); 02997 return MB_FAILURE; 02998 } 02999 procs.swap( tmp_procs ); 03000 } 03001 if( Interface::INTERSECT == operation && procs.empty() ) return MB_SUCCESS; 03002 } 03003 03004 return MB_SUCCESS; 03005 } 03006 03007 ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity, 03008 int* ps, 03009 EntityHandle* hs, 03010 unsigned char& pstat, 03011 unsigned int& num_ps ) 03012 { 03013 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 03014 if( pstat & PSTATUS_MULTISHARED ) 03015 { 03016 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" ); 03017 if( hs ) 03018 { 03019 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 03020 } 03021 num_ps = std::find( ps, ps + MAX_SHARING_PROCS, -1 ) - ps; 03022 } 03023 else if( pstat & PSTATUS_SHARED ) 03024 { 03025 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 03026 if( hs ) 03027 { 03028 result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" ); 03029 hs[1] = 0; 03030 } 03031 // Initialize past end of data 03032 ps[1] = -1; 03033 num_ps = 1; 03034 } 03035 else 03036 { 03037 ps[0] = -1; 03038 if( hs ) hs[0] = 0; 03039 num_ps = 0; 03040 } 03041 03042 assert( MAX_SHARING_PROCS >= num_ps ); 03043 03044 return MB_SUCCESS; 03045 } 03046 03047 ErrorCode ParallelComm::find_existing_entity( const bool is_iface, 03048 const int owner_p, 03049 const EntityHandle owner_h, 03050 const int num_ps, 03051 const EntityHandle* connect, 03052 const int num_connect, 03053 const EntityType this_type, 03054 std::vector< EntityHandle >& L2hloc, 03055 std::vector< EntityHandle >& L2hrem, 03056 std::vector< unsigned int >& L2p, 03057 EntityHandle& new_h ) 03058 { 03059 new_h = 0; 03060 if( !is_iface && num_ps > 2 ) 03061 { 03062 for( unsigned int i = 0; i < L2hrem.size(); i++ ) 03063 { 03064 if( L2hrem[i] == owner_h && owner_p == (int)L2p[i] ) 03065 { 03066 new_h = L2hloc[i]; 03067 return MB_SUCCESS; 03068 } 03069 } 03070 } 03071 03072 // If we got here and it's a vertex, we don't need to look further 03073 if( MBVERTEX == this_type || !connect || !num_connect ) return MB_SUCCESS; 03074 03075 Range tmp_range; 03076 ErrorCode result = mbImpl->get_adjacencies( connect, num_connect, CN::Dimension( this_type ), false, tmp_range );MB_CHK_SET_ERR( result, "Failed to get existing entity" ); 03077 if( !tmp_range.empty() ) 03078 { 03079 // Found a corresponding entity - return target 03080 new_h = *tmp_range.begin(); 03081 } 03082 else 03083 { 03084 new_h = 0; 03085 } 03086 03087 return MB_SUCCESS; 03088 } 03089 03090 ErrorCode ParallelComm::get_local_handles( const Range& remote_handles, 03091 Range& local_handles, 03092 const std::vector< EntityHandle >& new_ents ) 03093 { 03094 std::vector< EntityHandle > rh_vec; 03095 rh_vec.reserve( remote_handles.size() ); 03096 std::copy( remote_handles.begin(), remote_handles.end(), std::back_inserter( rh_vec ) ); 03097 ErrorCode result = get_local_handles( &rh_vec[0], remote_handles.size(), new_ents ); 03098 std::copy( rh_vec.begin(), rh_vec.end(), range_inserter( local_handles ) ); 03099 return result; 03100 } 03101 03102 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents ) 03103 { 03104 std::vector< EntityHandle > tmp_ents; 03105 std::copy( new_ents.begin(), new_ents.end(), std::back_inserter( tmp_ents ) ); 03106 return get_local_handles( from_vec, num_ents, tmp_ents ); 03107 } 03108 03109 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, 03110 int num_ents, 03111 const std::vector< EntityHandle >& new_ents ) 03112 { 03113 for( int i = 0; i < num_ents; i++ ) 03114 { 03115 if( TYPE_FROM_HANDLE( from_vec[i] ) == MBMAXTYPE ) 03116 { 03117 assert( ID_FROM_HANDLE( from_vec[i] ) < (int)new_ents.size() ); 03118 from_vec[i] = new_ents[ID_FROM_HANDLE( from_vec[i] )]; 03119 } 03120 } 03121 03122 return MB_SUCCESS; 03123 } 03124 03125 /* 03126 template <typename T> void 03127 insert_in_array(T* array, size_t array_size, size_t location, T value) 03128 { 03129 assert(location + 1 < array_size); 03130 for (size_t i = array_size - 1; i > location; i--) 03131 array[i] = array[i - 1]; 03132 array[location] = value; 03133 } 03134 */ 03135 03136 ErrorCode ParallelComm::pack_range_map( Range& key_range, EntityHandle val_start, HandleMap& handle_map ) 03137 { 03138 for( Range::const_pair_iterator key_it = key_range.const_pair_begin(); key_it != key_range.const_pair_end(); 03139 ++key_it ) 03140 { 03141 int tmp_num = ( *key_it ).second - ( *key_it ).first + 1; 03142 handle_map.insert( ( *key_it ).first, val_start, tmp_num ); 03143 val_start += tmp_num; 03144 } 03145 03146 return MB_SUCCESS; 03147 } 03148 03149 ErrorCode ParallelComm::pack_sets( Range& entities, Buffer* buff, const bool store_remote_handles, const int to_proc ) 03150 { 03151 // SETS: 03152 // . #sets 03153 // . for each set: 03154 // - options[#sets] (unsigned int) 03155 // - if (unordered) set range 03156 // - else if ordered 03157 // . #ents in set 03158 // . handles[#ents] 03159 // - #parents 03160 // - if (#parents) handles[#parents] 03161 // - #children 03162 // - if (#children) handles[#children] 03163 03164 // Now the sets; assume any sets the application wants to pass are in the entities list 03165 ErrorCode result; 03166 Range all_sets = entities.subset_by_type( MBENTITYSET ); 03167 03168 int buff_size = estimate_sets_buffer_size( all_sets, store_remote_handles ); 03169 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate sets buffer size" ); 03170 buff->check_space( buff_size ); 03171 03172 // Number of sets 03173 PACK_INT( buff->buff_ptr, all_sets.size() ); 03174 03175 // Options for all sets 03176 std::vector< unsigned int > options( all_sets.size() ); 03177 Range::iterator rit; 03178 std::vector< EntityHandle > members; 03179 int i; 03180 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03181 { 03182 result = mbImpl->get_meshset_options( *rit, options[i] );MB_CHK_SET_ERR( result, "Failed to get meshset options" ); 03183 } 03184 buff->check_space( all_sets.size() * sizeof( unsigned int ) ); 03185 PACK_VOID( buff->buff_ptr, &options[0], all_sets.size() * sizeof( unsigned int ) ); 03186 03187 // Pack parallel geometry unique id 03188 if( !all_sets.empty() ) 03189 { 03190 Tag uid_tag; 03191 int n_sets = all_sets.size(); 03192 bool b_pack = false; 03193 std::vector< int > id_data( n_sets ); 03194 result = 03195 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" ); 03196 03197 result = mbImpl->tag_get_data( uid_tag, all_sets, &id_data[0] ); 03198 if( MB_TAG_NOT_FOUND != result ) 03199 { 03200 if( MB_SUCCESS != result ) MB_SET_ERR( result, "Failed to get parallel geometry unique ids" ); 03201 for( i = 0; i < n_sets; i++ ) 03202 { 03203 if( id_data[i] != 0 ) 03204 { 03205 b_pack = true; 03206 break; 03207 } 03208 } 03209 } 03210 03211 if( b_pack ) 03212 { // If you find 03213 buff->check_space( ( n_sets + 1 ) * sizeof( int ) ); 03214 PACK_INT( buff->buff_ptr, n_sets ); 03215 PACK_INTS( buff->buff_ptr, &id_data[0], n_sets ); 03216 } 03217 else 03218 { 03219 buff->check_space( sizeof( int ) ); 03220 PACK_INT( buff->buff_ptr, 0 ); 03221 } 03222 } 03223 03224 // Vectors/ranges 03225 std::vector< EntityHandle > entities_vec( entities.size() ); 03226 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 03227 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03228 { 03229 members.clear(); 03230 result = mbImpl->get_entities_by_handle( *rit, members );MB_CHK_SET_ERR( result, "Failed to get entities in ordered set" ); 03231 result = 03232 get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" ); 03233 buff->check_space( members.size() * sizeof( EntityHandle ) + sizeof( int ) ); 03234 PACK_INT( buff->buff_ptr, members.size() ); 03235 PACK_EH( buff->buff_ptr, &members[0], members.size() ); 03236 } 03237 03238 // Pack parent/child sets 03239 if( !store_remote_handles ) 03240 { // Only works not store remote handles 03241 // Pack numbers of parents/children 03242 unsigned int tot_pch = 0; 03243 int num_pch; 03244 buff->check_space( 2 * all_sets.size() * sizeof( int ) ); 03245 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03246 { 03247 // Pack parents 03248 result = mbImpl->num_parent_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num parents" ); 03249 PACK_INT( buff->buff_ptr, num_pch ); 03250 tot_pch += num_pch; 03251 result = mbImpl->num_child_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num children" ); 03252 PACK_INT( buff->buff_ptr, num_pch ); 03253 tot_pch += num_pch; 03254 } 03255 03256 // Now pack actual parents/children 03257 members.clear(); 03258 members.reserve( tot_pch ); 03259 std::vector< EntityHandle > tmp_pch; 03260 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03261 { 03262 result = mbImpl->get_parent_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get parents" ); 03263 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) ); 03264 tmp_pch.clear(); 03265 result = mbImpl->get_child_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get children" ); 03266 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) ); 03267 tmp_pch.clear(); 03268 } 03269 assert( members.size() == tot_pch ); 03270 if( !members.empty() ) 03271 { 03272 result = get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, 03273 entities_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles for set parent/child sets" ); 03274 #ifndef NDEBUG 03275 // Check that all handles are either sets or maxtype 03276 for( unsigned int __j = 0; __j < members.size(); __j++ ) 03277 assert( ( TYPE_FROM_HANDLE( members[__j] ) == MBMAXTYPE && 03278 ID_FROM_HANDLE( members[__j] ) < (int)entities.size() ) || 03279 TYPE_FROM_HANDLE( members[__j] ) == MBENTITYSET ); 03280 #endif 03281 buff->check_space( members.size() * sizeof( EntityHandle ) ); 03282 PACK_EH( buff->buff_ptr, &members[0], members.size() ); 03283 } 03284 } 03285 else 03286 { 03287 buff->check_space( 2 * all_sets.size() * sizeof( int ) ); 03288 for( rit = all_sets.begin(); rit != all_sets.end(); ++rit ) 03289 { 03290 PACK_INT( buff->buff_ptr, 0 ); 03291 PACK_INT( buff->buff_ptr, 0 ); 03292 } 03293 } 03294 03295 // Pack the handles 03296 if( store_remote_handles && !all_sets.empty() ) 03297 { 03298 buff_size = RANGE_SIZE( all_sets ); 03299 buff->check_space( buff_size ); 03300 PACK_RANGE( buff->buff_ptr, all_sets ); 03301 } 03302 03303 myDebug->tprintf( 4, "Done packing sets.\n" ); 03304 03305 buff->set_stored_size(); 03306 03307 return MB_SUCCESS; 03308 } 03309 03310 ErrorCode ParallelComm::unpack_sets( unsigned char*& buff_ptr, 03311 std::vector< EntityHandle >& entities, 03312 const bool store_remote_handles, 03313 const int from_proc ) 03314 { 03315 // Now the sets; assume any sets the application wants to pass are in the entities list 03316 ErrorCode result; 03317 03318 bool no_sets = ( entities.empty() || ( mbImpl->type_from_handle( *entities.rbegin() ) == MBENTITYSET ) ); 03319 03320 Range new_sets; 03321 int num_sets; 03322 UNPACK_INT( buff_ptr, num_sets ); 03323 03324 if( !num_sets ) return MB_SUCCESS; 03325 03326 int i; 03327 Range::const_iterator rit; 03328 std::vector< EntityHandle > members; 03329 int num_ents; 03330 std::vector< unsigned int > options_vec( num_sets ); 03331 // Option value 03332 if( num_sets ) UNPACK_VOID( buff_ptr, &options_vec[0], num_sets * sizeof( unsigned int ) ); 03333 03334 // Unpack parallel geometry unique id 03335 int n_uid; 03336 UNPACK_INT( buff_ptr, n_uid ); 03337 if( n_uid > 0 && n_uid != num_sets ) 03338 { 03339 std::cerr << "The number of Parallel geometry unique ids should be same." << std::endl; 03340 } 03341 03342 if( n_uid > 0 ) 03343 { // If parallel geometry unique id is packed 03344 std::vector< int > uids( n_uid ); 03345 UNPACK_INTS( buff_ptr, &uids[0], n_uid ); 03346 03347 Tag uid_tag; 03348 result = 03349 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" ); 03350 03351 // Find existing sets 03352 for( i = 0; i < n_uid; i++ ) 03353 { 03354 EntityHandle set_handle; 03355 Range temp_sets; 03356 void* tag_vals[] = { &uids[i] }; 03357 if( uids[i] > 0 ) 03358 { 03359 result = mbImpl->get_entities_by_type_and_tag( 0, MBENTITYSET, &uid_tag, tag_vals, 1, temp_sets ); 03360 } 03361 if( !temp_sets.empty() ) 03362 { // Existing set 03363 set_handle = *temp_sets.begin(); 03364 } 03365 else 03366 { // Create a new set 03367 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" ); 03368 result = mbImpl->tag_set_data( uid_tag, &set_handle, 1, &uids[i] );MB_CHK_SET_ERR( result, "Failed to set parallel geometry unique ids" ); 03369 } 03370 new_sets.insert( set_handle ); 03371 } 03372 } 03373 else 03374 { 03375 // Create sets 03376 for( i = 0; i < num_sets; i++ ) 03377 { 03378 EntityHandle set_handle; 03379 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" ); 03380 03381 // Make sure new sets handles are monotonically increasing 03382 assert( set_handle > *new_sets.rbegin() ); 03383 new_sets.insert( set_handle ); 03384 } 03385 } 03386 03387 std::copy( new_sets.begin(), new_sets.end(), std::back_inserter( entities ) ); 03388 // Only need to sort if we came in with no sets on the end 03389 if( !no_sets ) std::sort( entities.begin(), entities.end() ); 03390 03391 for( rit = new_sets.begin(), i = 0; rit != new_sets.end(); ++rit, i++ ) 03392 { 03393 // Unpack entities as vector, with length 03394 UNPACK_INT( buff_ptr, num_ents ); 03395 members.resize( num_ents ); 03396 if( num_ents ) UNPACK_EH( buff_ptr, &members[0], num_ents ); 03397 result = get_local_handles( &members[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for ordered set contents" ); 03398 result = mbImpl->add_entities( *rit, &members[0], num_ents );MB_CHK_SET_ERR( result, "Failed to add ents to ordered set in unpack" ); 03399 } 03400 03401 std::vector< int > num_pch( 2 * new_sets.size() ); 03402 std::vector< int >::iterator vit; 03403 int tot_pch = 0; 03404 for( vit = num_pch.begin(); vit != num_pch.end(); ++vit ) 03405 { 03406 UNPACK_INT( buff_ptr, *vit ); 03407 tot_pch += *vit; 03408 } 03409 03410 members.resize( tot_pch ); 03411 UNPACK_EH( buff_ptr, &members[0], tot_pch ); 03412 result = get_local_handles( &members[0], tot_pch, entities );MB_CHK_SET_ERR( result, "Failed to get local handle for parent/child sets" ); 03413 03414 int num = 0; 03415 EntityHandle* mem_ptr = &members[0]; 03416 for( rit = new_sets.begin(); rit != new_sets.end(); ++rit ) 03417 { 03418 // Unpack parents/children 03419 int num_par = num_pch[num++], num_child = num_pch[num++]; 03420 if( num_par + num_child ) 03421 { 03422 for( i = 0; i < num_par; i++ ) 03423 { 03424 assert( 0 != mem_ptr[i] ); 03425 result = mbImpl->add_parent_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add parent to set in unpack" ); 03426 } 03427 mem_ptr += num_par; 03428 for( i = 0; i < num_child; i++ ) 03429 { 03430 assert( 0 != mem_ptr[i] ); 03431 result = mbImpl->add_child_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add child to set in unpack" ); 03432 } 03433 mem_ptr += num_child; 03434 } 03435 } 03436 03437 // Unpack source handles 03438 Range dum_range; 03439 if( store_remote_handles && !new_sets.empty() ) 03440 { 03441 UNPACK_RANGE( buff_ptr, dum_range ); 03442 result = update_remote_data( new_sets, dum_range, from_proc, 0 );MB_CHK_SET_ERR( result, "Failed to set sharing data for sets" ); 03443 } 03444 03445 myDebug->tprintf( 4, "Done unpacking sets." ); 03446 03447 return MB_SUCCESS; 03448 } 03449 03450 ErrorCode ParallelComm::pack_adjacencies( Range& /*entities*/, 03451 Range::const_iterator& /*start_rit*/, 03452 Range& /*whole_range*/, 03453 unsigned char*& /*buff_ptr*/, 03454 int& /*count*/, 03455 const bool /*just_count*/, 03456 const bool /*store_handles*/, 03457 const int /*to_proc*/ ) 03458 { 03459 return MB_FAILURE; 03460 } 03461 03462 ErrorCode ParallelComm::unpack_adjacencies( unsigned char*& /*buff_ptr*/, 03463 Range& /*entities*/, 03464 const bool /*store_handles*/, 03465 const int /*from_proc*/ ) 03466 { 03467 return MB_FAILURE; 03468 } 03469 03470 ErrorCode ParallelComm::pack_tags( Range& entities, 03471 const std::vector< Tag >& src_tags, 03472 const std::vector< Tag >& dst_tags, 03473 const std::vector< Range >& tag_ranges, 03474 Buffer* buff, 03475 const bool store_remote_handles, 03476 const int to_proc ) 03477 { 03478 ErrorCode result; 03479 std::vector< Tag >::const_iterator tag_it, dst_it; 03480 std::vector< Range >::const_iterator rit; 03481 int count = 0; 03482 03483 for( tag_it = src_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); ++tag_it, ++rit ) 03484 { 03485 result = packed_tag_size( *tag_it, *rit, count ); 03486 if( MB_SUCCESS != result ) return result; 03487 } 03488 03489 // Number of tags 03490 count += sizeof( int ); 03491 03492 buff->check_space( count ); 03493 03494 PACK_INT( buff->buff_ptr, src_tags.size() ); 03495 03496 std::vector< EntityHandle > entities_vec( entities.size() ); 03497 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 03498 03499 for( tag_it = src_tags.begin(), dst_it = dst_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); 03500 ++tag_it, ++dst_it, ++rit ) 03501 { 03502 result = pack_tag( *tag_it, *dst_it, *rit, entities_vec, buff, store_remote_handles, to_proc ); 03503 if( MB_SUCCESS != result ) return result; 03504 } 03505 03506 myDebug->tprintf( 4, "Done packing tags." ); 03507 03508 buff->set_stored_size(); 03509 03510 return MB_SUCCESS; 03511 } 03512 03513 ErrorCode ParallelComm::packed_tag_size( Tag tag, const Range& tagged_entities, int& count ) 03514 { 03515 // For dense tags, compute size assuming all entities have that tag 03516 // For sparse tags, get number of entities w/ that tag to compute size 03517 03518 std::vector< int > var_len_sizes; 03519 std::vector< const void* > var_len_values; 03520 03521 // Default value 03522 count += sizeof( int ); 03523 if( NULL != tag->get_default_value() ) count += tag->get_default_value_size(); 03524 03525 // Size, type, data type 03526 count += 3 * sizeof( int ); 03527 03528 // Name 03529 count += sizeof( int ); 03530 count += tag->get_name().size(); 03531 03532 // Range of tag 03533 count += sizeof( int ) + tagged_entities.size() * sizeof( EntityHandle ); 03534 03535 if( tag->get_size() == MB_VARIABLE_LENGTH ) 03536 { 03537 const int num_ent = tagged_entities.size(); 03538 // Send a tag size for each entity 03539 count += num_ent * sizeof( int ); 03540 // Send tag data for each entity 03541 var_len_sizes.resize( num_ent ); 03542 var_len_values.resize( num_ent ); 03543 ErrorCode result = 03544 tag->get_data( sequenceManager, errorHandler, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get lenghts of variable-length tag values" ); 03545 count += std::accumulate( var_len_sizes.begin(), var_len_sizes.end(), 0 ); 03546 } 03547 else 03548 { 03549 // Tag data values for range or vector 03550 count += tagged_entities.size() * tag->get_size(); 03551 } 03552 03553 return MB_SUCCESS; 03554 } 03555 03556 ErrorCode ParallelComm::pack_tag( Tag src_tag, 03557 Tag dst_tag, 03558 const Range& tagged_entities, 03559 const std::vector< EntityHandle >& whole_vec, 03560 Buffer* buff, 03561 const bool store_remote_handles, 03562 const int to_proc ) 03563 { 03564 ErrorCode result; 03565 std::vector< int > var_len_sizes; 03566 std::vector< const void* > var_len_values; 03567 03568 if( src_tag != dst_tag ) 03569 { 03570 if( dst_tag->get_size() != src_tag->get_size() ) return MB_TYPE_OUT_OF_RANGE; 03571 if( dst_tag->get_data_type() != src_tag->get_data_type() && dst_tag->get_data_type() != MB_TYPE_OPAQUE && 03572 src_tag->get_data_type() != MB_TYPE_OPAQUE ) 03573 return MB_TYPE_OUT_OF_RANGE; 03574 } 03575 03576 // Size, type, data type 03577 buff->check_space( 3 * sizeof( int ) ); 03578 PACK_INT( buff->buff_ptr, src_tag->get_size() ); 03579 TagType this_type; 03580 result = mbImpl->tag_get_type( dst_tag, this_type ); 03581 PACK_INT( buff->buff_ptr, (int)this_type ); 03582 DataType data_type = src_tag->get_data_type(); 03583 PACK_INT( buff->buff_ptr, (int)data_type ); 03584 int type_size = TagInfo::size_from_data_type( data_type ); 03585 03586 // Default value 03587 if( NULL == src_tag->get_default_value() ) 03588 { 03589 buff->check_space( sizeof( int ) ); 03590 PACK_INT( buff->buff_ptr, 0 ); 03591 } 03592 else 03593 { 03594 buff->check_space( src_tag->get_default_value_size() ); 03595 PACK_BYTES( buff->buff_ptr, src_tag->get_default_value(), src_tag->get_default_value_size() ); 03596 } 03597 03598 // Name 03599 buff->check_space( src_tag->get_name().size() ); 03600 PACK_BYTES( buff->buff_ptr, dst_tag->get_name().c_str(), dst_tag->get_name().size() ); 03601 03602 myDebug->tprintf( 4, "Packing tag \"%s\"", src_tag->get_name().c_str() ); 03603 if( src_tag != dst_tag ) myDebug->tprintf( 4, " (as tag \"%s\")", dst_tag->get_name().c_str() ); 03604 myDebug->tprintf( 4, "\n" ); 03605 03606 // Pack entities 03607 buff->check_space( tagged_entities.size() * sizeof( EntityHandle ) + sizeof( int ) ); 03608 PACK_INT( buff->buff_ptr, tagged_entities.size() ); 03609 std::vector< EntityHandle > dum_tagged_entities( tagged_entities.size() ); 03610 result = get_remote_handles( store_remote_handles, tagged_entities, &dum_tagged_entities[0], to_proc, whole_vec ); 03611 if( MB_SUCCESS != result ) 03612 { 03613 if( myDebug->get_verbosity() == 3 ) 03614 { 03615 std::cerr << "Failed to get remote handles for tagged entities:" << std::endl; 03616 tagged_entities.print( " " ); 03617 } 03618 MB_SET_ERR( result, "Failed to get remote handles for tagged entities" ); 03619 } 03620 03621 PACK_EH( buff->buff_ptr, &dum_tagged_entities[0], dum_tagged_entities.size() ); 03622 03623 const size_t num_ent = tagged_entities.size(); 03624 if( src_tag->get_size() == MB_VARIABLE_LENGTH ) 03625 { 03626 var_len_sizes.resize( num_ent, 0 ); 03627 var_len_values.resize( num_ent, 0 ); 03628 result = mbImpl->tag_get_by_ptr( src_tag, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get variable-length tag data in pack_tags" ); 03629 buff->check_space( num_ent * sizeof( int ) ); 03630 PACK_INTS( buff->buff_ptr, &var_len_sizes[0], num_ent ); 03631 for( unsigned int i = 0; i < num_ent; i++ ) 03632 { 03633 buff->check_space( var_len_sizes[i] ); 03634 PACK_VOID( buff->buff_ptr, var_len_values[i], type_size * var_len_sizes[i] ); 03635 } 03636 } 03637 else 03638 { 03639 buff->check_space( num_ent * src_tag->get_size() ); 03640 // Should be OK to read directly into buffer, since tags are untyped and 03641 // handled by memcpy 03642 result = mbImpl->tag_get_data( src_tag, tagged_entities, buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get tag data in pack_tags" ); 03643 buff->buff_ptr += num_ent * src_tag->get_size(); 03644 PC( num_ent * src_tag->get_size(), " void" ); 03645 } 03646 03647 return MB_SUCCESS; 03648 } 03649 03650 ErrorCode ParallelComm::get_tag_send_list( const Range& whole_range, 03651 std::vector< Tag >& all_tags, 03652 std::vector< Range >& tag_ranges ) 03653 { 03654 std::vector< Tag > tmp_tags; 03655 ErrorCode result = mbImpl->tag_get_tags( tmp_tags );MB_CHK_SET_ERR( result, "Failed to get tags in pack_tags" ); 03656 03657 std::vector< Tag >::iterator tag_it; 03658 for( tag_it = tmp_tags.begin(); tag_it != tmp_tags.end(); ++tag_it ) 03659 { 03660 std::string tag_name; 03661 result = mbImpl->tag_get_name( *tag_it, tag_name ); 03662 if( tag_name.c_str()[0] == '_' && tag_name.c_str()[1] == '_' ) continue; 03663 03664 Range tmp_range; 03665 result = ( *tag_it )->get_tagged_entities( sequenceManager, tmp_range );MB_CHK_SET_ERR( result, "Failed to get entities for tag in pack_tags" ); 03666 tmp_range = intersect( tmp_range, whole_range ); 03667 03668 if( tmp_range.empty() ) continue; 03669 03670 // OK, we'll be sending this tag 03671 all_tags.push_back( *tag_it ); 03672 tag_ranges.push_back( Range() ); 03673 tag_ranges.back().swap( tmp_range ); 03674 } 03675 03676 return MB_SUCCESS; 03677 } 03678 03679 ErrorCode ParallelComm::unpack_tags( unsigned char*& buff_ptr, 03680 std::vector< EntityHandle >& entities, 03681 const bool /*store_remote_handles*/, 03682 const int /*from_proc*/, 03683 const MPI_Op* const mpi_op ) 03684 { 03685 // Tags 03686 // Get all the tags 03687 // For dense tags, compute size assuming all entities have that tag 03688 // For sparse tags, get number of entities w/ that tag to compute size 03689 03690 ErrorCode result; 03691 03692 int num_tags; 03693 UNPACK_INT( buff_ptr, num_tags ); 03694 std::vector< const void* > var_len_vals; 03695 std::vector< unsigned char > dum_vals; 03696 std::vector< EntityHandle > dum_ehvals; 03697 03698 for( int i = 0; i < num_tags; i++ ) 03699 { 03700 // Tag handle 03701 Tag tag_handle; 03702 03703 // Size, data type 03704 int tag_size, tag_data_type, tag_type; 03705 UNPACK_INT( buff_ptr, tag_size ); 03706 UNPACK_INT( buff_ptr, tag_type ); 03707 UNPACK_INT( buff_ptr, tag_data_type ); 03708 03709 // Default value 03710 int def_val_size; 03711 UNPACK_INT( buff_ptr, def_val_size ); 03712 void* def_val_ptr = NULL; 03713 if( def_val_size ) 03714 { 03715 def_val_ptr = buff_ptr; 03716 buff_ptr += def_val_size; 03717 UPC( tag_size, " void" ); 03718 } 03719 03720 // Name 03721 int name_len; 03722 UNPACK_INT( buff_ptr, name_len ); 03723 std::string tag_name( reinterpret_cast< char* >( buff_ptr ), name_len ); 03724 buff_ptr += name_len; 03725 UPC( 64, " chars" ); 03726 03727 myDebug->tprintf( 4, "Unpacking tag %s\n", tag_name.c_str() ); 03728 03729 // Create the tag 03730 if( tag_size == MB_VARIABLE_LENGTH ) 03731 result = mbImpl->tag_get_handle( tag_name.c_str(), def_val_size, (DataType)tag_data_type, tag_handle, 03732 MB_TAG_VARLEN | MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr ); 03733 else 03734 result = mbImpl->tag_get_handle( tag_name.c_str(), tag_size, (DataType)tag_data_type, tag_handle, 03735 MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr ); 03736 if( MB_SUCCESS != result ) return result; 03737 03738 // Get handles and convert to local handles 03739 int num_ents; 03740 UNPACK_INT( buff_ptr, num_ents ); 03741 std::vector< EntityHandle > dum_ents( num_ents ); 03742 UNPACK_EH( buff_ptr, &dum_ents[0], num_ents ); 03743 03744 // In this case handles are indices into new entity range; need to convert 03745 // to local handles 03746 result = get_local_handles( &dum_ents[0], num_ents, entities );MB_CHK_SET_ERR( result, "Unable to convert to local handles" ); 03747 03748 // If it's a handle type, also convert tag vals in-place in buffer 03749 if( MB_TYPE_HANDLE == tag_type ) 03750 { 03751 dum_ehvals.resize( num_ents ); 03752 UNPACK_EH( buff_ptr, &dum_ehvals[0], num_ents ); 03753 result = get_local_handles( &dum_ehvals[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for tag vals" ); 03754 } 03755 03756 DataType data_type; 03757 mbImpl->tag_get_data_type( tag_handle, data_type ); 03758 int type_size = TagInfo::size_from_data_type( data_type ); 03759 03760 if( !dum_ents.empty() ) 03761 { 03762 if( tag_size == MB_VARIABLE_LENGTH ) 03763 { 03764 // Be careful of alignment here. If the integers are aligned 03765 // in the buffer, we can use them directly. Otherwise we must 03766 // copy them. 03767 std::vector< int > var_lengths( num_ents ); 03768 UNPACK_INTS( buff_ptr, &var_lengths[0], num_ents ); 03769 UPC( sizeof( int ) * num_ents, " void" ); 03770 03771 // Get pointers into buffer for each tag value 03772 var_len_vals.resize( num_ents ); 03773 for( std::vector< EntityHandle >::size_type j = 0; j < (std::vector< EntityHandle >::size_type)num_ents; 03774 j++ ) 03775 { 03776 var_len_vals[j] = buff_ptr; 03777 buff_ptr += var_lengths[j] * type_size; 03778 UPC( var_lengths[j], " void" ); 03779 } 03780 result = 03781 mbImpl->tag_set_by_ptr( tag_handle, &dum_ents[0], num_ents, &var_len_vals[0], &var_lengths[0] );MB_CHK_SET_ERR( result, "Failed to set tag data when unpacking variable-length tag" ); 03782 } 03783 else 03784 { 03785 // Get existing values of dst tag 03786 dum_vals.resize( tag_size * num_ents ); 03787 if( mpi_op ) 03788 { 03789 int tag_length; 03790 result = mbImpl->tag_get_length( tag_handle, tag_length );MB_CHK_SET_ERR( result, "Failed to get tag length" ); 03791 result = mbImpl->tag_get_data( tag_handle, &dum_ents[0], num_ents, &dum_vals[0] );MB_CHK_SET_ERR( result, "Failed to get existing value of dst tag on entities" ); 03792 result = reduce_void( tag_data_type, *mpi_op, tag_length * num_ents, &dum_vals[0], buff_ptr );MB_CHK_SET_ERR( result, "Failed to perform mpi op on dst tags" ); 03793 } 03794 result = mbImpl->tag_set_data( tag_handle, &dum_ents[0], num_ents, buff_ptr );MB_CHK_SET_ERR( result, "Failed to set range-based tag data when unpacking tag" ); 03795 buff_ptr += num_ents * tag_size; 03796 UPC( num_ents * tag_size, " void" ); 03797 } 03798 } 03799 } 03800 03801 myDebug->tprintf( 4, "Done unpacking tags.\n" ); 03802 03803 return MB_SUCCESS; 03804 } 03805 03806 template < class T > 03807 T LAND( const T& arg1, const T& arg2 ) 03808 { 03809 return arg1 && arg2; 03810 } 03811 template < class T > 03812 T LOR( const T& arg1, const T& arg2 ) 03813 { 03814 return arg1 || arg2; 03815 } 03816 template < class T > 03817 T LXOR( const T& arg1, const T& arg2 ) 03818 { 03819 return ( ( arg1 && !arg2 ) || ( !arg1 && arg2 ) ); 03820 } 03821 template < class T > 03822 T MAX( const T& arg1, const T& arg2 ) 03823 { 03824 return ( arg1 > arg2 ? arg1 : arg2 ); 03825 } 03826 template < class T > 03827 T MIN( const T& arg1, const T& arg2 ) 03828 { 03829 return ( arg1 < arg2 ? arg1 : arg2 ); 03830 } 03831 template < class T > 03832 T ADD( const T& arg1, const T& arg2 ) 03833 { 03834 return arg1 + arg2; 03835 } 03836 template < class T > 03837 T MULT( const T& arg1, const T& arg2 ) 03838 { 03839 return arg1 * arg2; 03840 } 03841 03842 template < class T > 03843 ErrorCode ParallelComm::reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals ) 03844 { 03845 T* old_tmp = reinterpret_cast< T* >( old_vals ); 03846 // T *new_tmp = reinterpret_cast<T*>(new_vals); 03847 // new vals pointer needs to be aligned , some compilers will optimize and will shift 03848 03849 std::vector< T > new_values; 03850 new_values.resize( num_ents ); 03851 memcpy( &new_values[0], new_vals, num_ents * sizeof( T ) ); 03852 T* new_tmp = &new_values[0]; 03853 03854 if( mpi_op == MPI_SUM ) 03855 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, ADD< T > ); 03856 else if( mpi_op == MPI_PROD ) 03857 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MULT< T > ); 03858 else if( mpi_op == MPI_MAX ) 03859 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MAX< T > ); 03860 else if( mpi_op == MPI_MIN ) 03861 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MIN< T > ); 03862 else if( mpi_op == MPI_LAND ) 03863 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LAND< T > ); 03864 else if( mpi_op == MPI_LOR ) 03865 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LOR< T > ); 03866 else if( mpi_op == MPI_LXOR ) 03867 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LXOR< T > ); 03868 else if( mpi_op == MPI_BAND || mpi_op == MPI_BOR || mpi_op == MPI_BXOR ) 03869 { 03870 std::cerr << "Bitwise operations not allowed in tag reductions." << std::endl; 03871 return MB_FAILURE; 03872 } 03873 else if( mpi_op != MPI_OP_NULL ) 03874 { 03875 std::cerr << "Unknown MPI operation type." << std::endl; 03876 return MB_TYPE_OUT_OF_RANGE; 03877 } 03878 03879 // copy now the result back where it should be 03880 memcpy( new_vals, new_tmp, num_ents * sizeof( T ) ); 03881 std::vector< T >().swap( new_values ); // way to release allocated vector 03882 03883 return MB_SUCCESS; 03884 } 03885 03886 ErrorCode ParallelComm::reduce_void( int tag_data_type, 03887 const MPI_Op mpi_op, 03888 int num_ents, 03889 void* old_vals, 03890 void* new_vals ) 03891 { 03892 ErrorCode result; 03893 switch( tag_data_type ) 03894 { 03895 case MB_TYPE_INTEGER: 03896 result = reduce< int >( mpi_op, num_ents, old_vals, new_vals ); 03897 break; 03898 case MB_TYPE_DOUBLE: 03899 result = reduce< double >( mpi_op, num_ents, old_vals, new_vals ); 03900 break; 03901 case MB_TYPE_BIT: 03902 result = reduce< unsigned char >( mpi_op, num_ents, old_vals, new_vals ); 03903 break; 03904 default: 03905 result = MB_SUCCESS; 03906 break; 03907 } 03908 03909 return result; 03910 } 03911 03912 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, int resolve_dim, int shared_dim, const Tag* id_tag ) 03913 { 03914 ErrorCode result; 03915 Range proc_ents; 03916 03917 // Check for structured mesh, and do it differently if it is 03918 ScdInterface* scdi; 03919 result = mbImpl->query_interface( scdi ); 03920 if( scdi ) 03921 { 03922 result = scdi->tag_shared_vertices( this, this_set ); 03923 if( MB_SUCCESS == result ) 03924 { 03925 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 03926 return result; 03927 } 03928 } 03929 03930 if( 0 == this_set ) 03931 { 03932 // Get the entities in the partition sets 03933 for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit ) 03934 { 03935 Range tmp_ents; 03936 result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true ); 03937 if( MB_SUCCESS != result ) return result; 03938 proc_ents.merge( tmp_ents ); 03939 } 03940 } 03941 else 03942 { 03943 result = mbImpl->get_entities_by_handle( this_set, proc_ents, true ); 03944 if( MB_SUCCESS != result ) return result; 03945 } 03946 03947 // Resolve dim is maximal dim of entities in proc_ents 03948 if( -1 == resolve_dim ) 03949 { 03950 if( !proc_ents.empty() ) resolve_dim = mbImpl->dimension_from_handle( *proc_ents.rbegin() ); 03951 } 03952 03953 // proc_ents should all be of same dimension 03954 if( resolve_dim > shared_dim && 03955 mbImpl->dimension_from_handle( *proc_ents.rbegin() ) != mbImpl->dimension_from_handle( *proc_ents.begin() ) ) 03956 { 03957 Range::iterator lower = proc_ents.lower_bound( CN::TypeDimensionMap[0].first ), 03958 upper = proc_ents.upper_bound( CN::TypeDimensionMap[resolve_dim - 1].second ); 03959 proc_ents.erase( lower, upper ); 03960 } 03961 03962 // Must call even if we don't have any entities, to make sure 03963 // collective comm'n works 03964 return resolve_shared_ents( this_set, proc_ents, resolve_dim, shared_dim, NULL, id_tag ); 03965 } 03966 03967 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, 03968 Range& proc_ents, 03969 int resolve_dim, 03970 int shared_dim, 03971 Range* skin_ents, 03972 const Tag* id_tag ) 03973 { 03974 #ifdef MOAB_HAVE_MPE 03975 if( myDebug->get_verbosity() == 2 ) 03976 { 03977 define_mpe(); 03978 MPE_Log_event( RESOLVE_START, procConfig.proc_rank(), "Entering resolve_shared_ents." ); 03979 } 03980 #endif 03981 03982 ErrorCode result; 03983 myDebug->tprintf( 1, "Resolving shared entities.\n" ); 03984 03985 if( resolve_dim < shared_dim ) 03986 { 03987 MB_SET_ERR( MB_FAILURE, "MOAB does not support vertex-based partitions, only element-based ones" ); 03988 } 03989 03990 if( -1 == shared_dim ) 03991 { 03992 if( !proc_ents.empty() ) 03993 shared_dim = mbImpl->dimension_from_handle( *proc_ents.begin() ) - 1; 03994 else if( resolve_dim == 3 ) 03995 shared_dim = 2; 03996 } 03997 int max_global_resolve_dim = -1; 03998 int err = MPI_Allreduce( &resolve_dim, &max_global_resolve_dim, 1, MPI_INT, MPI_MAX, proc_config().proc_comm() ); 03999 if( MPI_SUCCESS != err ) 04000 { 04001 MB_SET_ERR( MB_FAILURE, "Unable to guess global resolve_dim" ); 04002 } 04003 if( shared_dim < 0 || resolve_dim < 0 ) 04004 { 04005 // MB_SET_ERR(MB_FAILURE, "Unable to guess shared_dim or resolve_dim"); 04006 resolve_dim = max_global_resolve_dim; 04007 shared_dim = resolve_dim - 1; 04008 } 04009 04010 if( resolve_dim < 0 || shared_dim < 0 ) return MB_SUCCESS; 04011 // no task has any mesh, get out 04012 04013 // Get the skin entities by dimension 04014 Range tmp_skin_ents[4]; 04015 04016 // Get the entities to be skinned 04017 // Find the skin 04018 int skin_dim = resolve_dim - 1; 04019 if( !skin_ents ) 04020 { 04021 skin_ents = tmp_skin_ents; 04022 skin_ents[resolve_dim] = proc_ents; 04023 Skinner skinner( mbImpl ); 04024 result = 04025 skinner.find_skin( this_set, skin_ents[skin_dim + 1], false, skin_ents[skin_dim], NULL, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" ); 04026 myDebug->tprintf( 1, "Found skin: skin_dim: %d resolve_dim: %d , now resolving.\n", skin_dim, resolve_dim ); 04027 myDebug->tprintf( 3, "skin_ents[0].size(): %d skin_ents[1].size(): %d \n", (int)skin_ents[0].size(), 04028 (int)skin_ents[1].size() ); 04029 // Get entities adjacent to skin ents from shared_dim down to zero 04030 for( int this_dim = skin_dim - 1; this_dim >= 0; this_dim-- ) 04031 { 04032 result = 04033 mbImpl->get_adjacencies( skin_ents[skin_dim], this_dim, true, skin_ents[this_dim], Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" ); 04034 04035 if( this_set && skin_dim == 2 && this_dim == 1 ) 04036 { 04037 result = mbImpl->add_entities( this_set, skin_ents[this_dim] );MB_CHK_ERR( result ); 04038 } 04039 } 04040 } 04041 else if( skin_ents[resolve_dim].empty() ) 04042 skin_ents[resolve_dim] = proc_ents; 04043 04044 // Global id tag 04045 Tag gid_tag; 04046 if( id_tag ) 04047 gid_tag = *id_tag; 04048 else 04049 { 04050 bool tag_created = false; 04051 int def_val = -1; 04052 result = mbImpl->tag_get_handle( GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE | MB_TAG_CREAT, 04053 &def_val, &tag_created ); 04054 if( MB_ALREADY_ALLOCATED != result && MB_SUCCESS != result ) 04055 { 04056 MB_SET_ERR( result, "Failed to create/get gid tag handle" ); 04057 } 04058 else if( tag_created ) 04059 { 04060 // Just created it, so we need global ids 04061 result = assign_global_ids( this_set, skin_dim + 1, true, true, true );MB_CHK_SET_ERR( result, "Failed to assign global ids" ); 04062 } 04063 } 04064 04065 DataType tag_type; 04066 result = mbImpl->tag_get_data_type( gid_tag, tag_type );MB_CHK_SET_ERR( result, "Failed to get tag data type" ); 04067 int bytes_per_tag; 04068 result = mbImpl->tag_get_bytes( gid_tag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed to get number of bytes per tag" ); 04069 // On 64 bits, long and int are different 04070 // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?) 04071 04072 // Get gids for skin ents in a vector, to pass to gs 04073 std::vector< long > lgid_data( skin_ents[0].size() ); 04074 // Size is either long or int 04075 // On 64 bit is 8 or 4 04076 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) ) 04077 { // It is a special id tag 04078 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &lgid_data[0] );MB_CHK_SET_ERR( result, "Couldn't get gid tag for skin vertices" ); 04079 } 04080 else if( 4 == bytes_per_tag ) 04081 { // Must be GLOBAL_ID tag or 32 bits ... 04082 std::vector< int > gid_data( lgid_data.size() ); 04083 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &gid_data[0] );MB_CHK_SET_ERR( result, "Failed to get gid tag for skin vertices" ); 04084 std::copy( gid_data.begin(), gid_data.end(), lgid_data.begin() ); 04085 } 04086 else 04087 { 04088 // Not supported flag 04089 MB_SET_ERR( MB_FAILURE, "Unsupported id tag" ); 04090 } 04091 04092 // Put handles in vector for passing to gs setup 04093 std::vector< Ulong > handle_vec; // Assumes that we can do conversion from Ulong to EntityHandle 04094 std::copy( skin_ents[0].begin(), skin_ents[0].end(), std::back_inserter( handle_vec ) ); 04095 04096 #ifdef MOAB_HAVE_MPE 04097 if( myDebug->get_verbosity() == 2 ) 04098 { 04099 MPE_Log_event( SHAREDV_START, procConfig.proc_rank(), "Creating crystal router." ); 04100 } 04101 #endif 04102 04103 // Get a crystal router 04104 gs_data::crystal_data* cd = procConfig.crystal_router(); 04105 04106 /* 04107 // Get total number of entities; will overshoot highest global id, but 04108 // that's OK 04109 int num_total[2] = {0, 0}, num_local[2] = {0, 0}; 04110 result = mbImpl->get_number_entities_by_dimension(this_set, 0, num_local); 04111 if (MB_SUCCESS != result)return result; 04112 int failure = MPI_Allreduce(num_local, num_total, 1, 04113 MPI_INT, MPI_SUM, procConfig.proc_comm()); 04114 if (failure) { 04115 MB_SET_ERR(MB_FAILURE, "Allreduce for total number of shared ents failed"); 04116 } 04117 */ 04118 // Call gather-scatter to get shared ids & procs 04119 gs_data* gsd = new gs_data(); 04120 // assert(sizeof(ulong_) == sizeof(EntityHandle)); 04121 result = gsd->initialize( skin_ents[0].size(), &lgid_data[0], &handle_vec[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" ); 04122 04123 // Get shared proc tags 04124 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 04125 result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 04126 04127 // Load shared verts into a tuple, then sort by index 04128 TupleList shared_verts; 04129 shared_verts.initialize( 2, 0, 1, 0, skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ) ); 04130 shared_verts.enableWriteAccess(); 04131 04132 unsigned int i = 0, j = 0; 04133 for( unsigned int p = 0; p < gsd->nlinfo->_np; p++ ) 04134 for( unsigned int np = 0; np < gsd->nlinfo->_nshared[p]; np++ ) 04135 { 04136 shared_verts.vi_wr[i++] = gsd->nlinfo->_sh_ind[j]; 04137 shared_verts.vi_wr[i++] = gsd->nlinfo->_target[p]; 04138 shared_verts.vul_wr[j] = gsd->nlinfo->_ulabels[j]; 04139 j++; 04140 shared_verts.inc_n(); 04141 } 04142 04143 myDebug->tprintf( 3, " shared verts size %d \n", (int)shared_verts.get_n() ); 04144 04145 int max_size = skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ); 04146 moab::TupleList::buffer sort_buffer; 04147 sort_buffer.buffer_init( max_size ); 04148 shared_verts.sort( 0, &sort_buffer ); 04149 sort_buffer.reset(); 04150 04151 // Set sharing procs and handles tags on skin ents 04152 int maxp = -1; 04153 std::vector< int > sharing_procs( MAX_SHARING_PROCS ); 04154 std::fill( sharing_procs.begin(), sharing_procs.end(), maxp ); 04155 j = 0; 04156 i = 0; 04157 04158 // Get ents shared by 1 or n procs 04159 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs; 04160 Range proc_verts; 04161 result = mbImpl->get_adjacencies( proc_ents, 0, false, proc_verts, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get proc_verts" ); 04162 04163 myDebug->print( 3, " resolve shared ents: proc verts ", proc_verts ); 04164 result = tag_shared_verts( shared_verts, skin_ents, proc_nvecs, proc_verts );MB_CHK_SET_ERR( result, "Failed to tag shared verts" ); 04165 04166 #ifdef MOAB_HAVE_MPE 04167 if( myDebug->get_verbosity() == 2 ) 04168 { 04169 MPE_Log_event( SHAREDV_END, procConfig.proc_rank(), "Finished tag_shared_verts." ); 04170 } 04171 #endif 04172 04173 // Get entities shared by 1 or n procs 04174 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );MB_CHK_SET_ERR( result, "Failed to tag shared entities" ); 04175 04176 shared_verts.reset(); 04177 04178 if( myDebug->get_verbosity() > 0 ) 04179 { 04180 for( std::map< std::vector< int >, std::vector< EntityHandle > >::const_iterator mit = proc_nvecs.begin(); 04181 mit != proc_nvecs.end(); ++mit ) 04182 { 04183 myDebug->tprintf( 1, "Iface: " ); 04184 for( std::vector< int >::const_iterator vit = ( mit->first ).begin(); vit != ( mit->first ).end(); ++vit ) 04185 myDebug->printf( 1, " %d", *vit ); 04186 myDebug->print( 1, "\n" ); 04187 } 04188 } 04189 04190 // Create the sets for each interface; store them as tags on 04191 // the interface instance 04192 Range iface_sets; 04193 result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" ); 04194 04195 // Establish comm procs and buffers for them 04196 std::set< unsigned int > procs; 04197 result = get_interface_procs( procs, true );MB_CHK_SET_ERR( result, "Failed to get interface procs" ); 04198 04199 #ifndef NDEBUG 04200 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Shared handle check failed after interface vertex exchange" ); 04201 #endif 04202 04203 // Resolve shared entity remote handles; implemented in ghost cell exchange 04204 // code because it's so similar 04205 result = exchange_ghost_cells( -1, -1, 0, 0, true, true );MB_CHK_SET_ERR( result, "Failed to resolve shared entity remote handles" ); 04206 04207 // Now build parent/child links for interface sets 04208 result = create_iface_pc_links();MB_CHK_SET_ERR( result, "Failed to create interface parent/child links" ); 04209 04210 gsd->reset(); 04211 delete gsd; 04212 04213 #ifdef MOAB_HAVE_MPE 04214 if( myDebug->get_verbosity() == 2 ) 04215 { 04216 MPE_Log_event( RESOLVE_END, procConfig.proc_rank(), "Exiting resolve_shared_ents." ); 04217 } 04218 #endif 04219 04220 // std::ostringstream ent_str; 04221 // ent_str << "mesh." << procConfig.proc_rank() << ".h5m"; 04222 // mbImpl->write_mesh(ent_str.str().c_str()); 04223 04224 // Done 04225 return result; 04226 } 04227 04228 void ParallelComm::define_mpe() 04229 { 04230 #ifdef MOAB_HAVE_MPE 04231 if( myDebug->get_verbosity() == 2 ) 04232 { 04233 // Define mpe states used for logging 04234 int success; 04235 MPE_Log_get_state_eventIDs( &IFACE_START, &IFACE_END ); 04236 MPE_Log_get_state_eventIDs( &GHOST_START, &GHOST_END ); 04237 MPE_Log_get_state_eventIDs( &SHAREDV_START, &SHAREDV_END ); 04238 MPE_Log_get_state_eventIDs( &RESOLVE_START, &RESOLVE_END ); 04239 MPE_Log_get_state_eventIDs( &ENTITIES_START, &ENTITIES_END ); 04240 MPE_Log_get_state_eventIDs( &RHANDLES_START, &RHANDLES_END ); 04241 MPE_Log_get_state_eventIDs( &OWNED_START, &OWNED_END ); 04242 success = MPE_Describe_state( IFACE_START, IFACE_END, "Resolve interface ents", "green" ); 04243 assert( MPE_LOG_OK == success ); 04244 success = MPE_Describe_state( GHOST_START, GHOST_END, "Exchange ghost ents", "red" ); 04245 assert( MPE_LOG_OK == success ); 04246 success = MPE_Describe_state( SHAREDV_START, SHAREDV_END, "Resolve interface vertices", "blue" ); 04247 assert( MPE_LOG_OK == success ); 04248 success = MPE_Describe_state( RESOLVE_START, RESOLVE_END, "Resolve shared ents", "purple" ); 04249 assert( MPE_LOG_OK == success ); 04250 success = MPE_Describe_state( ENTITIES_START, ENTITIES_END, "Exchange shared ents", "yellow" ); 04251 assert( MPE_LOG_OK == success ); 04252 success = MPE_Describe_state( RHANDLES_START, RHANDLES_END, "Remote handles", "cyan" ); 04253 assert( MPE_LOG_OK == success ); 04254 success = MPE_Describe_state( OWNED_START, OWNED_END, "Exchange owned ents", "black" ); 04255 assert( MPE_LOG_OK == success ); 04256 } 04257 #endif 04258 } 04259 04260 ErrorCode ParallelComm::resolve_shared_ents( ParallelComm** pc, 04261 const unsigned int np, 04262 EntityHandle this_set, 04263 const int part_dim ) 04264 { 04265 std::vector< Range > verts( np ); 04266 int tot_verts = 0; 04267 unsigned int p, i, j, v; 04268 ErrorCode rval; 04269 for( p = 0; p < np; p++ ) 04270 { 04271 Skinner skinner( pc[p]->get_moab() ); 04272 Range part_ents, skin_ents; 04273 rval = pc[p]->get_moab()->get_entities_by_dimension( this_set, part_dim, part_ents ); 04274 if( MB_SUCCESS != rval ) return rval; 04275 rval = skinner.find_skin( this_set, part_ents, false, skin_ents, 0, true, true, true ); 04276 if( MB_SUCCESS != rval ) return rval; 04277 rval = pc[p]->get_moab()->get_adjacencies( skin_ents, 0, true, verts[p], Interface::UNION ); 04278 if( MB_SUCCESS != rval ) return rval; 04279 tot_verts += verts[p].size(); 04280 } 04281 04282 TupleList shared_ents; 04283 shared_ents.initialize( 2, 0, 1, 0, tot_verts ); 04284 shared_ents.enableWriteAccess(); 04285 04286 i = 0; 04287 j = 0; 04288 std::vector< int > gids; 04289 Range::iterator rit; 04290 Tag gid_tag; 04291 for( p = 0; p < np; p++ ) 04292 { 04293 gid_tag = pc[p]->get_moab()->globalId_tag(); 04294 04295 gids.resize( verts[p].size() ); 04296 rval = pc[p]->get_moab()->tag_get_data( gid_tag, verts[p], &gids[0] ); 04297 if( MB_SUCCESS != rval ) return rval; 04298 04299 for( v = 0, rit = verts[p].begin(); v < gids.size(); v++, ++rit ) 04300 { 04301 shared_ents.vi_wr[i++] = gids[v]; 04302 shared_ents.vi_wr[i++] = p; 04303 shared_ents.vul_wr[j] = *rit; 04304 j++; 04305 shared_ents.inc_n(); 04306 } 04307 } 04308 04309 moab::TupleList::buffer sort_buffer; 04310 sort_buffer.buffer_init( tot_verts ); 04311 shared_ents.sort( 0, &sort_buffer ); 04312 sort_buffer.reset(); 04313 04314 j = 0; 04315 i = 0; 04316 std::vector< EntityHandle > handles; 04317 std::vector< int > procs; 04318 04319 while( i < shared_ents.get_n() ) 04320 { 04321 handles.clear(); 04322 procs.clear(); 04323 04324 // Count & accumulate sharing procs 04325 int this_gid = shared_ents.vi_rd[j]; 04326 while( i < shared_ents.get_n() && shared_ents.vi_rd[j] == this_gid ) 04327 { 04328 j++; 04329 procs.push_back( shared_ents.vi_rd[j++] ); 04330 handles.push_back( shared_ents.vul_rd[i++] ); 04331 } 04332 if( 1 == procs.size() ) continue; 04333 04334 for( v = 0; v < procs.size(); v++ ) 04335 { 04336 rval = pc[procs[v]]->update_remote_data( handles[v], &procs[0], &handles[0], procs.size(), 04337 ( procs[0] == (int)pc[procs[v]]->rank() 04338 ? PSTATUS_INTERFACE 04339 : ( PSTATUS_NOT_OWNED | PSTATUS_INTERFACE ) ) ); 04340 if( MB_SUCCESS != rval ) return rval; 04341 } 04342 } 04343 04344 std::set< unsigned int > psets; 04345 for( p = 0; p < np; p++ ) 04346 { 04347 rval = pc[p]->create_interface_sets( this_set, part_dim, part_dim - 1 ); 04348 if( MB_SUCCESS != rval ) return rval; 04349 // Establish comm procs and buffers for them 04350 psets.clear(); 04351 rval = pc[p]->get_interface_procs( psets, true ); 04352 if( MB_SUCCESS != rval ) return rval; 04353 } 04354 04355 shared_ents.reset(); 04356 04357 return MB_SUCCESS; 04358 } 04359 04360 ErrorCode ParallelComm::tag_iface_entities() 04361 { 04362 ErrorCode result = MB_SUCCESS; 04363 Range iface_ents, tmp_ents, rmv_ents; 04364 std::vector< unsigned char > pstat; 04365 unsigned char set_pstat; 04366 Range::iterator rit2; 04367 unsigned int i; 04368 04369 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 04370 { 04371 iface_ents.clear(); 04372 04373 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get interface set contents" ); 04374 pstat.resize( iface_ents.size() ); 04375 result = mbImpl->tag_get_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set entities" ); 04376 result = mbImpl->tag_get_data( pstatus_tag(), &( *rit ), 1, &set_pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set" ); 04377 rmv_ents.clear(); 04378 for( rit2 = iface_ents.begin(), i = 0; rit2 != iface_ents.end(); ++rit2, i++ ) 04379 { 04380 if( !( pstat[i] & PSTATUS_INTERFACE ) ) 04381 { 04382 rmv_ents.insert( *rit2 ); 04383 pstat[i] = 0x0; 04384 } 04385 } 04386 result = mbImpl->remove_entities( *rit, rmv_ents );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" ); 04387 04388 if( !( set_pstat & PSTATUS_NOT_OWNED ) ) continue; 04389 // If we're here, we need to set the notowned status on (remaining) set contents 04390 04391 // Remove rmv_ents from the contents list 04392 iface_ents = subtract( iface_ents, rmv_ents ); 04393 // Compress the pstat vector (removing 0x0's) 04394 std::remove_if( pstat.begin(), pstat.end(), 04395 std::bind( std::equal_to< unsigned char >(), std::placeholders::_1, 0x0 ) ); 04396 // std::bind2nd(std::equal_to<unsigned char>(), 0x0)); 04397 // https://stackoverflow.com/questions/32739018/a-replacement-for-stdbind2nd 04398 // Fold the not_owned bit into remaining values 04399 unsigned int sz = iface_ents.size(); 04400 for( i = 0; i < sz; i++ ) 04401 pstat[i] |= PSTATUS_NOT_OWNED; 04402 04403 // Set the tag on the entities 04404 result = mbImpl->tag_set_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus values for interface set entities" ); 04405 } 04406 04407 return MB_SUCCESS; 04408 } 04409 04410 ErrorCode ParallelComm::set_pstatus_entities( Range& pstatus_ents, 04411 unsigned char pstatus_val, 04412 bool lower_dim_ents, 04413 bool verts_too, 04414 int operation ) 04415 { 04416 std::vector< unsigned char > pstatus_vals( pstatus_ents.size() ); 04417 Range all_ents, *range_ptr = &pstatus_ents; 04418 ErrorCode result; 04419 if( lower_dim_ents || verts_too ) 04420 { 04421 all_ents = pstatus_ents; 04422 range_ptr = &all_ents; 04423 int start_dim = ( lower_dim_ents ? mbImpl->dimension_from_handle( *pstatus_ents.rbegin() ) - 1 : 0 ); 04424 for( ; start_dim >= 0; start_dim-- ) 04425 { 04426 result = mbImpl->get_adjacencies( all_ents, start_dim, true, all_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get adjacencies for pstatus entities" ); 04427 } 04428 } 04429 if( Interface::UNION == operation ) 04430 { 04431 result = mbImpl->tag_get_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 04432 for( unsigned int i = 0; i < pstatus_vals.size(); i++ ) 04433 pstatus_vals[i] |= pstatus_val; 04434 } 04435 else 04436 { 04437 for( unsigned int i = 0; i < pstatus_vals.size(); i++ ) 04438 pstatus_vals[i] = pstatus_val; 04439 } 04440 result = mbImpl->tag_set_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 04441 04442 return MB_SUCCESS; 04443 } 04444 04445 ErrorCode ParallelComm::set_pstatus_entities( EntityHandle* pstatus_ents, 04446 int num_ents, 04447 unsigned char pstatus_val, 04448 bool lower_dim_ents, 04449 bool verts_too, 04450 int operation ) 04451 { 04452 std::vector< unsigned char > pstatus_vals( num_ents ); 04453 ErrorCode result; 04454 if( lower_dim_ents || verts_too ) 04455 { 04456 // In this case, call the range-based version 04457 Range tmp_range; 04458 std::copy( pstatus_ents, pstatus_ents + num_ents, range_inserter( tmp_range ) ); 04459 return set_pstatus_entities( tmp_range, pstatus_val, lower_dim_ents, verts_too, operation ); 04460 } 04461 04462 if( Interface::UNION == operation ) 04463 { 04464 result = mbImpl->tag_get_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 04465 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ ) 04466 pstatus_vals[i] |= pstatus_val; 04467 } 04468 else 04469 { 04470 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ ) 04471 pstatus_vals[i] = pstatus_val; 04472 } 04473 result = mbImpl->tag_set_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 04474 04475 return MB_SUCCESS; 04476 } 04477 04478 static size_t choose_owner_idx( const std::vector< unsigned >& proc_list ) 04479 { 04480 // Try to assign owners randomly so we get a good distribution, 04481 // (note: specifying the same seed on all procs is essential) 04482 unsigned val = 0; 04483 for( size_t i = 0; i < proc_list.size(); i++ ) 04484 val ^= proc_list[i]; 04485 srand( (int)( val ) ); 04486 return rand() % proc_list.size(); 04487 } 04488 04489 struct set_tuple 04490 { 04491 unsigned idx; 04492 unsigned proc; 04493 EntityHandle handle; 04494 inline bool operator<( set_tuple other ) const 04495 { 04496 return ( idx == other.idx ) ? ( proc < other.proc ) : ( idx < other.idx ); 04497 } 04498 }; 04499 04500 ErrorCode ParallelComm::resolve_shared_sets( EntityHandle file, const Tag* idtag ) 04501 { 04502 // Find all sets with any of the following tags: 04503 const char* const shared_set_tag_names[] = { GEOM_DIMENSION_TAG_NAME, MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, 04504 NEUMANN_SET_TAG_NAME, PARALLEL_PARTITION_TAG_NAME }; 04505 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] ); 04506 Range candidate_sets; 04507 ErrorCode result = MB_FAILURE; 04508 04509 // If we're not given an ID tag to use to globally identify sets, 04510 // then fall back to using known tag values 04511 if( !idtag ) 04512 { 04513 Tag gid, tag; 04514 gid = mbImpl->globalId_tag(); 04515 if( NULL != gid ) result = mbImpl->tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tag ); 04516 if( MB_SUCCESS == result ) 04517 { 04518 for( int d = 0; d < 4; d++ ) 04519 { 04520 candidate_sets.clear(); 04521 const void* vals[] = { &d }; 04522 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, vals, 1, candidate_sets ); 04523 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, gid ); 04524 } 04525 } 04526 04527 for( int i = 1; i < num_tags; i++ ) 04528 { 04529 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag ); 04530 if( MB_SUCCESS == result ) 04531 { 04532 candidate_sets.clear(); 04533 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets ); 04534 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, tag ); 04535 } 04536 } 04537 04538 return MB_SUCCESS; 04539 } 04540 04541 for( int i = 0; i < num_tags; i++ ) 04542 { 04543 Tag tag; 04544 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag, MB_TAG_ANY ); 04545 if( MB_SUCCESS != result ) continue; 04546 04547 mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets, Interface::UNION ); 04548 } 04549 04550 // Find any additional sets that contain shared entities 04551 Range::iterator hint = candidate_sets.begin(); 04552 Range all_sets; 04553 mbImpl->get_entities_by_type( file, MBENTITYSET, all_sets ); 04554 all_sets = subtract( all_sets, candidate_sets ); 04555 Range::iterator it = all_sets.begin(); 04556 while( it != all_sets.end() ) 04557 { 04558 Range contents; 04559 mbImpl->get_entities_by_handle( *it, contents ); 04560 contents.erase( contents.lower_bound( MBENTITYSET ), contents.end() ); 04561 filter_pstatus( contents, PSTATUS_SHARED, PSTATUS_OR ); 04562 if( contents.empty() ) 04563 { 04564 ++it; 04565 } 04566 else 04567 { 04568 hint = candidate_sets.insert( hint, *it ); 04569 it = all_sets.erase( it ); 04570 } 04571 } 04572 04573 // Find any additionl sets that contain or are parents of potential shared sets 04574 Range prev_list = candidate_sets; 04575 while( !prev_list.empty() ) 04576 { 04577 it = all_sets.begin(); 04578 Range new_list; 04579 hint = new_list.begin(); 04580 while( it != all_sets.end() ) 04581 { 04582 Range contents; 04583 mbImpl->get_entities_by_type( *it, MBENTITYSET, contents ); 04584 if( !intersect( prev_list, contents ).empty() ) 04585 { 04586 hint = new_list.insert( hint, *it ); 04587 it = all_sets.erase( it ); 04588 } 04589 else 04590 { 04591 new_list.clear(); 04592 mbImpl->get_child_meshsets( *it, contents ); 04593 if( !intersect( prev_list, contents ).empty() ) 04594 { 04595 hint = new_list.insert( hint, *it ); 04596 it = all_sets.erase( it ); 04597 } 04598 else 04599 { 04600 ++it; 04601 } 04602 } 04603 } 04604 04605 candidate_sets.merge( new_list ); 04606 prev_list.swap( new_list ); 04607 } 04608 04609 return resolve_shared_sets( candidate_sets, *idtag ); 04610 } 04611 04612 #ifndef NDEBUG 04613 bool is_sorted_unique( std::vector< unsigned >& v ) 04614 { 04615 for( size_t i = 1; i < v.size(); i++ ) 04616 if( v[i - 1] >= v[i] ) return false; 04617 return true; 04618 } 04619 #endif 04620 04621 ErrorCode ParallelComm::resolve_shared_sets( Range& sets, Tag idtag ) 04622 { 04623 ErrorCode result; 04624 const unsigned rk = proc_config().proc_rank(); 04625 MPI_Comm cm = proc_config().proc_comm(); 04626 04627 // Build sharing list for all sets 04628 04629 // Get ids for sets in a vector, to pass to gs 04630 std::vector< long > larray; // Allocate sufficient space for longs 04631 std::vector< Ulong > handles; 04632 Range tmp_sets; 04633 // The id tag can be size 4 or size 8 04634 // Based on that, convert to int or to long, similarly to what we do 04635 // for resolving shared vertices; 04636 // This code must work on 32 bit too, where long is 4 bytes, also 04637 // so test first size 4, then we should be fine 04638 DataType tag_type; 04639 result = mbImpl->tag_get_data_type( idtag, tag_type );MB_CHK_SET_ERR( result, "Failed getting tag data type" ); 04640 int bytes_per_tag; 04641 result = mbImpl->tag_get_bytes( idtag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed getting number of bytes per tag" ); 04642 // On 64 bits, long and int are different 04643 // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?) 04644 04645 for( Range::iterator rit = sets.begin(); rit != sets.end(); ++rit ) 04646 { 04647 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) ) 04648 { // It is a special id tag 04649 long dum; 04650 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum ); 04651 if( MB_SUCCESS == result ) 04652 { 04653 larray.push_back( dum ); 04654 handles.push_back( *rit ); 04655 tmp_sets.insert( tmp_sets.end(), *rit ); 04656 } 04657 } 04658 else if( 4 == bytes_per_tag ) 04659 { // Must be GLOBAL_ID tag or MATERIAL_ID, etc 04660 int dum; 04661 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum ); 04662 if( MB_SUCCESS == result ) 04663 { 04664 larray.push_back( dum ); 04665 handles.push_back( *rit ); 04666 tmp_sets.insert( tmp_sets.end(), *rit ); 04667 } 04668 } 04669 } 04670 04671 const size_t nsets = handles.size(); 04672 04673 // Get handle array for sets 04674 // This is not true on windows machine, 64 bits: entity handle is 64 bit, long is 32 04675 // assert(sizeof(EntityHandle) <= sizeof(unsigned long)); 04676 04677 // Do communication of data 04678 gs_data::crystal_data* cd = procConfig.crystal_router(); 04679 gs_data* gsd = new gs_data(); 04680 result = gsd->initialize( nsets, &larray[0], &handles[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" ); 04681 04682 // Convert from global IDs grouped by process rank to list 04683 // of <idx, rank> pairs so that we can sort primarily 04684 // by idx and secondarily by rank (we want lists of procs for each 04685 // idx, not lists if indices for each proc). 04686 size_t ntuple = 0; 04687 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ ) 04688 ntuple += gsd->nlinfo->_nshared[p]; 04689 std::vector< set_tuple > tuples; 04690 tuples.reserve( ntuple ); 04691 size_t j = 0; 04692 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ ) 04693 { 04694 for( unsigned np = 0; np < gsd->nlinfo->_nshared[p]; np++ ) 04695 { 04696 set_tuple t; 04697 t.idx = gsd->nlinfo->_sh_ind[j]; 04698 t.proc = gsd->nlinfo->_target[p]; 04699 t.handle = gsd->nlinfo->_ulabels[j]; 04700 tuples.push_back( t ); 04701 j++; 04702 } 04703 } 04704 std::sort( tuples.begin(), tuples.end() ); 04705 04706 // Release crystal router stuff 04707 gsd->reset(); 04708 delete gsd; 04709 04710 // Storing sharing data for each set 04711 size_t ti = 0; 04712 unsigned idx = 0; 04713 std::vector< unsigned > procs; 04714 Range::iterator si = tmp_sets.begin(); 04715 while( si != tmp_sets.end() && ti < tuples.size() ) 04716 { 04717 assert( idx <= tuples[ti].idx ); 04718 if( idx < tuples[ti].idx ) si += ( tuples[ti].idx - idx ); 04719 idx = tuples[ti].idx; 04720 04721 procs.clear(); 04722 size_t ti_init = ti; 04723 while( ti < tuples.size() && tuples[ti].idx == idx ) 04724 { 04725 procs.push_back( tuples[ti].proc ); 04726 ++ti; 04727 } 04728 assert( is_sorted_unique( procs ) ); 04729 04730 result = sharedSetData->set_sharing_procs( *si, procs ); 04731 if( MB_SUCCESS != result ) 04732 { 04733 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl; 04734 std::cerr.flush(); 04735 MPI_Abort( cm, 1 ); 04736 } 04737 04738 // Add this proc to list of sharing procs in correct position 04739 // so that all procs select owner based on same list 04740 std::vector< unsigned >::iterator it = std::lower_bound( procs.begin(), procs.end(), rk ); 04741 assert( it == procs.end() || *it > rk ); 04742 procs.insert( it, rk ); 04743 size_t owner_idx = choose_owner_idx( procs ); 04744 EntityHandle owner_handle; 04745 if( procs[owner_idx] == rk ) 04746 owner_handle = *si; 04747 else if( procs[owner_idx] > rk ) 04748 owner_handle = tuples[ti_init + owner_idx - 1].handle; 04749 else 04750 owner_handle = tuples[ti_init + owner_idx].handle; 04751 result = sharedSetData->set_owner( *si, procs[owner_idx], owner_handle ); 04752 if( MB_SUCCESS != result ) 04753 { 04754 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl; 04755 std::cerr.flush(); 04756 MPI_Abort( cm, 1 ); 04757 } 04758 04759 ++si; 04760 ++idx; 04761 } 04762 04763 return MB_SUCCESS; 04764 } 04765 // populate sets with ghost entities, if necessary 04766 ErrorCode ParallelComm::augment_default_sets_with_ghosts( EntityHandle file_set ) 04767 { 04768 // gather all default sets we are interested in, material, neumann, etc 04769 // we will skip geometry sets, because they are not uniquely identified with their tag value 04770 // maybe we will add another tag, like category 04771 04772 if( procConfig.proc_size() < 2 ) return MB_SUCCESS; // no reason to stop by 04773 const char* const shared_set_tag_names[] = { MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, NEUMANN_SET_TAG_NAME, 04774 PARALLEL_PARTITION_TAG_NAME }; 04775 04776 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] ); 04777 04778 Range* rangeSets = new Range[num_tags]; 04779 Tag* tags = new Tag[num_tags + 1]; // one extra for global id tag, which is an int, so far 04780 04781 int my_rank = rank(); 04782 int** tagVals = new int*[num_tags]; 04783 for( int i = 0; i < num_tags; i++ ) 04784 tagVals[i] = NULL; 04785 ErrorCode rval; 04786 04787 // for each tag, we keep a local map, from the value to the actual set with that value 04788 // we assume that the tag values are unique, for a given set, otherwise we 04789 // do not know to which set to add the entity 04790 04791 typedef std::map< int, EntityHandle > MVal; 04792 typedef std::map< int, EntityHandle >::iterator itMVal; 04793 MVal* localMaps = new MVal[num_tags]; 04794 04795 for( int i = 0; i < num_tags; i++ ) 04796 { 04797 04798 rval = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tags[i], MB_TAG_ANY ); 04799 if( MB_SUCCESS != rval ) continue; 04800 rval = mbImpl->get_entities_by_type_and_tag( file_set, MBENTITYSET, &( tags[i] ), 0, 1, rangeSets[i], 04801 Interface::UNION );MB_CHK_SET_ERR( rval, "can't get sets with a tag" ); 04802 04803 if( rangeSets[i].size() > 0 ) 04804 { 04805 tagVals[i] = new int[rangeSets[i].size()]; 04806 // fill up with the tag values 04807 rval = mbImpl->tag_get_data( tags[i], rangeSets[i], tagVals[i] );MB_CHK_SET_ERR( rval, "can't get set tag values" ); 04808 // now for inverse mapping: 04809 for( int j = 0; j < (int)rangeSets[i].size(); j++ ) 04810 { 04811 localMaps[i][tagVals[i][j]] = rangeSets[i][j]; 04812 } 04813 } 04814 } 04815 // get the global id tag too 04816 tags[num_tags] = mbImpl->globalId_tag(); 04817 04818 TupleList remoteEnts; 04819 // processor to send to, type of tag (0-mat,) tag value, remote handle 04820 // 1-diri 04821 // 2-neum 04822 // 3-part 04823 // 04824 int initialSize = (int)sharedEnts.size(); // estimate that on average, each shared ent 04825 // will be sent to one processor, for one tag 04826 // we will actually send only entities that are owned locally, and from those 04827 // only those that do have a special tag (material, neumann, etc) 04828 // if we exceed the capacity, we resize the tuple 04829 remoteEnts.initialize( 3, 0, 1, 0, initialSize ); 04830 remoteEnts.enableWriteAccess(); 04831 04832 // now, for each owned entity, get the remote handle(s) and Proc(s), and verify if it 04833 // belongs to one of the sets; if yes, create a tuple and append it 04834 04835 std::set< EntityHandle > own_and_sha; 04836 int ir = 0, jr = 0; 04837 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 04838 { 04839 // ghosted eh 04840 EntityHandle geh = *vit; 04841 if( own_and_sha.find( geh ) != own_and_sha.end() ) // already encountered 04842 continue; 04843 int procs[MAX_SHARING_PROCS]; 04844 EntityHandle handles[MAX_SHARING_PROCS]; 04845 int nprocs; 04846 unsigned char pstat; 04847 rval = get_sharing_data( geh, procs, handles, pstat, nprocs ); 04848 if( rval != MB_SUCCESS ) 04849 { 04850 for( int i = 0; i < num_tags; i++ ) 04851 delete[] tagVals[i]; 04852 delete[] tagVals; 04853 04854 MB_CHK_SET_ERR( rval, "Failed to get sharing data" ); 04855 } 04856 if( pstat & PSTATUS_NOT_OWNED ) continue; // we will send info only for entities that we own 04857 own_and_sha.insert( geh ); 04858 for( int i = 0; i < num_tags; i++ ) 04859 { 04860 for( int j = 0; j < (int)rangeSets[i].size(); j++ ) 04861 { 04862 EntityHandle specialSet = rangeSets[i][j]; // this set has tag i, value tagVals[i][j]; 04863 if( mbImpl->contains_entities( specialSet, &geh, 1 ) ) 04864 { 04865 // this ghosted entity is in a special set, so form the tuple 04866 // to send to the processors that do not own this 04867 for( int k = 0; k < nprocs; k++ ) 04868 { 04869 if( procs[k] != my_rank ) 04870 { 04871 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 ) 04872 { 04873 // resize, so we do not overflow 04874 int oldSize = remoteEnts.get_max(); 04875 // increase with 50% the capacity 04876 remoteEnts.resize( oldSize + oldSize / 2 + 1 ); 04877 } 04878 remoteEnts.vi_wr[ir++] = procs[k]; // send to proc 04879 remoteEnts.vi_wr[ir++] = i; // for the tags [i] (0-3) 04880 remoteEnts.vi_wr[ir++] = tagVals[i][j]; // actual value of the tag 04881 remoteEnts.vul_wr[jr++] = handles[k]; 04882 remoteEnts.inc_n(); 04883 } 04884 } 04885 } 04886 } 04887 } 04888 // if the local entity has a global id, send it too, so we avoid 04889 // another "exchange_tags" for global id 04890 int gid; 04891 rval = mbImpl->tag_get_data( tags[num_tags], &geh, 1, &gid );MB_CHK_SET_ERR( rval, "Failed to get global id" ); 04892 if( gid != 0 ) 04893 { 04894 for( int k = 0; k < nprocs; k++ ) 04895 { 04896 if( procs[k] != my_rank ) 04897 { 04898 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 ) 04899 { 04900 // resize, so we do not overflow 04901 int oldSize = remoteEnts.get_max(); 04902 // increase with 50% the capacity 04903 remoteEnts.resize( oldSize + oldSize / 2 + 1 ); 04904 } 04905 remoteEnts.vi_wr[ir++] = procs[k]; // send to proc 04906 remoteEnts.vi_wr[ir++] = num_tags; // for the tags [j] (4) 04907 remoteEnts.vi_wr[ir++] = gid; // actual value of the tag 04908 remoteEnts.vul_wr[jr++] = handles[k]; 04909 remoteEnts.inc_n(); 04910 } 04911 } 04912 } 04913 } 04914 04915 #ifndef NDEBUG 04916 if( my_rank == 1 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 1, before augment routing" ); 04917 MPI_Barrier( procConfig.proc_comm() ); 04918 int sentEnts = remoteEnts.get_n(); 04919 assert( ( sentEnts == jr ) && ( 3 * sentEnts == ir ) ); 04920 #endif 04921 // exchange the info now, and send to 04922 gs_data::crystal_data* cd = this->procConfig.crystal_router(); 04923 // All communication happens here; no other mpi calls 04924 // Also, this is a collective call 04925 rval = cd->gs_transfer( 1, remoteEnts, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" ); 04926 #ifndef NDEBUG 04927 if( my_rank == 0 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 0, after augment routing" ); 04928 MPI_Barrier( procConfig.proc_comm() ); 04929 #endif 04930 04931 // now process the data received from other processors 04932 int received = remoteEnts.get_n(); 04933 for( int i = 0; i < received; i++ ) 04934 { 04935 // int from = ents_to_delete.vi_rd[i]; 04936 EntityHandle geh = (EntityHandle)remoteEnts.vul_rd[i]; 04937 int from_proc = remoteEnts.vi_rd[3 * i]; 04938 if( my_rank == from_proc ) 04939 std::cout << " unexpected receive from my rank " << my_rank << " during augmenting with ghosts\n "; 04940 int tag_type = remoteEnts.vi_rd[3 * i + 1]; 04941 assert( ( 0 <= tag_type ) && ( tag_type <= num_tags ) ); 04942 int value = remoteEnts.vi_rd[3 * i + 2]; 04943 if( tag_type == num_tags ) 04944 { 04945 // it is global id 04946 rval = mbImpl->tag_set_data( tags[num_tags], &geh, 1, &value );MB_CHK_SET_ERR( rval, "Error in setting gid tag" ); 04947 } 04948 else 04949 { 04950 // now, based on value and tag type, see if we have that value in the map 04951 MVal& lmap = localMaps[tag_type]; 04952 itMVal itm = lmap.find( value ); 04953 if( itm == lmap.end() ) 04954 { 04955 // the value was not found yet in the local map, so we have to create the set 04956 EntityHandle newSet; 04957 rval = mbImpl->create_meshset( MESHSET_SET, newSet );MB_CHK_SET_ERR( rval, "can't create new set" ); 04958 lmap[value] = newSet; 04959 // set the tag value 04960 rval = mbImpl->tag_set_data( tags[tag_type], &newSet, 1, &value );MB_CHK_SET_ERR( rval, "can't set tag for new set" ); 04961 04962 // we also need to add the new created set to the file set, if not null 04963 if( file_set ) 04964 { 04965 rval = mbImpl->add_entities( file_set, &newSet, 1 );MB_CHK_SET_ERR( rval, "can't add new set to the file set" ); 04966 } 04967 } 04968 // add the entity to the set pointed to by the map 04969 rval = mbImpl->add_entities( lmap[value], &geh, 1 );MB_CHK_SET_ERR( rval, "can't add ghost ent to the set" ); 04970 } 04971 } 04972 04973 for( int i = 0; i < num_tags; i++ ) 04974 delete[] tagVals[i]; 04975 delete[] tagVals; 04976 delete[] rangeSets; 04977 delete[] tags; 04978 delete[] localMaps; 04979 return MB_SUCCESS; 04980 } 04981 ErrorCode ParallelComm::create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim ) 04982 { 04983 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs; 04984 04985 // Build up the list of shared entities 04986 int procs[MAX_SHARING_PROCS]; 04987 EntityHandle handles[MAX_SHARING_PROCS]; 04988 ErrorCode result; 04989 int nprocs; 04990 unsigned char pstat; 04991 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 04992 { 04993 if( shared_dim != -1 && mbImpl->dimension_from_handle( *vit ) > shared_dim ) continue; 04994 result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 04995 std::sort( procs, procs + nprocs ); 04996 std::vector< int > tmp_procs( procs, procs + nprocs ); 04997 assert( tmp_procs.size() != 2 ); 04998 proc_nvecs[tmp_procs].push_back( *vit ); 04999 } 05000 05001 Skinner skinner( mbImpl ); 05002 Range skin_ents[4]; 05003 result = mbImpl->get_entities_by_dimension( this_set, resolve_dim, skin_ents[resolve_dim] );MB_CHK_SET_ERR( result, "Failed to get skin entities by dimension" ); 05004 result = 05005 skinner.find_skin( this_set, skin_ents[resolve_dim], false, skin_ents[resolve_dim - 1], 0, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" ); 05006 if( shared_dim > 1 ) 05007 { 05008 result = mbImpl->get_adjacencies( skin_ents[resolve_dim - 1], resolve_dim - 2, true, skin_ents[resolve_dim - 2], 05009 Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" ); 05010 } 05011 05012 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs ); 05013 05014 return create_interface_sets( proc_nvecs ); 05015 } 05016 05017 ErrorCode ParallelComm::create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs ) 05018 { 05019 if( proc_nvecs.empty() ) return MB_SUCCESS; 05020 05021 int proc_ids[MAX_SHARING_PROCS]; 05022 EntityHandle proc_handles[MAX_SHARING_PROCS]; 05023 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05024 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in create_interface_sets" ); 05025 Range::iterator rit; 05026 05027 // Create interface sets, tag them, and tag their contents with iface set tag 05028 std::vector< unsigned char > pstatus; 05029 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator vit = proc_nvecs.begin(); 05030 vit != proc_nvecs.end(); ++vit ) 05031 { 05032 // Create the set 05033 EntityHandle new_set; 05034 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" ); 05035 interfaceSets.insert( new_set ); 05036 05037 // Add entities 05038 assert( !vit->second.empty() ); 05039 result = mbImpl->add_entities( new_set, &( vit->second )[0], ( vit->second ).size() );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 05040 // Tag set with the proc rank(s) 05041 if( vit->first.size() == 1 ) 05042 { 05043 assert( ( vit->first )[0] != (int)procConfig.proc_rank() ); 05044 result = mbImpl->tag_set_data( shp_tag, &new_set, 1, &( vit->first )[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05045 proc_handles[0] = 0; 05046 result = mbImpl->tag_set_data( shh_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05047 } 05048 else 05049 { 05050 // Pad tag data out to MAX_SHARING_PROCS with -1 05051 if( vit->first.size() > MAX_SHARING_PROCS ) 05052 { 05053 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_set ) ) 05054 << ' ' << ID_FROM_HANDLE( new_set ) << " on process " << proc_config().proc_rank() 05055 << std::endl; 05056 std::cerr.flush(); 05057 MPI_Abort( proc_config().proc_comm(), 66 ); 05058 } 05059 // assert(vit->first.size() <= MAX_SHARING_PROCS); 05060 std::copy( vit->first.begin(), vit->first.end(), proc_ids ); 05061 std::fill( proc_ids + vit->first.size(), proc_ids + MAX_SHARING_PROCS, -1 ); 05062 result = mbImpl->tag_set_data( shps_tag, &new_set, 1, proc_ids );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05063 unsigned int ind = std::find( proc_ids, proc_ids + vit->first.size(), procConfig.proc_rank() ) - proc_ids; 05064 assert( ind < vit->first.size() ); 05065 std::fill( proc_handles, proc_handles + MAX_SHARING_PROCS, 0 ); 05066 proc_handles[ind] = new_set; 05067 result = mbImpl->tag_set_data( shhs_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05068 } 05069 05070 // Get the owning proc, then set the pstatus tag on iface set 05071 int min_proc = ( vit->first )[0]; 05072 unsigned char pval = ( PSTATUS_SHARED | PSTATUS_INTERFACE ); 05073 if( min_proc < (int)procConfig.proc_rank() ) pval |= PSTATUS_NOT_OWNED; 05074 if( vit->first.size() > 1 ) pval |= PSTATUS_MULTISHARED; 05075 result = mbImpl->tag_set_data( pstat_tag, &new_set, 1, &pval );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" ); 05076 05077 // Tag the vertices with the same thing 05078 pstatus.clear(); 05079 std::vector< EntityHandle > verts; 05080 for( std::vector< EntityHandle >::iterator v2it = ( vit->second ).begin(); v2it != ( vit->second ).end(); 05081 ++v2it ) 05082 if( mbImpl->type_from_handle( *v2it ) == MBVERTEX ) verts.push_back( *v2it ); 05083 pstatus.resize( verts.size(), pval ); 05084 if( !verts.empty() ) 05085 { 05086 result = mbImpl->tag_set_data( pstat_tag, &verts[0], verts.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set vertices with pstatus" ); 05087 } 05088 } 05089 05090 return MB_SUCCESS; 05091 } 05092 05093 ErrorCode ParallelComm::create_iface_pc_links() 05094 { 05095 // Now that we've resolved the entities in the iface sets, 05096 // set parent/child links between the iface sets 05097 05098 // First tag all entities in the iface sets 05099 Tag tmp_iface_tag; 05100 EntityHandle tmp_iface_set = 0; 05101 ErrorCode result = mbImpl->tag_get_handle( "__tmp_iface", 1, MB_TYPE_HANDLE, tmp_iface_tag, 05102 MB_TAG_DENSE | MB_TAG_CREAT, &tmp_iface_set );MB_CHK_SET_ERR( result, "Failed to create temporary interface set tag" ); 05103 05104 Range iface_ents; 05105 std::vector< EntityHandle > tag_vals; 05106 Range::iterator rit; 05107 05108 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 05109 { 05110 // tag entities with interface set 05111 iface_ents.clear(); 05112 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in interface set" ); 05113 05114 if( iface_ents.empty() ) continue; 05115 05116 tag_vals.resize( iface_ents.size() ); 05117 std::fill( tag_vals.begin(), tag_vals.end(), *rit ); 05118 result = mbImpl->tag_set_data( tmp_iface_tag, iface_ents, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to tag iface entities with interface set" ); 05119 } 05120 05121 // Now go back through interface sets and add parent/child links 05122 Range tmp_ents2; 05123 for( int d = 2; d >= 0; d-- ) 05124 { 05125 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 05126 { 05127 // Get entities on this interface 05128 iface_ents.clear(); 05129 result = mbImpl->get_entities_by_handle( *rit, iface_ents, true );MB_CHK_SET_ERR( result, "Failed to get entities by handle" ); 05130 if( iface_ents.empty() || mbImpl->dimension_from_handle( *iface_ents.rbegin() ) != d ) continue; 05131 05132 // Get higher-dimensional entities and their interface sets 05133 result = mbImpl->get_adjacencies( &( *iface_ents.begin() ), 1, d + 1, false, tmp_ents2 );MB_CHK_SET_ERR( result, "Failed to get adjacencies for interface sets" ); 05134 tag_vals.resize( tmp_ents2.size() ); 05135 result = mbImpl->tag_get_data( tmp_iface_tag, tmp_ents2, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to get tmp iface tag for interface sets" ); 05136 05137 // Go through and for any on interface make it a parent 05138 EntityHandle last_set = 0; 05139 for( unsigned int i = 0; i < tag_vals.size(); i++ ) 05140 { 05141 if( tag_vals[i] && tag_vals[i] != last_set ) 05142 { 05143 result = mbImpl->add_parent_child( tag_vals[i], *rit );MB_CHK_SET_ERR( result, "Failed to add parent/child link for interface set" ); 05144 last_set = tag_vals[i]; 05145 } 05146 } 05147 } 05148 } 05149 05150 // Delete the temporary tag 05151 result = mbImpl->tag_delete( tmp_iface_tag );MB_CHK_SET_ERR( result, "Failed to delete tmp iface tag" ); 05152 05153 return MB_SUCCESS; 05154 } 05155 05156 ErrorCode ParallelComm::get_proc_nvecs( int resolve_dim, 05157 int shared_dim, 05158 Range* skin_ents, 05159 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs ) 05160 { 05161 // Set sharing procs tags on other skin ents 05162 ErrorCode result; 05163 const EntityHandle* connect; 05164 int num_connect; 05165 std::set< int > sharing_procs; 05166 std::vector< EntityHandle > dum_connect; 05167 std::vector< int > sp_vec; 05168 05169 for( int d = 3; d > 0; d-- ) 05170 { 05171 if( resolve_dim == d ) continue; 05172 05173 for( Range::iterator rit = skin_ents[d].begin(); rit != skin_ents[d].end(); ++rit ) 05174 { 05175 // Get connectivity 05176 result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect );MB_CHK_SET_ERR( result, "Failed to get connectivity on non-vertex skin entities" ); 05177 05178 int op = ( resolve_dim < shared_dim ? Interface::UNION : Interface::INTERSECT ); 05179 result = get_sharing_data( connect, num_connect, sharing_procs, op );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_proc_nvecs" ); 05180 if( sharing_procs.empty() || 05181 ( sharing_procs.size() == 1 && *sharing_procs.begin() == (int)procConfig.proc_rank() ) ) 05182 continue; 05183 05184 // Need to specify sharing data correctly for entities or they will 05185 // end up in a different interface set than corresponding vertices 05186 if( sharing_procs.size() == 2 ) 05187 { 05188 std::set< int >::iterator it = sharing_procs.find( proc_config().proc_rank() ); 05189 assert( it != sharing_procs.end() ); 05190 sharing_procs.erase( it ); 05191 } 05192 05193 // Intersection is the owning proc(s) for this skin ent 05194 sp_vec.clear(); 05195 std::copy( sharing_procs.begin(), sharing_procs.end(), std::back_inserter( sp_vec ) ); 05196 assert( sp_vec.size() != 2 ); 05197 proc_nvecs[sp_vec].push_back( *rit ); 05198 } 05199 } 05200 05201 #ifndef NDEBUG 05202 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05203 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05204 mit != proc_nvecs.end(); ++mit ) 05205 { 05206 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05207 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05208 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05209 assert( vit == tmp_vec.end() ); 05210 } 05211 #endif 05212 05213 return MB_SUCCESS; 05214 } 05215 05216 // Overloaded form of tag_shared_verts 05217 // Tuple coming in is of form (arbitrary value, remoteProc, localHandle, remoteHandle) 05218 // Also will check for doubles in the list if the list is sorted 05219 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents, 05220 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs, 05221 Range& /*proc_verts*/, 05222 unsigned int i_extra ) 05223 { 05224 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05225 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" ); 05226 05227 unsigned int j = 0, i = 0; 05228 std::vector< int > sharing_procs, sharing_procs2, tag_procs; 05229 std::vector< EntityHandle > sharing_handles, sharing_handles2, tag_lhandles, tag_rhandles; 05230 std::vector< unsigned char > pstatus; 05231 05232 // Were on tuple j/2 05233 if( i_extra ) i += i_extra; 05234 while( j < 2 * shared_ents.get_n() ) 05235 { 05236 // Count & accumulate sharing procs 05237 EntityHandle this_ent = shared_ents.vul_rd[j], other_ent = 0; 05238 int other_proc = -1; 05239 while( j < 2 * shared_ents.get_n() && shared_ents.vul_rd[j] == this_ent ) 05240 { 05241 j++; 05242 // Shouldn't have same proc 05243 assert( shared_ents.vi_rd[i] != (int)procConfig.proc_rank() ); 05244 // Grab the remote data if its not a dublicate 05245 if( shared_ents.vul_rd[j] != other_ent || shared_ents.vi_rd[i] != other_proc ) 05246 { 05247 assert( 0 != shared_ents.vul_rd[j] ); 05248 sharing_procs.push_back( shared_ents.vi_rd[i] ); 05249 sharing_handles.push_back( shared_ents.vul_rd[j] ); 05250 } 05251 other_proc = shared_ents.vi_rd[i]; 05252 other_ent = shared_ents.vul_rd[j]; 05253 j++; 05254 i += 1 + i_extra; 05255 } 05256 05257 if( sharing_procs.size() > 1 ) 05258 { 05259 // Add current proc/handle to list 05260 sharing_procs.push_back( procConfig.proc_rank() ); 05261 sharing_handles.push_back( this_ent ); 05262 05263 // Sort sharing_procs and sharing_handles such that 05264 // sharing_procs is in ascending order. Use temporary 05265 // lists and binary search to re-order sharing_handles. 05266 sharing_procs2 = sharing_procs; 05267 std::sort( sharing_procs2.begin(), sharing_procs2.end() ); 05268 sharing_handles2.resize( sharing_handles.size() ); 05269 for( size_t k = 0; k < sharing_handles.size(); k++ ) 05270 { 05271 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) - 05272 sharing_procs2.begin(); 05273 sharing_handles2[idx] = sharing_handles[k]; 05274 } 05275 sharing_procs.swap( sharing_procs2 ); 05276 sharing_handles.swap( sharing_handles2 ); 05277 } 05278 05279 assert( sharing_procs.size() != 2 ); 05280 proc_nvecs[sharing_procs].push_back( this_ent ); 05281 05282 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED ); 05283 if( sharing_procs.size() == 1 ) 05284 { 05285 tag_procs.push_back( sharing_procs[0] ); 05286 tag_lhandles.push_back( this_ent ); 05287 tag_rhandles.push_back( sharing_handles[0] ); 05288 pstatus.push_back( share_flag ); 05289 } 05290 else 05291 { 05292 // Pad lists 05293 // assert(sharing_procs.size() <= MAX_SHARING_PROCS); 05294 if( sharing_procs.size() > MAX_SHARING_PROCS ) 05295 { 05296 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process " 05297 << proc_config().proc_rank() << std::endl; 05298 std::cerr.flush(); 05299 MPI_Abort( proc_config().proc_comm(), 66 ); 05300 } 05301 sharing_procs.resize( MAX_SHARING_PROCS, -1 ); 05302 sharing_handles.resize( MAX_SHARING_PROCS, 0 ); 05303 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" ); 05304 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" ); 05305 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05306 sharedEnts.insert( this_ent ); 05307 } 05308 05309 // Reset sharing proc(s) tags 05310 sharing_procs.clear(); 05311 sharing_handles.clear(); 05312 } 05313 05314 if( !tag_procs.empty() ) 05315 { 05316 result = mbImpl->tag_set_data( shp_tag, &tag_lhandles[0], tag_procs.size(), &tag_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" ); 05317 result = mbImpl->tag_set_data( shh_tag, &tag_lhandles[0], tag_procs.size(), &tag_rhandles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" ); 05318 result = mbImpl->tag_set_data( pstat_tag, &tag_lhandles[0], tag_procs.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05319 for( std::vector< EntityHandle >::iterator vvt = tag_lhandles.begin(); vvt != tag_lhandles.end(); vvt++ ) 05320 sharedEnts.insert( *vvt ); 05321 } 05322 05323 #ifndef NDEBUG 05324 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05325 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05326 mit != proc_nvecs.end(); ++mit ) 05327 { 05328 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05329 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05330 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05331 assert( vit == tmp_vec.end() ); 05332 } 05333 #endif 05334 05335 return MB_SUCCESS; 05336 } 05337 05338 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents, 05339 Range* skin_ents, 05340 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs, 05341 Range& /*proc_verts*/ ) 05342 { 05343 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05344 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" ); 05345 05346 unsigned int j = 0, i = 0; 05347 std::vector< int > sharing_procs, sharing_procs2; 05348 std::vector< EntityHandle > sharing_handles, sharing_handles2, skin_verts( skin_ents[0].size() ); 05349 for( Range::iterator rit = skin_ents[0].begin(); rit != skin_ents[0].end(); ++rit, i++ ) 05350 skin_verts[i] = *rit; 05351 i = 0; 05352 05353 while( j < 2 * shared_ents.get_n() ) 05354 { 05355 // Count & accumulate sharing procs 05356 int this_idx = shared_ents.vi_rd[j]; 05357 EntityHandle this_ent = skin_verts[this_idx]; 05358 while( j < 2 * shared_ents.get_n() && shared_ents.vi_rd[j] == this_idx ) 05359 { 05360 j++; 05361 // Shouldn't have same proc 05362 assert( shared_ents.vi_rd[j] != (int)procConfig.proc_rank() ); 05363 sharing_procs.push_back( shared_ents.vi_rd[j++] ); 05364 sharing_handles.push_back( shared_ents.vul_rd[i++] ); 05365 } 05366 05367 if( sharing_procs.size() > 1 ) 05368 { 05369 // Add current proc/handle to list 05370 sharing_procs.push_back( procConfig.proc_rank() ); 05371 sharing_handles.push_back( this_ent ); 05372 } 05373 05374 // Sort sharing_procs and sharing_handles such that 05375 // sharing_procs is in ascending order. Use temporary 05376 // lists and binary search to re-order sharing_handles. 05377 sharing_procs2 = sharing_procs; 05378 std::sort( sharing_procs2.begin(), sharing_procs2.end() ); 05379 sharing_handles2.resize( sharing_handles.size() ); 05380 for( size_t k = 0; k < sharing_handles.size(); k++ ) 05381 { 05382 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) - 05383 sharing_procs2.begin(); 05384 sharing_handles2[idx] = sharing_handles[k]; 05385 } 05386 sharing_procs.swap( sharing_procs2 ); 05387 sharing_handles.swap( sharing_handles2 ); 05388 05389 assert( sharing_procs.size() != 2 ); 05390 proc_nvecs[sharing_procs].push_back( this_ent ); 05391 05392 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED ); 05393 if( sharing_procs.size() == 1 ) 05394 { 05395 result = mbImpl->tag_set_data( shp_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" ); 05396 result = mbImpl->tag_set_data( shh_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" ); 05397 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &share_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05398 sharedEnts.insert( this_ent ); 05399 } 05400 else 05401 { 05402 // Pad lists 05403 // assert(sharing_procs.size() <= MAX_SHARING_PROCS); 05404 if( sharing_procs.size() > MAX_SHARING_PROCS ) 05405 { 05406 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process " 05407 << proc_config().proc_rank() << std::endl; 05408 std::cerr.flush(); 05409 MPI_Abort( proc_config().proc_comm(), 66 ); 05410 } 05411 sharing_procs.resize( MAX_SHARING_PROCS, -1 ); 05412 sharing_handles.resize( MAX_SHARING_PROCS, 0 ); 05413 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" ); 05414 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" ); 05415 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05416 sharedEnts.insert( this_ent ); 05417 } 05418 05419 // Reset sharing proc(s) tags 05420 sharing_procs.clear(); 05421 sharing_handles.clear(); 05422 } 05423 05424 #ifndef NDEBUG 05425 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05426 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05427 mit != proc_nvecs.end(); ++mit ) 05428 { 05429 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05430 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05431 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05432 assert( vit == tmp_vec.end() ); 05433 } 05434 #endif 05435 05436 return MB_SUCCESS; 05437 } 05438 05439 //! Get processors with which this processor communicates; sets are sorted by processor 05440 ErrorCode ParallelComm::get_interface_procs( std::set< unsigned int >& procs_set, bool get_buffs ) 05441 { 05442 // Make sure the sharing procs vector is empty 05443 procs_set.clear(); 05444 05445 // Pre-load vector of single-proc tag values 05446 unsigned int i, j; 05447 std::vector< int > iface_proc( interfaceSets.size() ); 05448 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), interfaceSets, &iface_proc[0] );MB_CHK_SET_ERR( result, "Failed to get iface_proc for iface sets" ); 05449 05450 // Get sharing procs either from single-proc vector or by getting 05451 // multi-proc tag value 05452 int tmp_iface_procs[MAX_SHARING_PROCS]; 05453 std::fill( tmp_iface_procs, tmp_iface_procs + MAX_SHARING_PROCS, -1 ); 05454 Range::iterator rit; 05455 for( rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); ++rit, i++ ) 05456 { 05457 if( -1 != iface_proc[i] ) 05458 { 05459 assert( iface_proc[i] != (int)procConfig.proc_rank() ); 05460 procs_set.insert( (unsigned int)iface_proc[i] ); 05461 } 05462 else 05463 { 05464 // Get the sharing_procs tag 05465 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, tmp_iface_procs );MB_CHK_SET_ERR( result, "Failed to get iface_procs for iface set" ); 05466 for( j = 0; j < MAX_SHARING_PROCS; j++ ) 05467 { 05468 if( -1 != tmp_iface_procs[j] && tmp_iface_procs[j] != (int)procConfig.proc_rank() ) 05469 procs_set.insert( (unsigned int)tmp_iface_procs[j] ); 05470 else if( -1 == tmp_iface_procs[j] ) 05471 { 05472 std::fill( tmp_iface_procs, tmp_iface_procs + j, -1 ); 05473 break; 05474 } 05475 } 05476 } 05477 } 05478 05479 if( get_buffs ) 05480 { 05481 for( std::set< unsigned int >::iterator sit = procs_set.begin(); sit != procs_set.end(); ++sit ) 05482 get_buffers( *sit ); 05483 } 05484 05485 return MB_SUCCESS; 05486 } 05487 05488 ErrorCode ParallelComm::get_pstatus( EntityHandle entity, unsigned char& pstatus_val ) 05489 { 05490 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstatus_val );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" ); 05491 return result; 05492 } 05493 05494 ErrorCode ParallelComm::get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents ) 05495 { 05496 Range ents; 05497 ErrorCode result; 05498 05499 if( -1 == dim ) 05500 { 05501 result = mbImpl->get_entities_by_handle( 0, ents );MB_CHK_SET_ERR( result, "Failed to get all entities" ); 05502 } 05503 else 05504 { 05505 result = mbImpl->get_entities_by_dimension( 0, dim, ents );MB_CHK_SET_ERR( result, "Failed to get entities of dimension " << dim ); 05506 } 05507 05508 std::vector< unsigned char > pstatus( ents.size() ); 05509 result = mbImpl->tag_get_data( pstatus_tag(), ents, &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" ); 05510 Range::iterator rit = ents.begin(); 05511 int i = 0; 05512 if( pstatus_val ) 05513 { 05514 for( ; rit != ents.end(); i++, ++rit ) 05515 { 05516 if( pstatus[i] & pstatus_val && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) ) 05517 pstatus_ents.insert( *rit ); 05518 } 05519 } 05520 else 05521 { 05522 for( ; rit != ents.end(); i++, ++rit ) 05523 { 05524 if( !pstatus[i] && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) ) 05525 pstatus_ents.insert( *rit ); 05526 } 05527 } 05528 05529 return MB_SUCCESS; 05530 } 05531 05532 ErrorCode ParallelComm::check_global_ids( EntityHandle this_set, 05533 const int dimension, 05534 const int start_id, 05535 const bool largest_dim_only, 05536 const bool parallel, 05537 const bool owned_only ) 05538 { 05539 // Global id tag 05540 Tag gid_tag = mbImpl->globalId_tag(); 05541 int def_val = -1; 05542 Range dum_range; 05543 05544 void* tag_ptr = &def_val; 05545 ErrorCode result = mbImpl->get_entities_by_type_and_tag( this_set, MBVERTEX, &gid_tag, &tag_ptr, 1, dum_range );MB_CHK_SET_ERR( result, "Failed to get entities by MBVERTEX type and gid tag" ); 05546 05547 if( !dum_range.empty() ) 05548 { 05549 // Just created it, so we need global ids 05550 result = assign_global_ids( this_set, dimension, start_id, largest_dim_only, parallel, owned_only );MB_CHK_SET_ERR( result, "Failed assigning global ids" ); 05551 } 05552 05553 return MB_SUCCESS; 05554 } 05555 05556 bool ParallelComm::is_iface_proc( EntityHandle this_set, int to_proc ) 05557 { 05558 int sharing_procs[MAX_SHARING_PROCS]; 05559 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 ); 05560 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &this_set, 1, sharing_procs ); 05561 if( MB_SUCCESS == result && to_proc == sharing_procs[0] ) return true; 05562 05563 result = mbImpl->tag_get_data( sharedps_tag(), &this_set, 1, sharing_procs ); 05564 if( MB_SUCCESS != result ) return false; 05565 05566 for( int i = 0; i < MAX_SHARING_PROCS; i++ ) 05567 { 05568 if( to_proc == sharing_procs[i] ) 05569 return true; 05570 else if( -1 == sharing_procs[i] ) 05571 return false; 05572 } 05573 05574 return false; 05575 } 05576 05577 ErrorCode ParallelComm::filter_pstatus( Range& ents, 05578 unsigned char pstat, 05579 unsigned char op, 05580 int to_proc, 05581 Range* returned_ents ) 05582 { 05583 Range tmp_ents; 05584 05585 // assert(!ents.empty()); 05586 if( ents.empty() ) 05587 { 05588 if( returned_ents ) returned_ents->clear(); 05589 return MB_SUCCESS; 05590 } 05591 05592 // Put into tmp_ents any entities which are not owned locally or 05593 // who are already shared with to_proc 05594 std::vector< unsigned char > shared_flags( ents.size() ), shared_flags2; 05595 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), ents, &shared_flags[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus flag" ); 05596 Range::const_iterator rit, hint = tmp_ents.begin(); 05597 ; 05598 int i; 05599 if( op == PSTATUS_OR ) 05600 { 05601 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05602 { 05603 if( ( ( shared_flags[i] & ~pstat ) ^ shared_flags[i] ) & pstat ) 05604 { 05605 hint = tmp_ents.insert( hint, *rit ); 05606 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05607 } 05608 } 05609 } 05610 else if( op == PSTATUS_AND ) 05611 { 05612 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05613 { 05614 if( ( shared_flags[i] & pstat ) == pstat ) 05615 { 05616 hint = tmp_ents.insert( hint, *rit ); 05617 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05618 } 05619 } 05620 } 05621 else if( op == PSTATUS_NOT ) 05622 { 05623 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05624 { 05625 if( !( shared_flags[i] & pstat ) ) 05626 { 05627 hint = tmp_ents.insert( hint, *rit ); 05628 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05629 } 05630 } 05631 } 05632 else 05633 { 05634 assert( false ); 05635 return MB_FAILURE; 05636 } 05637 05638 if( -1 != to_proc ) 05639 { 05640 int sharing_procs[MAX_SHARING_PROCS]; 05641 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 ); 05642 Range tmp_ents2; 05643 hint = tmp_ents2.begin(); 05644 05645 for( rit = tmp_ents.begin(), i = 0; rit != tmp_ents.end(); ++rit, i++ ) 05646 { 05647 // We need to check sharing procs 05648 if( shared_flags2[i] & PSTATUS_MULTISHARED ) 05649 { 05650 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag" ); 05651 assert( -1 != sharing_procs[0] ); 05652 for( unsigned int j = 0; j < MAX_SHARING_PROCS; j++ ) 05653 { 05654 // If to_proc shares this entity, add it to list 05655 if( sharing_procs[j] == to_proc ) 05656 { 05657 hint = tmp_ents2.insert( hint, *rit ); 05658 } 05659 else if( -1 == sharing_procs[j] ) 05660 break; 05661 05662 sharing_procs[j] = -1; 05663 } 05664 } 05665 else if( shared_flags2[i] & PSTATUS_SHARED ) 05666 { 05667 result = mbImpl->tag_get_data( sharedp_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag" ); 05668 assert( -1 != sharing_procs[0] ); 05669 if( sharing_procs[0] == to_proc ) hint = tmp_ents2.insert( hint, *rit ); 05670 sharing_procs[0] = -1; 05671 } 05672 else 05673 assert( "should never get here" && false ); 05674 } 05675 05676 tmp_ents.swap( tmp_ents2 ); 05677 } 05678 05679 if( returned_ents ) 05680 returned_ents->swap( tmp_ents ); 05681 else 05682 ents.swap( tmp_ents ); 05683 05684 return MB_SUCCESS; 05685 } 05686 05687 ErrorCode ParallelComm::exchange_ghost_cells( int ghost_dim, 05688 int bridge_dim, 05689 int num_layers, 05690 int addl_ents, 05691 bool store_remote_handles, 05692 bool wait_all, 05693 EntityHandle* file_set ) 05694 { 05695 #ifdef MOAB_HAVE_MPE 05696 if( myDebug->get_verbosity() == 2 ) 05697 { 05698 if( !num_layers ) 05699 MPE_Log_event( IFACE_START, procConfig.proc_rank(), "Starting interface exchange." ); 05700 else 05701 MPE_Log_event( GHOST_START, procConfig.proc_rank(), "Starting ghost exchange." ); 05702 } 05703 #endif 05704 05705 myDebug->tprintf( 1, "Entering exchange_ghost_cells with num_layers = %d\n", num_layers ); 05706 if( myDebug->get_verbosity() == 4 ) 05707 { 05708 msgs.clear(); 05709 msgs.reserve( MAX_SHARING_PROCS ); 05710 } 05711 05712 // If we're only finding out about existing ents, we have to be storing 05713 // remote handles too 05714 assert( num_layers > 0 || store_remote_handles ); 05715 05716 const bool is_iface = !num_layers; 05717 05718 // Get the b-dimensional interface(s) with with_proc, where b = bridge_dim 05719 05720 int success; 05721 ErrorCode result = MB_SUCCESS; 05722 int incoming1 = 0, incoming2 = 0; 05723 05724 reset_all_buffers(); 05725 05726 // When this function is called, buffProcs should already have any 05727 // communicating procs 05728 05729 //=========================================== 05730 // Post ghost irecv's for ghost entities from all communicating procs 05731 //=========================================== 05732 #ifdef MOAB_HAVE_MPE 05733 if( myDebug->get_verbosity() == 2 ) 05734 { 05735 MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." ); 05736 } 05737 #endif 05738 05739 // Index reqs the same as buffer/sharing procs indices 05740 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ), 05741 recv_remoteh_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05742 std::vector< unsigned int >::iterator proc_it; 05743 int ind, p; 05744 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05745 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 05746 { 05747 incoming1++; 05748 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 05749 MB_MESG_ENTS_SIZE, incoming1 ); 05750 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 05751 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] ); 05752 if( success != MPI_SUCCESS ) 05753 { 05754 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 05755 } 05756 } 05757 05758 //=========================================== 05759 // Get entities to be sent to neighbors 05760 //=========================================== 05761 Range sent_ents[MAX_SHARING_PROCS], allsent, tmp_range; 05762 TupleList entprocs; 05763 int dum_ack_buff; 05764 result = get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents, allsent, entprocs );MB_CHK_SET_ERR( result, "get_sent_ents failed" ); 05765 05766 // augment file set with the entities to be sent 05767 // we might have created new entities if addl_ents>0, edges and/or faces 05768 if( addl_ents > 0 && file_set && !allsent.empty() ) 05769 { 05770 result = mbImpl->add_entities( *file_set, allsent );MB_CHK_SET_ERR( result, "Failed to add new sub-entities to set" ); 05771 } 05772 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(), 05773 (unsigned long)allsent.size() ); 05774 05775 //=========================================== 05776 // Pack and send ents from this proc to others 05777 //=========================================== 05778 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ ) 05779 { 05780 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", sent_ents[p].compactness(), 05781 (unsigned long)sent_ents[p].size() ); 05782 05783 // Reserve space on front for size and for initial buff size 05784 localOwnedBuffs[p]->reset_buffer( sizeof( int ) ); 05785 05786 // Entities 05787 result = pack_entities( sent_ents[p], localOwnedBuffs[p], store_remote_handles, buffProcs[p], is_iface, 05788 &entprocs, &allsent );MB_CHK_SET_ERR( result, "Packing entities failed" ); 05789 05790 if( myDebug->get_verbosity() == 4 ) 05791 { 05792 msgs.resize( msgs.size() + 1 ); 05793 msgs.back() = new Buffer( *localOwnedBuffs[p] ); 05794 } 05795 05796 // Send the buffer (size stored in front in send_buffer) 05797 result = send_buffer( *proc_it, localOwnedBuffs[p], MB_MESG_ENTS_SIZE, sendReqs[3 * p], 05798 recv_ent_reqs[3 * p + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE, 05799 ( !is_iface && store_remote_handles ? // this used for ghosting only 05800 localOwnedBuffs[p] 05801 : NULL ), 05802 &recv_remoteh_reqs[3 * p], &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" ); 05803 } 05804 05805 entprocs.reset(); 05806 05807 //=========================================== 05808 // Receive/unpack new entities 05809 //=========================================== 05810 // Number of incoming messages for ghosts is the number of procs we 05811 // communicate with; for iface, it's the number of those with lower rank 05812 MPI_Status status; 05813 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() ); 05814 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() ); 05815 std::vector< std::vector< int > > L1p( buffProcs.size() ); 05816 std::vector< EntityHandle > L2hloc, L2hrem; 05817 std::vector< unsigned int > L2p; 05818 std::vector< EntityHandle > new_ents; 05819 05820 while( incoming1 ) 05821 { 05822 // Wait for all recvs of ghost ents before proceeding to sending remote handles, 05823 // b/c some procs may have sent to a 3rd proc ents owned by me; 05824 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 05825 05826 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status ); 05827 if( MPI_SUCCESS != success ) 05828 { 05829 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 05830 } 05831 05832 PRINT_DEBUG_RECD( status ); 05833 05834 // OK, received something; decrement incoming counter 05835 incoming1--; 05836 bool done = false; 05837 05838 // In case ind is for ack, we need index of one before it 05839 unsigned int base_ind = 3 * ( ind / 3 ); 05840 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1], 05841 recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1], 05842 sendReqs[base_ind + 2], done, 05843 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ), 05844 MB_MESG_REMOTEH_SIZE, // maybe base_ind+1? 05845 &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 05846 05847 if( done ) 05848 { 05849 if( myDebug->get_verbosity() == 4 ) 05850 { 05851 msgs.resize( msgs.size() + 1 ); 05852 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] ); 05853 } 05854 05855 // Message completely received - process buffer that was sent 05856 remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 05857 result = unpack_entities( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, ind / 3, is_iface, 05858 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents ); 05859 if( MB_SUCCESS != result ) 05860 { 05861 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl; 05862 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false ); 05863 return result; 05864 } 05865 05866 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 05867 { 05868 // Post irecv's for remote handles from new proc; shouldn't be iface, 05869 // since we know about all procs we share with 05870 assert( !is_iface ); 05871 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05872 for( unsigned int i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 ) 05873 { 05874 localOwnedBuffs[i / 3]->reset_buffer(); 05875 incoming2++; 05876 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr, 05877 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 ); 05878 success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, 05879 buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(), 05880 &recv_remoteh_reqs[i] ); 05881 if( success != MPI_SUCCESS ) 05882 { 05883 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" ); 05884 } 05885 } 05886 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05887 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05888 } 05889 } 05890 } 05891 05892 // Add requests for any new addl procs 05893 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 05894 { 05895 // Shouldn't get here... 05896 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in ghost exchange" ); 05897 } 05898 05899 #ifdef MOAB_HAVE_MPE 05900 if( myDebug->get_verbosity() == 2 ) 05901 { 05902 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." ); 05903 } 05904 #endif 05905 05906 if( is_iface ) 05907 { 05908 // Need to check over entities I sent and make sure I received 05909 // handles for them from all expected procs; if not, need to clean 05910 // them up 05911 result = check_clean_iface( allsent ); 05912 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05913 05914 // Now set the shared/interface tag on non-vertex entities on interface 05915 result = tag_iface_entities();MB_CHK_SET_ERR( result, "Failed to tag iface entities" ); 05916 05917 #ifndef NDEBUG 05918 result = check_sent_ents( allsent ); 05919 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05920 result = check_all_shared_handles( true ); 05921 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05922 #endif 05923 05924 #ifdef MOAB_HAVE_MPE 05925 if( myDebug->get_verbosity() == 2 ) 05926 { 05927 MPE_Log_event( IFACE_END, procConfig.proc_rank(), "Ending interface exchange." ); 05928 } 05929 #endif 05930 05931 //=========================================== 05932 // Wait if requested 05933 //=========================================== 05934 if( wait_all ) 05935 { 05936 if( myDebug->get_verbosity() == 5 ) 05937 { 05938 success = MPI_Barrier( procConfig.proc_comm() ); 05939 } 05940 else 05941 { 05942 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 05943 success = MPI_Waitall( 3 * buffProcs.size(), &recv_ent_reqs[0], mult_status ); 05944 if( MPI_SUCCESS != success ) 05945 { 05946 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05947 } 05948 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 05949 if( MPI_SUCCESS != success ) 05950 { 05951 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05952 } 05953 /*success = MPI_Waitall(3*buffProcs.size(), &recv_remoteh_reqs[0], mult_status); 05954 if (MPI_SUCCESS != success) { 05955 MB_SET_ERR(MB_FAILURE, "Failed in waitall in ghost exchange"); 05956 }*/ 05957 } 05958 } 05959 05960 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 05961 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==true \n" ); 05962 05963 return MB_SUCCESS; 05964 } 05965 05966 // we still need to wait on sendReqs, if they are not fulfilled yet 05967 if( wait_all ) 05968 { 05969 if( myDebug->get_verbosity() == 5 ) 05970 { 05971 success = MPI_Barrier( procConfig.proc_comm() ); 05972 } 05973 else 05974 { 05975 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 05976 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 05977 if( MPI_SUCCESS != success ) 05978 { 05979 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05980 } 05981 } 05982 } 05983 //=========================================== 05984 // Send local handles for new ghosts to owner, then add 05985 // those to ghost list for that owner 05986 //=========================================== 05987 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ ) 05988 { 05989 05990 // Reserve space on front for size and for initial buff size 05991 remoteOwnedBuffs[p]->reset_buffer( sizeof( int ) ); 05992 05993 result = pack_remote_handles( L1hloc[p], L1hrem[p], L1p[p], *proc_it, remoteOwnedBuffs[p] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 05994 remoteOwnedBuffs[p]->set_stored_size(); 05995 05996 if( myDebug->get_verbosity() == 4 ) 05997 { 05998 msgs.resize( msgs.size() + 1 ); 05999 msgs.back() = new Buffer( *remoteOwnedBuffs[p] ); 06000 } 06001 result = send_buffer( buffProcs[p], remoteOwnedBuffs[p], MB_MESG_REMOTEH_SIZE, sendReqs[3 * p], 06002 recv_remoteh_reqs[3 * p + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 06003 } 06004 06005 //=========================================== 06006 // Process remote handles of my ghosteds 06007 //=========================================== 06008 while( incoming2 ) 06009 { 06010 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 06011 success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status ); 06012 if( MPI_SUCCESS != success ) 06013 { 06014 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 06015 } 06016 06017 // OK, received something; decrement incoming counter 06018 incoming2--; 06019 06020 PRINT_DEBUG_RECD( status ); 06021 06022 bool done = false; 06023 unsigned int base_ind = 3 * ( ind / 3 ); 06024 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1], 06025 recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3], 06026 sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 06027 if( done ) 06028 { 06029 // Incoming remote handles 06030 if( myDebug->get_verbosity() == 4 ) 06031 { 06032 msgs.resize( msgs.size() + 1 ); 06033 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] ); 06034 } 06035 localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 06036 result = 06037 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 06038 } 06039 } 06040 06041 #ifdef MOAB_HAVE_MPE 06042 if( myDebug->get_verbosity() == 2 ) 06043 { 06044 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." ); 06045 MPE_Log_event( GHOST_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." ); 06046 } 06047 #endif 06048 06049 //=========================================== 06050 // Wait if requested 06051 //=========================================== 06052 if( wait_all ) 06053 { 06054 if( myDebug->get_verbosity() == 5 ) 06055 { 06056 success = MPI_Barrier( procConfig.proc_comm() ); 06057 } 06058 else 06059 { 06060 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 06061 success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status ); 06062 if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 06063 } 06064 if( MPI_SUCCESS != success ) 06065 { 06066 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 06067 } 06068 } 06069 06070 #ifndef NDEBUG 06071 result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" ); 06072 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Failed check on all shared handles" ); 06073 #endif 06074 06075 if( file_set && !new_ents.empty() ) 06076 { 06077 result = mbImpl->add_entities( *file_set, &new_ents[0], new_ents.size() );MB_CHK_SET_ERR( result, "Failed to add new entities to set" ); 06078 } 06079 06080 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 06081 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==false \n" ); 06082 06083 return MB_SUCCESS; 06084 } 06085 06086 ErrorCode ParallelComm::send_buffer( const unsigned int to_proc, 06087 Buffer* send_buff, 06088 int mesg_tag, 06089 MPI_Request& send_req, 06090 MPI_Request& ack_req, 06091 int* ack_buff, 06092 int& this_incoming, 06093 int next_mesg_tag, 06094 Buffer* next_recv_buff, 06095 MPI_Request* next_recv_req, 06096 int* next_incoming ) 06097 { 06098 ErrorCode result = MB_SUCCESS; 06099 int success; 06100 06101 // If small message, post recv for remote handle message 06102 if( send_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE && next_recv_buff ) 06103 { 06104 ( *next_incoming )++; 06105 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, next_mesg_tag, 06106 *next_incoming ); 06107 success = MPI_Irecv( next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, to_proc, next_mesg_tag, 06108 procConfig.proc_comm(), next_recv_req ); 06109 if( success != MPI_SUCCESS ) 06110 { 06111 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for next message in ghost exchange" ); 06112 } 06113 } 06114 // If large, we'll need an ack before sending the rest 06115 else if( send_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE ) 06116 { 06117 this_incoming++; 06118 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, (unsigned char*)ack_buff, sizeof( int ), mesg_tag - 1, 06119 this_incoming ); 06120 success = MPI_Irecv( (void*)ack_buff, sizeof( int ), MPI_UNSIGNED_CHAR, to_proc, mesg_tag - 1, 06121 procConfig.proc_comm(), &ack_req ); 06122 if( success != MPI_SUCCESS ) 06123 { 06124 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for entity ack in ghost exchange" ); 06125 } 06126 } 06127 06128 // Send the buffer 06129 PRINT_DEBUG_ISEND( procConfig.proc_rank(), to_proc, send_buff->mem_ptr, mesg_tag, 06130 std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ) ); 06131 assert( 0 <= send_buff->get_stored_size() && send_buff->get_stored_size() <= (int)send_buff->alloc_size ); 06132 success = MPI_Isend( send_buff->mem_ptr, std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ), 06133 MPI_UNSIGNED_CHAR, to_proc, mesg_tag, procConfig.proc_comm(), &send_req ); 06134 if( success != MPI_SUCCESS ) return MB_FAILURE; 06135 06136 return result; 06137 } 06138 06139 ErrorCode ParallelComm::recv_buffer( int mesg_tag_expected, 06140 const MPI_Status& mpi_status, 06141 Buffer* recv_buff, 06142 MPI_Request& recv_req, 06143 MPI_Request& /*ack_recvd_req*/, 06144 int& this_incoming, 06145 Buffer* send_buff, 06146 MPI_Request& send_req, 06147 MPI_Request& sent_ack_req, 06148 bool& done, 06149 Buffer* next_buff, 06150 int next_tag, 06151 MPI_Request* next_req, 06152 int* next_incoming ) 06153 { 06154 // Process a received message; if there will be more coming, 06155 // post a receive for 2nd part then send an ack message 06156 int from_proc = mpi_status.MPI_SOURCE; 06157 int success; 06158 06159 // Set the buff_ptr on the recv_buffer; needs to point beyond any 06160 // valid data already in the buffer 06161 recv_buff->reset_ptr( std::min( recv_buff->get_stored_size(), (int)recv_buff->alloc_size ) ); 06162 06163 if( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE ) 06164 { 06165 // 1st message & large - allocate buffer, post irecv for 2nd message, 06166 // then send ack 06167 recv_buff->reserve( recv_buff->get_stored_size() ); 06168 assert( recv_buff->alloc_size > INITIAL_BUFF_SIZE ); 06169 06170 // Will expect a 2nd message 06171 this_incoming++; 06172 06173 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr + INITIAL_BUFF_SIZE, 06174 recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, mesg_tag_expected + 1, this_incoming ); 06175 success = MPI_Irecv( recv_buff->mem_ptr + INITIAL_BUFF_SIZE, recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, 06176 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &recv_req ); 06177 if( success != MPI_SUCCESS ) 06178 { 06179 MB_SET_ERR( MB_FAILURE, "Failed to post 2nd iRecv in ghost exchange" ); 06180 } 06181 06182 // Send ack, doesn't matter what data actually is 06183 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr, mesg_tag_expected - 1, 06184 sizeof( int ) ); 06185 success = MPI_Isend( recv_buff->mem_ptr, sizeof( int ), MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected - 1, 06186 procConfig.proc_comm(), &sent_ack_req ); 06187 if( success != MPI_SUCCESS ) 06188 { 06189 MB_SET_ERR( MB_FAILURE, "Failed to send ack in ghost exchange" ); 06190 } 06191 } 06192 else if( mpi_status.MPI_TAG == mesg_tag_expected - 1 ) 06193 { 06194 // Got an ack back, send the 2nd half of message 06195 06196 // Should be a large message if we got this 06197 assert( *( (size_t*)send_buff->mem_ptr ) > INITIAL_BUFF_SIZE ); 06198 06199 // Post irecv for next message, then send 2nd message 06200 if( next_buff ) 06201 { 06202 // We'll expect a return message 06203 ( *next_incoming )++; 06204 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, next_buff->mem_ptr, INITIAL_BUFF_SIZE, next_tag, 06205 *next_incoming ); 06206 06207 success = MPI_Irecv( next_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, next_tag, 06208 procConfig.proc_comm(), next_req ); 06209 if( success != MPI_SUCCESS ) 06210 { 06211 MB_SET_ERR( MB_FAILURE, "Failed to post next irecv in ghost exchange" ); 06212 } 06213 } 06214 06215 // Send 2nd message 06216 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, send_buff->mem_ptr + INITIAL_BUFF_SIZE, 06217 mesg_tag_expected + 1, send_buff->get_stored_size() - INITIAL_BUFF_SIZE ); 06218 06219 assert( send_buff->get_stored_size() - INITIAL_BUFF_SIZE < send_buff->alloc_size && 06220 0 <= send_buff->get_stored_size() ); 06221 success = MPI_Isend( send_buff->mem_ptr + INITIAL_BUFF_SIZE, send_buff->get_stored_size() - INITIAL_BUFF_SIZE, 06222 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &send_req ); 06223 if( success != MPI_SUCCESS ) 06224 { 06225 MB_SET_ERR( MB_FAILURE, "Failed to send 2nd message in ghost exchange" ); 06226 } 06227 } 06228 else if( ( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE ) || 06229 mpi_status.MPI_TAG == mesg_tag_expected + 1 ) 06230 { 06231 // Message completely received - signal that we're done 06232 done = true; 06233 } 06234 06235 return MB_SUCCESS; 06236 } 06237 06238 struct ProcList 06239 { 06240 int procs[MAX_SHARING_PROCS]; 06241 }; 06242 static bool operator<( const ProcList& a, const ProcList& b ) 06243 { 06244 for( int i = 0; i < MAX_SHARING_PROCS; i++ ) 06245 { 06246 if( a.procs[i] < b.procs[i] ) 06247 return true; 06248 else if( b.procs[i] < a.procs[i] ) 06249 return false; 06250 else if( a.procs[i] < 0 ) 06251 return false; 06252 } 06253 return false; 06254 } 06255 06256 ErrorCode ParallelComm::check_clean_iface( Range& allsent ) 06257 { 06258 // allsent is all entities I think are on interface; go over them, looking 06259 // for zero-valued handles, and fix any I find 06260 06261 // Keep lists of entities for which teh sharing data changed, grouped 06262 // by set of sharing procs. 06263 typedef std::map< ProcList, Range > procmap_t; 06264 procmap_t old_procs, new_procs; 06265 06266 ErrorCode result = MB_SUCCESS; 06267 Range::iterator rit; 06268 Range::reverse_iterator rvit; 06269 unsigned char pstatus; 06270 int nump; 06271 ProcList sharedp; 06272 EntityHandle sharedh[MAX_SHARING_PROCS]; 06273 for( rvit = allsent.rbegin(); rvit != allsent.rend(); ++rvit ) 06274 { 06275 result = get_sharing_data( *rvit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 06276 assert( "Should be shared with at least one other proc" && 06277 ( nump > 1 || sharedp.procs[0] != (int)procConfig.proc_rank() ) ); 06278 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 ); 06279 06280 // Look for first null handle in list 06281 int idx = std::find( sharedh, sharedh + nump, (EntityHandle)0 ) - sharedh; 06282 if( idx == nump ) continue; // All handles are valid 06283 06284 ProcList old_list( sharedp ); 06285 std::sort( old_list.procs, old_list.procs + nump ); 06286 old_procs[old_list].insert( *rvit ); 06287 06288 // Remove null handles and corresponding proc ranks from lists 06289 int new_nump = idx; 06290 bool removed_owner = !idx; 06291 for( ++idx; idx < nump; ++idx ) 06292 { 06293 if( sharedh[idx] ) 06294 { 06295 sharedh[new_nump] = sharedh[idx]; 06296 sharedp.procs[new_nump] = sharedp.procs[idx]; 06297 ++new_nump; 06298 } 06299 } 06300 sharedp.procs[new_nump] = -1; 06301 06302 if( removed_owner && new_nump > 1 ) 06303 { 06304 // The proc that we choose as the entity owner isn't sharing the 06305 // entity (doesn't have a copy of it). We need to pick a different 06306 // owner. Choose the proc with lowest rank. 06307 idx = std::min_element( sharedp.procs, sharedp.procs + new_nump ) - sharedp.procs; 06308 std::swap( sharedp.procs[0], sharedp.procs[idx] ); 06309 std::swap( sharedh[0], sharedh[idx] ); 06310 if( sharedp.procs[0] == (int)proc_config().proc_rank() ) pstatus &= ~PSTATUS_NOT_OWNED; 06311 } 06312 06313 result = set_sharing_data( *rvit, pstatus, nump, new_nump, sharedp.procs, sharedh );MB_CHK_SET_ERR( result, "Failed to set sharing data in check_clean_iface" ); 06314 06315 if( new_nump > 1 ) 06316 { 06317 if( new_nump == 2 ) 06318 { 06319 if( sharedp.procs[1] != (int)proc_config().proc_rank() ) 06320 { 06321 assert( sharedp.procs[0] == (int)proc_config().proc_rank() ); 06322 sharedp.procs[0] = sharedp.procs[1]; 06323 } 06324 sharedp.procs[1] = -1; 06325 } 06326 else 06327 { 06328 std::sort( sharedp.procs, sharedp.procs + new_nump ); 06329 } 06330 new_procs[sharedp].insert( *rvit ); 06331 } 06332 } 06333 06334 if( old_procs.empty() ) 06335 { 06336 assert( new_procs.empty() ); 06337 return MB_SUCCESS; 06338 } 06339 06340 // Update interface sets 06341 procmap_t::iterator pmit; 06342 // std::vector<unsigned char> pstatus_list; 06343 rit = interface_sets().begin(); 06344 while( rit != interface_sets().end() ) 06345 { 06346 result = get_sharing_data( *rit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data for interface set" ); 06347 assert( nump != 2 ); 06348 std::sort( sharedp.procs, sharedp.procs + nump ); 06349 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 ); 06350 06351 pmit = old_procs.find( sharedp ); 06352 if( pmit != old_procs.end() ) 06353 { 06354 result = mbImpl->remove_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" ); 06355 } 06356 06357 pmit = new_procs.find( sharedp ); 06358 if( pmit == new_procs.end() ) 06359 { 06360 int count; 06361 result = mbImpl->get_number_entities_by_handle( *rit, count );MB_CHK_SET_ERR( result, "Failed to get number of entities in interface set" ); 06362 if( !count ) 06363 { 06364 result = mbImpl->delete_entities( &*rit, 1 );MB_CHK_SET_ERR( result, "Failed to delete entities from interface set" ); 06365 rit = interface_sets().erase( rit ); 06366 } 06367 else 06368 { 06369 ++rit; 06370 } 06371 } 06372 else 06373 { 06374 result = mbImpl->add_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 06375 06376 // Remove those that we've processed so that we know which ones 06377 // are new. 06378 new_procs.erase( pmit ); 06379 ++rit; 06380 } 06381 } 06382 06383 // Create interface sets for new proc id combinations 06384 std::fill( sharedh, sharedh + MAX_SHARING_PROCS, 0 ); 06385 for( pmit = new_procs.begin(); pmit != new_procs.end(); ++pmit ) 06386 { 06387 EntityHandle new_set; 06388 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" ); 06389 interfaceSets.insert( new_set ); 06390 06391 // Add entities 06392 result = mbImpl->add_entities( new_set, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 06393 // Tag set with the proc rank(s) 06394 assert( pmit->first.procs[0] >= 0 ); 06395 pstatus = PSTATUS_SHARED | PSTATUS_INTERFACE; 06396 if( pmit->first.procs[1] == -1 ) 06397 { 06398 int other = pmit->first.procs[0]; 06399 assert( other != (int)procConfig.proc_rank() ); 06400 result = mbImpl->tag_set_data( sharedp_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06401 sharedh[0] = 0; 06402 result = mbImpl->tag_set_data( sharedh_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06403 if( other < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED; 06404 } 06405 else 06406 { 06407 result = mbImpl->tag_set_data( sharedps_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06408 result = mbImpl->tag_set_data( sharedhs_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06409 pstatus |= PSTATUS_MULTISHARED; 06410 if( pmit->first.procs[0] < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED; 06411 } 06412 06413 result = mbImpl->tag_set_data( pstatus_tag(), &new_set, 1, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" ); 06414 06415 // Set pstatus on all interface entities in set 06416 result = mbImpl->tag_clear_data( pstatus_tag(), pmit->second, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface entities with pstatus" ); 06417 } 06418 06419 return MB_SUCCESS; 06420 } 06421 06422 ErrorCode ParallelComm::set_sharing_data( EntityHandle ent, 06423 unsigned char pstatus, 06424 int old_nump, 06425 int new_nump, 06426 int* ps, 06427 EntityHandle* hs ) 06428 { 06429 // If new nump is less than 3, the entity is no longer mutishared 06430 if( old_nump > 2 && ( pstatus & PSTATUS_MULTISHARED ) && new_nump < 3 ) 06431 { 06432 // Unset multishared flag 06433 pstatus ^= PSTATUS_MULTISHARED; 06434 } 06435 06436 // Check for consistency in input data 06437 // DBG 06438 /* bool con1 = ((new_nump == 2 && pstatus&PSTATUS_SHARED && !(pstatus&PSTATUS_MULTISHARED)) || 06439 (new_nump > 2 && pstatus&PSTATUS_SHARED && pstatus&PSTATUS_MULTISHARED)); bool con2 = 06440 (!(pstatus&PSTATUS_GHOST) || pstatus&PSTATUS_SHARED); bool con3 = (new_nump < 3 || 06441 (pstatus&PSTATUS_NOT_OWNED && ps[0] != (int)rank()) || (!(pstatus&PSTATUS_NOT_OWNED) && ps[0] 06442 == (int)rank())); std::cout<<"current rank = "<<rank()<<std::endl; std::cout<<"condition 06443 1::"<<con1<<std::endl; std::cout<<"condition 2::"<<con2<<std::endl; std::cout<<"condition 06444 3::"<<con3<<std::endl;*/ 06445 06446 // DBG 06447 06448 assert( new_nump > 1 && 06449 ( ( new_nump == 2 && pstatus & PSTATUS_SHARED && 06450 !( pstatus & PSTATUS_MULTISHARED ) ) || // If <= 2 must not be multishared 06451 ( new_nump > 2 && pstatus & PSTATUS_SHARED && 06452 pstatus & PSTATUS_MULTISHARED ) ) && // If > 2 procs, must be multishared 06453 ( !( pstatus & PSTATUS_GHOST ) || pstatus & PSTATUS_SHARED ) && // If ghost, it must also be shared 06454 ( new_nump < 3 || 06455 ( pstatus & PSTATUS_NOT_OWNED && ps[0] != (int)rank() ) || // I'm not owner and first proc not me 06456 ( !( pstatus & PSTATUS_NOT_OWNED ) && ps[0] == (int)rank() ) ) // I'm owner and first proc is me 06457 ); 06458 06459 #ifndef NDEBUG 06460 { 06461 // Check for duplicates in proc list 06462 std::set< unsigned int > dumprocs; 06463 int dp = 0; 06464 for( ; dp < old_nump && -1 != ps[dp]; dp++ ) 06465 dumprocs.insert( ps[dp] ); 06466 assert( dp == (int)dumprocs.size() ); 06467 } 06468 #endif 06469 06470 ErrorCode result; 06471 // Reset any old data that needs to be 06472 if( old_nump > 2 && new_nump < 3 ) 06473 { 06474 // Need to remove multishared tags 06475 result = mbImpl->tag_delete_data( sharedps_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:1" ); 06476 result = mbImpl->tag_delete_data( sharedhs_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:2" ); 06477 // if (new_nump < 2) 06478 // pstatus = 0x0; 06479 // else if (ps[0] != (int)proc_config().proc_rank()) 06480 // pstatus |= PSTATUS_NOT_OWNED; 06481 } 06482 else if( ( old_nump < 3 && new_nump > 2 ) || ( old_nump > 1 && new_nump == 1 ) ) 06483 { 06484 // Reset sharedp and sharedh tags 06485 int tmp_p = -1; 06486 EntityHandle tmp_h = 0; 06487 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, &tmp_p );MB_CHK_SET_ERR( result, "set_sharing_data:3" ); 06488 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, &tmp_h );MB_CHK_SET_ERR( result, "set_sharing_data:4" ); 06489 } 06490 06491 assert( "check for multishared/owner I'm first proc" && 06492 ( !( pstatus & PSTATUS_MULTISHARED ) || ( pstatus & ( PSTATUS_NOT_OWNED | PSTATUS_GHOST ) ) || 06493 ( ps[0] == (int)rank() ) ) && 06494 "interface entities should have > 1 proc" && ( !( pstatus & PSTATUS_INTERFACE ) || new_nump > 1 ) && 06495 "ghost entities should have > 1 proc" && ( !( pstatus & PSTATUS_GHOST ) || new_nump > 1 ) ); 06496 06497 // Now set new data 06498 if( new_nump > 2 ) 06499 { 06500 result = mbImpl->tag_set_data( sharedps_tag(), &ent, 1, ps );MB_CHK_SET_ERR( result, "set_sharing_data:5" ); 06501 result = mbImpl->tag_set_data( sharedhs_tag(), &ent, 1, hs );MB_CHK_SET_ERR( result, "set_sharing_data:6" ); 06502 } 06503 else 06504 { 06505 unsigned int j = ( ps[0] == (int)procConfig.proc_rank() ? 1 : 0 ); 06506 assert( -1 != ps[j] ); 06507 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, ps + j );MB_CHK_SET_ERR( result, "set_sharing_data:7" ); 06508 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, hs + j );MB_CHK_SET_ERR( result, "set_sharing_data:8" ); 06509 } 06510 06511 result = mbImpl->tag_set_data( pstatus_tag(), &ent, 1, &pstatus );MB_CHK_SET_ERR( result, "set_sharing_data:9" ); 06512 06513 if( old_nump > 1 && new_nump < 2 ) sharedEnts.erase( ent ); 06514 06515 return result; 06516 } 06517 06518 ErrorCode ParallelComm::get_sent_ents( const bool is_iface, 06519 const int bridge_dim, 06520 const int ghost_dim, 06521 const int num_layers, 06522 const int addl_ents, 06523 Range* sent_ents, 06524 Range& allsent, 06525 TupleList& entprocs ) 06526 { 06527 ErrorCode result; 06528 unsigned int ind; 06529 std::vector< unsigned int >::iterator proc_it; 06530 Range tmp_range; 06531 06532 // Done in a separate loop over procs because sometimes later procs 06533 // need to add info to earlier procs' messages 06534 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 06535 { 06536 if( !is_iface ) 06537 { 06538 result = 06539 get_ghosted_entities( bridge_dim, ghost_dim, buffProcs[ind], num_layers, addl_ents, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get ghost layers" ); 06540 } 06541 else 06542 { 06543 result = get_iface_entities( buffProcs[ind], -1, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get interface layers" ); 06544 } 06545 06546 // Filter out entities already shared with destination 06547 tmp_range.clear(); 06548 result = filter_pstatus( sent_ents[ind], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 06549 if( !tmp_range.empty() ) sent_ents[ind] = subtract( sent_ents[ind], tmp_range ); 06550 06551 allsent.merge( sent_ents[ind] ); 06552 } 06553 06554 //=========================================== 06555 // Need to get procs each entity is sent to 06556 //=========================================== 06557 06558 // Get the total # of proc/handle pairs 06559 int npairs = 0; 06560 for( ind = 0; ind < buffProcs.size(); ind++ ) 06561 npairs += sent_ents[ind].size(); 06562 06563 // Allocate a TupleList of that size 06564 entprocs.initialize( 1, 0, 1, 0, npairs ); 06565 entprocs.enableWriteAccess(); 06566 06567 // Put the proc/handle pairs in the list 06568 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 06569 { 06570 for( Range::iterator rit = sent_ents[ind].begin(); rit != sent_ents[ind].end(); ++rit ) 06571 { 06572 entprocs.vi_wr[entprocs.get_n()] = *proc_it; 06573 entprocs.vul_wr[entprocs.get_n()] = *rit; 06574 entprocs.inc_n(); 06575 } 06576 } 06577 // Sort by handle 06578 moab::TupleList::buffer sort_buffer; 06579 sort_buffer.buffer_init( npairs ); 06580 entprocs.sort( 1, &sort_buffer ); 06581 06582 entprocs.disableWriteAccess(); 06583 sort_buffer.reset(); 06584 06585 return MB_SUCCESS; 06586 } 06587 06588 ErrorCode ParallelComm::exchange_ghost_cells( ParallelComm** pcs, 06589 unsigned int num_procs, 06590 int ghost_dim, 06591 int bridge_dim, 06592 int num_layers, 06593 int addl_ents, 06594 bool store_remote_handles, 06595 EntityHandle* file_sets ) 06596 { 06597 // Static version of function, exchanging info through buffers rather 06598 // than through messages 06599 06600 // If we're only finding out about existing ents, we have to be storing 06601 // remote handles too 06602 assert( num_layers > 0 || store_remote_handles ); 06603 06604 const bool is_iface = !num_layers; 06605 06606 unsigned int ind; 06607 ParallelComm* pc; 06608 ErrorCode result = MB_SUCCESS; 06609 06610 std::vector< Error* > ehs( num_procs ); 06611 for( unsigned int i = 0; i < num_procs; i++ ) 06612 { 06613 result = pcs[i]->get_moab()->query_interface( ehs[i] ); 06614 assert( MB_SUCCESS == result ); 06615 } 06616 06617 // When this function is called, buffProcs should already have any 06618 // communicating procs 06619 06620 //=========================================== 06621 // Get entities to be sent to neighbors 06622 //=========================================== 06623 06624 // Done in a separate loop over procs because sometimes later procs 06625 // need to add info to earlier procs' messages 06626 Range sent_ents[MAX_SHARING_PROCS][MAX_SHARING_PROCS], allsent[MAX_SHARING_PROCS]; 06627 06628 //=========================================== 06629 // Get entities to be sent to neighbors 06630 //=========================================== 06631 TupleList entprocs[MAX_SHARING_PROCS]; 06632 for( unsigned int p = 0; p < num_procs; p++ ) 06633 { 06634 pc = pcs[p]; 06635 result = pc->get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents[p], allsent[p], 06636 entprocs[p] );MB_CHK_SET_ERR( result, "p = " << p << ", get_sent_ents failed" ); 06637 06638 //=========================================== 06639 // Pack entities into buffers 06640 //=========================================== 06641 for( ind = 0; ind < pc->buffProcs.size(); ind++ ) 06642 { 06643 // Entities 06644 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06645 result = pc->pack_entities( sent_ents[p][ind], pc->localOwnedBuffs[ind], store_remote_handles, 06646 pc->buffProcs[ind], is_iface, &entprocs[p], &allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", packing entities failed" ); 06647 } 06648 06649 entprocs[p].reset(); 06650 } 06651 06652 //=========================================== 06653 // Receive/unpack new entities 06654 //=========================================== 06655 // Number of incoming messages for ghosts is the number of procs we 06656 // communicate with; for iface, it's the number of those with lower rank 06657 std::vector< std::vector< EntityHandle > > L1hloc[MAX_SHARING_PROCS], L1hrem[MAX_SHARING_PROCS]; 06658 std::vector< std::vector< int > > L1p[MAX_SHARING_PROCS]; 06659 std::vector< EntityHandle > L2hloc[MAX_SHARING_PROCS], L2hrem[MAX_SHARING_PROCS]; 06660 std::vector< unsigned int > L2p[MAX_SHARING_PROCS]; 06661 std::vector< EntityHandle > new_ents[MAX_SHARING_PROCS]; 06662 06663 for( unsigned int p = 0; p < num_procs; p++ ) 06664 { 06665 L1hloc[p].resize( pcs[p]->buffProcs.size() ); 06666 L1hrem[p].resize( pcs[p]->buffProcs.size() ); 06667 L1p[p].resize( pcs[p]->buffProcs.size() ); 06668 } 06669 06670 for( unsigned int p = 0; p < num_procs; p++ ) 06671 { 06672 pc = pcs[p]; 06673 06674 for( ind = 0; ind < pc->buffProcs.size(); ind++ ) 06675 { 06676 // Incoming ghost entities; unpack; returns entities received 06677 // both from sending proc and from owning proc (which may be different) 06678 06679 // Buffer could be empty, which means there isn't any message to 06680 // unpack (due to this comm proc getting added as a result of indirect 06681 // communication); just skip this unpack 06682 if( pc->localOwnedBuffs[ind]->get_stored_size() == 0 ) continue; 06683 06684 unsigned int to_p = pc->buffProcs[ind]; 06685 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06686 result = pcs[to_p]->unpack_entities( pc->localOwnedBuffs[ind]->buff_ptr, store_remote_handles, ind, 06687 is_iface, L1hloc[to_p], L1hrem[to_p], L1p[to_p], L2hloc[to_p], 06688 L2hrem[to_p], L2p[to_p], new_ents[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack entities" ); 06689 } 06690 } 06691 06692 if( is_iface ) 06693 { 06694 // Need to check over entities I sent and make sure I received 06695 // handles for them from all expected procs; if not, need to clean 06696 // them up 06697 for( unsigned int p = 0; p < num_procs; p++ ) 06698 { 06699 result = pcs[p]->check_clean_iface( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06700 } 06701 06702 #ifndef NDEBUG 06703 for( unsigned int p = 0; p < num_procs; p++ ) 06704 { 06705 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06706 } 06707 result = check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" ); 06708 #endif 06709 return MB_SUCCESS; 06710 } 06711 06712 //=========================================== 06713 // Send local handles for new ghosts to owner, then add 06714 // those to ghost list for that owner 06715 //=========================================== 06716 std::vector< unsigned int >::iterator proc_it; 06717 for( unsigned int p = 0; p < num_procs; p++ ) 06718 { 06719 pc = pcs[p]; 06720 06721 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ ) 06722 { 06723 // Skip if iface layer and higher-rank proc 06724 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06725 result = pc->pack_remote_handles( L1hloc[p][ind], L1hrem[p][ind], L1p[p][ind], *proc_it, 06726 pc->localOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to pack remote handles" ); 06727 } 06728 } 06729 06730 //=========================================== 06731 // Process remote handles of my ghosteds 06732 //=========================================== 06733 for( unsigned int p = 0; p < num_procs; p++ ) 06734 { 06735 pc = pcs[p]; 06736 06737 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ ) 06738 { 06739 // Incoming remote handles 06740 unsigned int to_p = pc->buffProcs[ind]; 06741 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06742 result = pcs[to_p]->unpack_remote_handles( p, pc->localOwnedBuffs[ind]->buff_ptr, L2hloc[to_p], 06743 L2hrem[to_p], L2p[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack remote handles" ); 06744 } 06745 } 06746 06747 #ifndef NDEBUG 06748 for( unsigned int p = 0; p < num_procs; p++ ) 06749 { 06750 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06751 } 06752 06753 result = ParallelComm::check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" ); 06754 #endif 06755 06756 if( file_sets ) 06757 { 06758 for( unsigned int p = 0; p < num_procs; p++ ) 06759 { 06760 if( new_ents[p].empty() ) continue; 06761 result = pcs[p]->get_moab()->add_entities( file_sets[p], &new_ents[p][0], new_ents[p].size() );MB_CHK_SET_ERR( result, "p = " << p << ", failed to add new entities to set" ); 06762 } 06763 } 06764 06765 return MB_SUCCESS; 06766 } 06767 06768 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& exchange_procs ) 06769 { 06770 // Set buffers 06771 int n_proc = exchange_procs.size(); 06772 for( int i = 0; i < n_proc; i++ ) 06773 get_buffers( exchange_procs[i] ); 06774 reset_all_buffers(); 06775 06776 // Post ghost irecv's for entities from all communicating procs 06777 // Index requests the same as buffer/sharing procs indices 06778 int success; 06779 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06780 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06781 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06782 06783 int incoming = 0; 06784 for( int i = 0; i < n_proc; i++ ) 06785 { 06786 int ind = get_buffers( exchange_procs[i] ); 06787 incoming++; 06788 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 06789 MB_MESG_ENTS_SIZE, incoming ); 06790 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 06791 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] ); 06792 if( success != MPI_SUCCESS ) 06793 { 06794 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" ); 06795 } 06796 } 06797 06798 return MB_SUCCESS; 06799 } 06800 06801 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs ) 06802 { 06803 // Set buffers 06804 int num = shared_procs.size(); 06805 for( int i = 0; i < num; i++ ) 06806 get_buffers( shared_procs[i] ); 06807 reset_all_buffers(); 06808 num = remoteOwnedBuffs.size(); 06809 for( int i = 0; i < num; i++ ) 06810 remoteOwnedBuffs[i]->set_stored_size(); 06811 num = localOwnedBuffs.size(); 06812 for( int i = 0; i < num; i++ ) 06813 localOwnedBuffs[i]->set_stored_size(); 06814 06815 // Post ghost irecv's for entities from all communicating procs 06816 // Index requests the same as buffer/sharing procs indices 06817 int success; 06818 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06819 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06820 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06821 06822 int incoming = 0; 06823 std::set< unsigned int >::iterator it = recv_procs.begin(); 06824 std::set< unsigned int >::iterator eit = recv_procs.end(); 06825 for( ; it != eit; ++it ) 06826 { 06827 int ind = get_buffers( *it ); 06828 incoming++; 06829 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 06830 MB_MESG_ENTS_SIZE, incoming ); 06831 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 06832 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] ); 06833 if( success != MPI_SUCCESS ) 06834 { 06835 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" ); 06836 } 06837 } 06838 06839 return MB_SUCCESS; 06840 } 06841 06842 ErrorCode ParallelComm::exchange_owned_meshs( std::vector< unsigned int >& exchange_procs, 06843 std::vector< Range* >& exchange_ents, 06844 std::vector< MPI_Request >& recv_ent_reqs, 06845 std::vector< MPI_Request >& recv_remoteh_reqs, 06846 bool store_remote_handles, 06847 bool wait_all, 06848 bool migrate, 06849 int dim ) 06850 { 06851 // Filter out entities already shared with destination 06852 // Exchange twice for entities and sets 06853 ErrorCode result; 06854 std::vector< unsigned int > exchange_procs_sets; 06855 std::vector< Range* > exchange_sets; 06856 int n_proc = exchange_procs.size(); 06857 for( int i = 0; i < n_proc; i++ ) 06858 { 06859 Range set_range = exchange_ents[i]->subset_by_type( MBENTITYSET ); 06860 *exchange_ents[i] = subtract( *exchange_ents[i], set_range ); 06861 Range* tmp_range = new Range( set_range ); 06862 exchange_sets.push_back( tmp_range ); 06863 exchange_procs_sets.push_back( exchange_procs[i] ); 06864 } 06865 06866 if( dim == 2 ) 06867 { 06868 // Exchange entities first 06869 result = exchange_owned_mesh( exchange_procs, exchange_ents, recvReqs, recvRemotehReqs, true, 06870 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" ); 06871