Mesh Oriented datABase
(version 5.4.1)
Array-based unstructured mesh datastructure
|
00001 #include "moab/Interface.hpp" 00002 #include "moab/ParallelComm.hpp" 00003 #include "moab/WriteUtilIface.hpp" 00004 #include "moab/ReadUtilIface.hpp" 00005 #include "SequenceManager.hpp" 00006 #include "moab/Error.hpp" 00007 #include "EntitySequence.hpp" 00008 #include "MBTagConventions.hpp" 00009 #include "moab/Skinner.hpp" 00010 #include "MBParallelConventions.h" 00011 #include "moab/Core.hpp" 00012 #include "ElementSequence.hpp" 00013 #include "moab/CN.hpp" 00014 #include "moab/RangeMap.hpp" 00015 #include "moab/MeshTopoUtil.hpp" 00016 #include "TagInfo.hpp" 00017 #include "DebugOutput.hpp" 00018 #include "SharedSetData.hpp" 00019 #include "moab/ScdInterface.hpp" 00020 #include "moab/TupleList.hpp" 00021 #include "moab/gs.hpp" 00022 00023 #include <iostream> 00024 #include <sstream> 00025 #include <algorithm> 00026 #include <functional> 00027 #include <numeric> 00028 00029 #include <cmath> 00030 #include <cstdlib> 00031 #include <cassert> 00032 00033 #ifdef MOAB_HAVE_MPI 00034 #include "moab_mpi.h" 00035 #endif 00036 #ifdef MOAB_HAVE_MPE 00037 #include "mpe.h" 00038 int IFACE_START, IFACE_END; 00039 int GHOST_START, GHOST_END; 00040 int SHAREDV_START, SHAREDV_END; 00041 int RESOLVE_START, RESOLVE_END; 00042 int ENTITIES_START, ENTITIES_END; 00043 int RHANDLES_START, RHANDLES_END; 00044 int OWNED_START, OWNED_END; 00045 #endif 00046 00047 namespace moab 00048 { 00049 00050 const unsigned int ParallelComm::INITIAL_BUFF_SIZE = 1024; 00051 00052 const int MAX_BCAST_SIZE = ( 1 << 28 ); 00053 00054 std::vector< ParallelComm::Buffer* > msgs; 00055 unsigned int __PACK_num = 0, __UNPACK_num = 0, __PACK_count = 0, __UNPACK_count = 0; 00056 std::string __PACK_string, __UNPACK_string; 00057 00058 #ifdef DEBUG_PACKING_TIMES 00059 #define PC( n, m ) \ 00060 { \ 00061 if( __PACK_num == (unsigned int)n && __PACK_string == m ) \ 00062 __PACK_count++; \ 00063 else \ 00064 { \ 00065 if( __PACK_count > 1 ) std::cerr << " (" << __PACK_count << "x)"; \ 00066 __PACK_count = 1; \ 00067 __PACK_string = m; \ 00068 __PACK_num = n; \ 00069 std::cerr << std::endl << "PACK: " << n << m; \ 00070 } \ 00071 } 00072 #define UPC( n, m ) \ 00073 { \ 00074 if( __UNPACK_num == (unsigned int)n && __UNPACK_string == m ) \ 00075 __UNPACK_count++; \ 00076 else \ 00077 { \ 00078 if( __UNPACK_count > 1 ) std::cerr << "(" << __UNPACK_count << "x)"; \ 00079 __UNPACK_count = 1; \ 00080 __UNPACK_string = m; \ 00081 __UNPACK_num = n; \ 00082 std::cerr << std::endl << "UNPACK: " << n << m; \ 00083 } \ 00084 } 00085 #else 00086 #define PC( n, m ) 00087 #define UPC( n, m ) 00088 #endif 00089 00090 template < typename T > 00091 static inline void UNPACK( unsigned char*& buff, T* val, size_t count ) 00092 { 00093 memcpy( val, buff, count * sizeof( T ) ); 00094 buff += count * sizeof( T ); 00095 } 00096 00097 template < typename T > 00098 static inline void PACK( unsigned char*& buff, const T* val, size_t count ) 00099 { 00100 memcpy( buff, val, count * sizeof( T ) ); 00101 buff += count * sizeof( T ); 00102 } 00103 00104 static inline void PACK_INTS( unsigned char*& buff, const int* int_val, size_t num ) 00105 { 00106 PACK( buff, int_val, num ); 00107 PC( num, " ints" ); 00108 } 00109 00110 static inline void PACK_INT( unsigned char*& buff, int int_val ) 00111 { 00112 PACK_INTS( buff, &int_val, 1 ); 00113 } 00114 00115 static inline void PACK_DBLS( unsigned char*& buff, const double* dbl_val, size_t num ) 00116 { 00117 PACK( buff, dbl_val, num ); 00118 PC( num, " doubles" ); 00119 } 00120 00121 // static inline 00122 // void PACK_DBL(unsigned char*& buff, const double dbl_val) 00123 //{ PACK_DBLS(buff, &dbl_val, 1); } 00124 00125 static inline void PACK_EH( unsigned char*& buff, const EntityHandle* eh_val, size_t num ) 00126 { 00127 PACK( buff, eh_val, num ); 00128 PC( num, " handles" ); 00129 } 00130 00131 // static inline 00132 // void PACK_CHAR_64(unsigned char*& buff, const char* str) 00133 //{ 00134 // memcpy(buff, str, 64); 00135 // buff += 64; 00136 // PC(64, " chars"); 00137 //} 00138 00139 static inline void PACK_VOID( unsigned char*& buff, const void* val, size_t num ) 00140 { 00141 PACK( buff, reinterpret_cast< const unsigned char* >( val ), num ); 00142 PC( num, " void" ); 00143 } 00144 00145 static inline void PACK_BYTES( unsigned char*& buff, const void* val, int num ) 00146 { 00147 PACK_INT( buff, num ); 00148 PACK_VOID( buff, val, num ); 00149 } 00150 00151 static inline void PACK_RANGE( unsigned char*& buff, const Range& rng ) 00152 { 00153 PACK_INT( buff, rng.psize() ); 00154 Range::const_pair_iterator cit; 00155 for( cit = rng.const_pair_begin(); cit != rng.const_pair_end(); ++cit ) 00156 { 00157 EntityHandle eh[2] = { cit->first, cit->second }; 00158 PACK_EH( buff, eh, 2 ); 00159 } 00160 PC( rng.psize(), "-subranged range" ); 00161 } 00162 00163 static inline void UNPACK_INTS( unsigned char*& buff, int* int_val, size_t num ) 00164 { 00165 UNPACK( buff, int_val, num ); 00166 UPC( num, " ints" ); 00167 } 00168 00169 static inline void UNPACK_INT( unsigned char*& buff, int& int_val ) 00170 { 00171 UNPACK_INTS( buff, &int_val, 1 ); 00172 } 00173 00174 static inline void UNPACK_DBLS( unsigned char*& buff, double* dbl_val, size_t num ) 00175 { 00176 UNPACK( buff, dbl_val, num ); 00177 UPC( num, " doubles" ); 00178 } 00179 00180 static inline void UNPACK_DBL( unsigned char*& buff, double& dbl_val ) 00181 { 00182 UNPACK_DBLS( buff, &dbl_val, 1 ); 00183 } 00184 00185 static inline void UNPACK_EH( unsigned char*& buff, EntityHandle* eh_val, size_t num ) 00186 { 00187 UNPACK( buff, eh_val, num ); 00188 UPC( num, " handles" ); 00189 } 00190 00191 // static inline 00192 // void UNPACK_CHAR_64(unsigned char*& buff, char* char_val) 00193 //{ 00194 // memcpy(buff, char_val, 64); 00195 // buff += 64; 00196 // UPC(64, " chars"); 00197 //} 00198 00199 static inline void UNPACK_VOID( unsigned char*& buff, void* val, size_t num ) 00200 { 00201 UNPACK( buff, reinterpret_cast< unsigned char* >( val ), num ); 00202 UPC( num, " void" ); 00203 } 00204 00205 static inline void UNPACK_TYPE( unsigned char*& buff, EntityType& type ) 00206 { 00207 int int_type = MBMAXTYPE; 00208 UNPACK_INT( buff, int_type ); 00209 type = static_cast< EntityType >( int_type ); 00210 assert( type >= MBVERTEX && type <= MBMAXTYPE ); 00211 } 00212 00213 static inline void UNPACK_RANGE( unsigned char*& buff, Range& rng ) 00214 { 00215 int num_subs; 00216 EntityHandle eh[2]; 00217 UNPACK_INT( buff, num_subs ); 00218 for( int i = 0; i < num_subs; i++ ) 00219 { 00220 UPC( num_subs, "-subranged range" ); 00221 UNPACK_EH( buff, eh, 2 ); 00222 rng.insert( eh[0], eh[1] ); 00223 } 00224 } 00225 00226 enum MBMessageTag 00227 { 00228 MB_MESG_ANY = MPI_ANY_TAG, 00229 MB_MESG_ENTS_ACK, 00230 MB_MESG_ENTS_SIZE, 00231 MB_MESG_ENTS_LARGE, 00232 MB_MESG_REMOTEH_ACK, 00233 MB_MESG_REMOTEH_SIZE, 00234 MB_MESG_REMOTEH_LARGE, 00235 MB_MESG_TAGS_ACK, 00236 MB_MESG_TAGS_SIZE, 00237 MB_MESG_TAGS_LARGE 00238 }; 00239 00240 static inline size_t RANGE_SIZE( const Range& rng ) 00241 { 00242 return 2 * sizeof( EntityHandle ) * rng.psize() + sizeof( int ); 00243 } 00244 00245 #define PRINT_DEBUG_ISEND( A, B, C, D, E ) print_debug_isend( ( A ), ( B ), ( C ), ( D ), ( E ) ) 00246 #define PRINT_DEBUG_IRECV( A, B, C, D, E, F ) print_debug_irecv( ( A ), ( B ), ( C ), ( D ), ( E ), ( F ) ) 00247 #define PRINT_DEBUG_RECD( A ) print_debug_recd( ( A ) ) 00248 #define PRINT_DEBUG_WAITANY( A, B, C ) print_debug_waitany( ( A ), ( B ), ( C ) ) 00249 00250 void ParallelComm::print_debug_isend( int from, int to, unsigned char* buff, int tag, int sz ) 00251 { 00252 myDebug->tprintf( 3, "Isend, %d->%d, buffer ptr = %p, tag=%d, size=%d\n", from, to, (void*)buff, tag, sz ); 00253 } 00254 00255 void ParallelComm::print_debug_irecv( int to, int from, unsigned char* buff, int sz, int tag, int incoming ) 00256 { 00257 myDebug->tprintf( 3, "Irecv, %d<-%d, buffer ptr = %p, tag=%d, size=%d", to, from, (void*)buff, tag, sz ); 00258 if( tag < MB_MESG_REMOTEH_ACK ) 00259 myDebug->printf( 3, ", incoming1=%d\n", incoming ); 00260 else if( tag < MB_MESG_TAGS_ACK ) 00261 myDebug->printf( 3, ", incoming2=%d\n", incoming ); 00262 else 00263 myDebug->printf( 3, ", incoming=%d\n", incoming ); 00264 } 00265 00266 void ParallelComm::print_debug_recd( MPI_Status status ) 00267 { 00268 if( myDebug->get_verbosity() == 3 ) 00269 { 00270 int this_count; 00271 int success = MPI_Get_count( &status, MPI_UNSIGNED_CHAR, &this_count ); 00272 if( MPI_SUCCESS != success ) this_count = -1; 00273 myDebug->tprintf( 3, "Received from %d, count = %d, tag = %d\n", status.MPI_SOURCE, this_count, 00274 status.MPI_TAG ); 00275 } 00276 } 00277 00278 void ParallelComm::print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc ) 00279 { 00280 if( myDebug->get_verbosity() == 3 ) 00281 { 00282 myDebug->tprintf( 3, "Waitany, p=%d, ", proc ); 00283 if( tag < MB_MESG_REMOTEH_ACK ) 00284 myDebug->print( 3, ", recv_ent_reqs=" ); 00285 else if( tag < MB_MESG_TAGS_ACK ) 00286 myDebug->print( 3, ", recv_remoteh_reqs=" ); 00287 else 00288 myDebug->print( 3, ", recv_tag_reqs=" ); 00289 for( unsigned int i = 0; i < reqs.size(); i++ ) 00290 myDebug->printf( 3, " %p", (void*)(intptr_t)reqs[i] ); 00291 myDebug->print( 3, "\n" ); 00292 } 00293 } 00294 00295 /** Name of tag used to store ParallelComm Index on mesh paritioning sets */ 00296 const char* PARTITIONING_PCOMM_TAG_NAME = "__PRTN_PCOMM"; 00297 00298 /** \brief Tag storing parallel communication objects 00299 * 00300 * This tag stores pointers to ParallelComm communication 00301 * objects; one of these is allocated for each different 00302 * communicator used to read mesh. ParallelComm stores 00303 * partition and interface sets corresponding to its parallel mesh. 00304 * By default, a parallel read uses the first ParallelComm object 00305 * on the interface instance; if instantiated with one, ReadParallel 00306 * adds this object to the interface instance too. 00307 * 00308 * Tag type: opaque 00309 * Tag size: MAX_SHARING_PROCS*sizeof(ParallelComm*) 00310 */ 00311 #define PARALLEL_COMM_TAG_NAME "__PARALLEL_COMM" 00312 00313 ParallelComm::ParallelComm( Interface* impl, MPI_Comm cm, int* id ) 00314 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ), 00315 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ), 00316 myDebug( NULL ) 00317 { 00318 initialize(); 00319 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() ); 00320 if( id ) *id = pcommID; 00321 } 00322 00323 ParallelComm::ParallelComm( Interface* impl, std::vector< unsigned char >& /*tmp_buff*/, MPI_Comm cm, int* id ) 00324 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ), 00325 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ), 00326 myDebug( NULL ) 00327 { 00328 initialize(); 00329 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() ); 00330 if( id ) *id = pcommID; 00331 } 00332 00333 ParallelComm::~ParallelComm() 00334 { 00335 remove_pcomm( this ); 00336 delete_all_buffers(); 00337 delete myDebug; 00338 delete sharedSetData; 00339 } 00340 00341 void ParallelComm::initialize() 00342 { 00343 Core* core = dynamic_cast< Core* >( mbImpl ); 00344 sequenceManager = core->sequence_manager(); 00345 mbImpl->query_interface( errorHandler ); 00346 00347 // Initialize MPI, if necessary 00348 int flag = 1; 00349 int retval = MPI_Initialized( &flag ); 00350 if( MPI_SUCCESS != retval || !flag ) 00351 { 00352 int argc = 0; 00353 char** argv = NULL; 00354 00355 // mpi not initialized yet - initialize here 00356 retval = MPI_Init( &argc, &argv ); 00357 assert( MPI_SUCCESS == retval ); 00358 } 00359 00360 // Reserve space for vectors 00361 buffProcs.reserve( MAX_SHARING_PROCS ); 00362 localOwnedBuffs.reserve( MAX_SHARING_PROCS ); 00363 remoteOwnedBuffs.reserve( MAX_SHARING_PROCS ); 00364 00365 pcommID = add_pcomm( this ); 00366 00367 if( !myDebug ) 00368 { 00369 myDebug = new DebugOutput( "ParallelComm", std::cerr ); 00370 myDebug->set_rank( procConfig.proc_rank() ); 00371 } 00372 } 00373 00374 int ParallelComm::add_pcomm( ParallelComm* pc ) 00375 { 00376 // Add this pcomm to instance tag 00377 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS, (ParallelComm*)NULL ); 00378 Tag pc_tag = pcomm_tag( mbImpl, true ); 00379 assert( 0 != pc_tag ); 00380 00381 const EntityHandle root = 0; 00382 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00383 if( MB_SUCCESS != result && MB_TAG_NOT_FOUND != result ) return -1; 00384 int index = 0; 00385 while( index < MAX_SHARING_PROCS && pc_array[index] ) 00386 index++; 00387 if( index == MAX_SHARING_PROCS ) 00388 { 00389 index = -1; 00390 assert( false ); 00391 } 00392 else 00393 { 00394 pc_array[index] = pc; 00395 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00396 } 00397 return index; 00398 } 00399 00400 void ParallelComm::remove_pcomm( ParallelComm* pc ) 00401 { 00402 // Remove this pcomm from instance tag 00403 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS ); 00404 Tag pc_tag = pcomm_tag( mbImpl, true ); 00405 00406 const EntityHandle root = 0; 00407 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00408 std::vector< ParallelComm* >::iterator pc_it = std::find( pc_array.begin(), pc_array.end(), pc ); 00409 assert( MB_SUCCESS == result && pc_it != pc_array.end() ); 00410 // Empty if test to get around compiler warning about unused var 00411 if( MB_SUCCESS == result ) 00412 { 00413 } 00414 00415 *pc_it = NULL; 00416 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] ); 00417 } 00418 00419 //! Assign a global id space, for largest-dimension or all entities (and 00420 //! in either case for vertices too) 00421 ErrorCode ParallelComm::assign_global_ids( EntityHandle this_set, 00422 const int dimension, 00423 const int start_id, 00424 const bool largest_dim_only, 00425 const bool parallel, 00426 const bool owned_only ) 00427 { 00428 Range entities[4]; 00429 ErrorCode result; 00430 std::vector< unsigned char > pstatus; 00431 for( int dim = 0; dim <= dimension; dim++ ) 00432 { 00433 if( dim == 0 || !largest_dim_only || dim == dimension ) 00434 { 00435 result = mbImpl->get_entities_by_dimension( this_set, dim, entities[dim] );MB_CHK_SET_ERR( result, "Failed to get vertices in assign_global_ids" ); 00436 } 00437 00438 // Need to filter out non-locally-owned entities!!! 00439 pstatus.resize( entities[dim].size() ); 00440 result = mbImpl->tag_get_data( pstatus_tag(), entities[dim], &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus in assign_global_ids" ); 00441 00442 Range dum_range; 00443 Range::iterator rit; 00444 unsigned int i; 00445 for( rit = entities[dim].begin(), i = 0; rit != entities[dim].end(); ++rit, i++ ) 00446 if( pstatus[i] & PSTATUS_NOT_OWNED ) dum_range.insert( *rit ); 00447 entities[dim] = subtract( entities[dim], dum_range ); 00448 } 00449 00450 return assign_global_ids( entities, dimension, start_id, parallel, owned_only ); 00451 } 00452 00453 //! Assign a global id space, for largest-dimension or all entities (and 00454 //! in either case for vertices too) 00455 ErrorCode ParallelComm::assign_global_ids( Range entities[], 00456 const int dimension, 00457 const int start_id, 00458 const bool parallel, 00459 const bool owned_only ) 00460 { 00461 int local_num_elements[4]; 00462 ErrorCode result; 00463 for( int dim = 0; dim <= dimension; dim++ ) 00464 { 00465 local_num_elements[dim] = entities[dim].size(); 00466 } 00467 00468 // Communicate numbers 00469 std::vector< int > num_elements( procConfig.proc_size() * 4 ); 00470 #ifdef MOAB_HAVE_MPI 00471 if( procConfig.proc_size() > 1 && parallel ) 00472 { 00473 int retval = 00474 MPI_Allgather( local_num_elements, 4, MPI_INT, &num_elements[0], 4, MPI_INT, procConfig.proc_comm() ); 00475 if( 0 != retval ) return MB_FAILURE; 00476 } 00477 else 00478 #endif 00479 for( int dim = 0; dim < 4; dim++ ) 00480 num_elements[dim] = local_num_elements[dim]; 00481 00482 // My entities start at one greater than total_elems[d] 00483 int total_elems[4] = { start_id, start_id, start_id, start_id }; 00484 00485 for( unsigned int proc = 0; proc < procConfig.proc_rank(); proc++ ) 00486 { 00487 for( int dim = 0; dim < 4; dim++ ) 00488 total_elems[dim] += num_elements[4 * proc + dim]; 00489 } 00490 00491 // Assign global ids now 00492 Tag gid_tag = mbImpl->globalId_tag(); 00493 00494 for( int dim = 0; dim < 4; dim++ ) 00495 { 00496 if( entities[dim].empty() ) continue; 00497 num_elements.resize( entities[dim].size() ); 00498 int i = 0; 00499 for( Range::iterator rit = entities[dim].begin(); rit != entities[dim].end(); ++rit ) 00500 num_elements[i++] = total_elems[dim]++; 00501 00502 result = mbImpl->tag_set_data( gid_tag, entities[dim], &num_elements[0] );MB_CHK_SET_ERR( result, "Failed to set global id tag in assign_global_ids" ); 00503 } 00504 00505 if( owned_only ) return MB_SUCCESS; 00506 00507 // Exchange tags 00508 for( int dim = 1; dim < 4; dim++ ) 00509 entities[0].merge( entities[dim] ); 00510 00511 return exchange_tags( gid_tag, entities[0] ); 00512 } 00513 00514 int ParallelComm::get_buffers( int to_proc, bool* is_new ) 00515 { 00516 int ind = -1; 00517 std::vector< unsigned int >::iterator vit = std::find( buffProcs.begin(), buffProcs.end(), to_proc ); 00518 if( vit == buffProcs.end() ) 00519 { 00520 assert( "shouldn't need buffer to myself" && to_proc != (int)procConfig.proc_rank() ); 00521 ind = buffProcs.size(); 00522 buffProcs.push_back( (unsigned int)to_proc ); 00523 localOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) ); 00524 remoteOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) ); 00525 if( is_new ) *is_new = true; 00526 } 00527 else 00528 { 00529 ind = vit - buffProcs.begin(); 00530 if( is_new ) *is_new = false; 00531 } 00532 assert( ind < MAX_SHARING_PROCS ); 00533 return ind; 00534 } 00535 00536 ErrorCode ParallelComm::broadcast_entities( const int from_proc, 00537 Range& entities, 00538 const bool adjacencies, 00539 const bool tags ) 00540 { 00541 #ifndef MOAB_HAVE_MPI 00542 return MB_FAILURE; 00543 #else 00544 00545 ErrorCode result = MB_SUCCESS; 00546 int success; 00547 int buff_size; 00548 00549 Buffer buff( INITIAL_BUFF_SIZE ); 00550 buff.reset_ptr( sizeof( int ) ); 00551 if( (int)procConfig.proc_rank() == from_proc ) 00552 { 00553 result = add_verts( entities );MB_CHK_SET_ERR( result, "Failed to add adj vertices" ); 00554 00555 buff.reset_ptr( sizeof( int ) ); 00556 result = pack_buffer( entities, adjacencies, tags, false, -1, &buff );MB_CHK_SET_ERR( result, "Failed to compute buffer size in broadcast_entities" ); 00557 buff.set_stored_size(); 00558 buff_size = buff.buff_ptr - buff.mem_ptr; 00559 } 00560 00561 success = MPI_Bcast( &buff_size, 1, MPI_INT, from_proc, procConfig.proc_comm() ); 00562 if( MPI_SUCCESS != success ) 00563 { 00564 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" ); 00565 } 00566 00567 if( !buff_size ) // No data 00568 return MB_SUCCESS; 00569 00570 if( (int)procConfig.proc_rank() != from_proc ) buff.reserve( buff_size ); 00571 00572 size_t offset = 0; 00573 while( buff_size ) 00574 { 00575 int sz = std::min( buff_size, MAX_BCAST_SIZE ); 00576 success = MPI_Bcast( buff.mem_ptr + offset, sz, MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() ); 00577 if( MPI_SUCCESS != success ) 00578 { 00579 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer failed" ); 00580 } 00581 00582 offset += sz; 00583 buff_size -= sz; 00584 } 00585 00586 if( (int)procConfig.proc_rank() != from_proc ) 00587 { 00588 std::vector< std::vector< EntityHandle > > dum1a, dum1b; 00589 std::vector< std::vector< int > > dum1p; 00590 std::vector< EntityHandle > dum2, dum4; 00591 std::vector< unsigned int > dum3; 00592 buff.reset_ptr( sizeof( int ) ); 00593 result = unpack_buffer( buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );MB_CHK_SET_ERR( result, "Failed to unpack buffer in broadcast_entities" ); 00594 std::copy( dum4.begin(), dum4.end(), range_inserter( entities ) ); 00595 } 00596 00597 return MB_SUCCESS; 00598 #endif 00599 } 00600 00601 ErrorCode ParallelComm::scatter_entities( const int from_proc, 00602 std::vector< Range >& entities, 00603 const bool adjacencies, 00604 const bool tags ) 00605 { 00606 #ifndef MOAB_HAVE_MPI 00607 return MB_FAILURE; 00608 #else 00609 ErrorCode result = MB_SUCCESS; 00610 int i, success, buff_size, prev_size; 00611 int nProcs = (int)procConfig.proc_size(); 00612 int* sendCounts = new int[nProcs]; 00613 int* displacements = new int[nProcs]; 00614 sendCounts[0] = sizeof( int ); 00615 displacements[0] = 0; 00616 Buffer buff( INITIAL_BUFF_SIZE ); 00617 buff.reset_ptr( sizeof( int ) ); 00618 buff.set_stored_size(); 00619 unsigned int my_proc = procConfig.proc_rank(); 00620 00621 // Get buffer size array for each remote processor 00622 if( my_proc == (unsigned int)from_proc ) 00623 { 00624 for( i = 1; i < nProcs; i++ ) 00625 { 00626 prev_size = buff.buff_ptr - buff.mem_ptr; 00627 buff.reset_ptr( prev_size + sizeof( int ) ); 00628 result = add_verts( entities[i] );MB_CHK_SET_ERR( result, "Failed to add verts" ); 00629 00630 result = pack_buffer( entities[i], adjacencies, tags, false, -1, &buff ); 00631 if( MB_SUCCESS != result ) 00632 { 00633 delete[] sendCounts; 00634 delete[] displacements; 00635 MB_SET_ERR( result, "Failed to pack buffer in scatter_entities" ); 00636 } 00637 00638 buff_size = buff.buff_ptr - buff.mem_ptr - prev_size; 00639 *( (int*)( buff.mem_ptr + prev_size ) ) = buff_size; 00640 sendCounts[i] = buff_size; 00641 } 00642 } 00643 00644 // Broadcast buffer size array 00645 success = MPI_Bcast( sendCounts, nProcs, MPI_INT, from_proc, procConfig.proc_comm() ); 00646 if( MPI_SUCCESS != success ) 00647 { 00648 delete[] sendCounts; 00649 delete[] displacements; 00650 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" ); 00651 } 00652 00653 for( i = 1; i < nProcs; i++ ) 00654 { 00655 displacements[i] = displacements[i - 1] + sendCounts[i - 1]; 00656 } 00657 00658 Buffer rec_buff; 00659 rec_buff.reserve( sendCounts[my_proc] ); 00660 00661 // Scatter actual geometry 00662 success = MPI_Scatterv( buff.mem_ptr, sendCounts, displacements, MPI_UNSIGNED_CHAR, rec_buff.mem_ptr, 00663 sendCounts[my_proc], MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() ); 00664 00665 if( MPI_SUCCESS != success ) 00666 { 00667 delete[] sendCounts; 00668 delete[] displacements; 00669 MB_SET_ERR( MB_FAILURE, "MPI_Scatterv of buffer failed" ); 00670 } 00671 00672 // Unpack in remote processors 00673 if( my_proc != (unsigned int)from_proc ) 00674 { 00675 std::vector< std::vector< EntityHandle > > dum1a, dum1b; 00676 std::vector< std::vector< int > > dum1p; 00677 std::vector< EntityHandle > dum2, dum4; 00678 std::vector< unsigned int > dum3; 00679 rec_buff.reset_ptr( sizeof( int ) ); 00680 result = unpack_buffer( rec_buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 ); 00681 if( MB_SUCCESS != result ) 00682 { 00683 delete[] sendCounts; 00684 delete[] displacements; 00685 MB_SET_ERR( result, "Failed to unpack buffer in scatter_entities" ); 00686 } 00687 00688 std::copy( dum4.begin(), dum4.end(), range_inserter( entities[my_proc] ) ); 00689 } 00690 00691 delete[] sendCounts; 00692 delete[] displacements; 00693 00694 return MB_SUCCESS; 00695 #endif 00696 } 00697 00698 ErrorCode ParallelComm::send_entities( const int to_proc, 00699 Range& orig_ents, 00700 const bool adjs, 00701 const bool tags, 00702 const bool store_remote_handles, 00703 const bool is_iface, 00704 Range& /*final_ents*/, 00705 int& incoming1, 00706 int& incoming2, 00707 TupleList& entprocs, 00708 std::vector< MPI_Request >& recv_remoteh_reqs, 00709 bool /*wait_all*/ ) 00710 { 00711 #ifndef MOAB_HAVE_MPI 00712 return MB_FAILURE; 00713 #else 00714 // Pack entities to local buffer 00715 int ind = get_buffers( to_proc ); 00716 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 00717 00718 // Add vertices 00719 ErrorCode result = add_verts( orig_ents );MB_CHK_SET_ERR( result, "Failed to add verts in send_entities" ); 00720 00721 // Filter out entities already shared with destination 00722 Range tmp_range; 00723 result = filter_pstatus( orig_ents, PSTATUS_SHARED, PSTATUS_AND, to_proc, &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 00724 if( !tmp_range.empty() ) 00725 { 00726 orig_ents = subtract( orig_ents, tmp_range ); 00727 } 00728 00729 result = pack_buffer( orig_ents, adjs, tags, store_remote_handles, to_proc, localOwnedBuffs[ind], &entprocs );MB_CHK_SET_ERR( result, "Failed to pack buffer in send_entities" ); 00730 00731 // Send buffer 00732 result = send_buffer( to_proc, localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], recvReqs[2 * ind + 1], 00733 (int*)( remoteOwnedBuffs[ind]->mem_ptr ), 00734 //&ackbuff, 00735 incoming1, MB_MESG_REMOTEH_SIZE, 00736 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind] : NULL ), 00737 &recv_remoteh_reqs[2 * ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to send buffer" ); 00738 00739 return MB_SUCCESS; 00740 #endif 00741 } 00742 00743 ErrorCode ParallelComm::send_entities( std::vector< unsigned int >& send_procs, 00744 std::vector< Range* >& send_ents, 00745 int& incoming1, 00746 int& incoming2, 00747 const bool store_remote_handles ) 00748 { 00749 #ifdef MOAB_HAVE_MPE 00750 if( myDebug->get_verbosity() == 2 ) 00751 { 00752 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_entities." ); 00753 } 00754 #endif 00755 myDebug->tprintf( 1, "Entering send_entities\n" ); 00756 if( myDebug->get_verbosity() == 4 ) 00757 { 00758 msgs.clear(); 00759 msgs.reserve( MAX_SHARING_PROCS ); 00760 } 00761 00762 unsigned int i; 00763 int ind; 00764 ErrorCode result = MB_SUCCESS; 00765 00766 // Set buffProcs with communicating procs 00767 unsigned int n_proc = send_procs.size(); 00768 for( i = 0; i < n_proc; i++ ) 00769 { 00770 ind = get_buffers( send_procs[i] ); 00771 result = add_verts( *send_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" ); 00772 00773 // Filter out entities already shared with destination 00774 Range tmp_range; 00775 result = filter_pstatus( *send_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 00776 if( !tmp_range.empty() ) 00777 { 00778 *send_ents[i] = subtract( *send_ents[i], tmp_range ); 00779 } 00780 } 00781 00782 //=========================================== 00783 // Get entities to be sent to neighbors 00784 // Need to get procs each entity is sent to 00785 //=========================================== 00786 Range allsent, tmp_range; 00787 int npairs = 0; 00788 TupleList entprocs; 00789 for( i = 0; i < n_proc; i++ ) 00790 { 00791 int n_ents = send_ents[i]->size(); 00792 if( n_ents > 0 ) 00793 { 00794 npairs += n_ents; // Get the total # of proc/handle pairs 00795 allsent.merge( *send_ents[i] ); 00796 } 00797 } 00798 00799 // Allocate a TupleList of that size 00800 entprocs.initialize( 1, 0, 1, 0, npairs ); 00801 entprocs.enableWriteAccess(); 00802 00803 // Put the proc/handle pairs in the list 00804 for( i = 0; i < n_proc; i++ ) 00805 { 00806 for( Range::iterator rit = send_ents[i]->begin(); rit != send_ents[i]->end(); ++rit ) 00807 { 00808 entprocs.vi_wr[entprocs.get_n()] = send_procs[i]; 00809 entprocs.vul_wr[entprocs.get_n()] = *rit; 00810 entprocs.inc_n(); 00811 } 00812 } 00813 00814 // Sort by handle 00815 moab::TupleList::buffer sort_buffer; 00816 sort_buffer.buffer_init( npairs ); 00817 entprocs.sort( 1, &sort_buffer ); 00818 entprocs.disableWriteAccess(); 00819 sort_buffer.reset(); 00820 00821 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(), 00822 (unsigned long)allsent.size() ); 00823 00824 //=========================================== 00825 // Pack and send ents from this proc to others 00826 //=========================================== 00827 for( i = 0; i < n_proc; i++ ) 00828 { 00829 if( send_ents[i]->size() > 0 ) 00830 { 00831 ind = get_buffers( send_procs[i] ); 00832 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", send_ents[i]->compactness(), 00833 (unsigned long)send_ents[i]->size() ); 00834 // Reserve space on front for size and for initial buff size 00835 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 00836 result = pack_buffer( *send_ents[i], false, true, store_remote_handles, buffProcs[ind], 00837 localOwnedBuffs[ind], &entprocs, &allsent ); 00838 00839 if( myDebug->get_verbosity() == 4 ) 00840 { 00841 msgs.resize( msgs.size() + 1 ); 00842 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 00843 } 00844 00845 // Send the buffer (size stored in front in send_buffer) 00846 result = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], 00847 recvReqs[2 * ind + 1], &ackbuff, incoming1, MB_MESG_REMOTEH_SIZE, 00848 ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recvRemotehReqs[2 * ind], 00849 &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost send" ); 00850 } 00851 } 00852 entprocs.reset(); 00853 00854 #ifdef MOAB_HAVE_MPE 00855 if( myDebug->get_verbosity() == 2 ) 00856 { 00857 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_entities." ); 00858 } 00859 #endif 00860 00861 return MB_SUCCESS; 00862 } 00863 00864 ///////////////////////////////////////////////////////////////////////////////// 00865 // Send and Receive routines for a sequence of entities: use case UMR 00866 ///////////////////////////////////////////////////////////////////////////////// 00867 void print_buff( unsigned char* ch, int size ) 00868 { 00869 for( int i = 0; i < size; i++ ) 00870 std::cout << ch[i]; 00871 std::cout << "\n"; 00872 } 00873 ErrorCode ParallelComm::send_recv_entities( std::vector< int >& send_procs, 00874 std::vector< std::vector< int > >& msgsizes, 00875 std::vector< std::vector< EntityHandle > >& senddata, 00876 std::vector< std::vector< EntityHandle > >& recvdata ) 00877 { 00878 #ifdef USE_MPE 00879 if( myDebug->get_verbosity() == 2 ) 00880 { 00881 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_recv_entities." ); 00882 } 00883 #endif 00884 myDebug->tprintf( 1, "Entering send_recv_entities\n" ); 00885 if( myDebug->get_verbosity() == 4 ) 00886 { 00887 msgs.clear(); 00888 msgs.reserve( MAX_SHARING_PROCS ); 00889 } 00890 00891 // unsigned int i; 00892 int i, ind, success; 00893 ErrorCode error = MB_SUCCESS; 00894 00895 //=========================================== 00896 // Pack and send ents from this proc to others 00897 //=========================================== 00898 00899 // std::cout<<"resetting all buffers"<<std::endl; 00900 00901 reset_all_buffers(); 00902 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 00903 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 00904 int ack_buff; 00905 int incoming = 0; 00906 00907 std::vector< unsigned int >::iterator sit; 00908 00909 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 00910 { 00911 incoming++; 00912 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 00913 MB_MESG_ENTS_SIZE, incoming ); 00914 00915 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit, 00916 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] ); 00917 if( success != MPI_SUCCESS ) 00918 { 00919 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in send_recv_entities" ); 00920 } 00921 } 00922 00923 // std::set<unsigned int>::iterator it; 00924 for( i = 0; i < (int)send_procs.size(); i++ ) 00925 { 00926 // Get index of the shared processor in the local buffer 00927 ind = get_buffers( send_procs[i] ); 00928 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 00929 00930 int buff_size = msgsizes[i].size() * sizeof( int ) + senddata[i].size() * sizeof( EntityHandle ); 00931 localOwnedBuffs[ind]->check_space( buff_size ); 00932 00933 // Pack entities 00934 std::vector< int > msg; 00935 msg.insert( msg.end(), msgsizes[i].begin(), msgsizes[i].end() ); 00936 PACK_INTS( localOwnedBuffs[ind]->buff_ptr, &msg[0], msg.size() ); 00937 00938 std::vector< EntityHandle > entities; 00939 entities.insert( entities.end(), senddata[i].begin(), senddata[i].end() ); 00940 PACK_EH( localOwnedBuffs[ind]->buff_ptr, &entities[0], entities.size() ); 00941 localOwnedBuffs[ind]->set_stored_size(); 00942 00943 if( myDebug->get_verbosity() == 4 ) 00944 { 00945 msgs.resize( msgs.size() + 1 ); 00946 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 00947 } 00948 00949 // Send the buffer (size stored in front in send_buffer) 00950 error = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind], 00951 recv_ent_reqs[3 * ind + 2], &ack_buff, incoming );MB_CHK_SET_ERR( error, "Failed to Isend in send_recv_entities" ); 00952 } 00953 00954 //=========================================== 00955 // Receive and unpack ents from received data 00956 //=========================================== 00957 00958 while( incoming ) 00959 { 00960 00961 MPI_Status status; 00962 int index_in_recv_requests; 00963 00964 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 00965 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &index_in_recv_requests, &status ); 00966 if( MPI_SUCCESS != success ) 00967 { 00968 MB_SET_ERR( MB_FAILURE, "Failed in waitany in send_recv_entities" ); 00969 } 00970 00971 // Processor index in the list is divided by 3 00972 ind = index_in_recv_requests / 3; 00973 00974 PRINT_DEBUG_RECD( status ); 00975 00976 // OK, received something; decrement incoming counter 00977 incoming--; 00978 00979 bool done = false; 00980 00981 error = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind], 00982 recv_ent_reqs[3 * ind + 1], // This is for receiving the second message 00983 recv_ent_reqs[3 * ind + 2], // This would be for ack, but it is not 00984 // used; consider removing it 00985 incoming, localOwnedBuffs[ind], 00986 sendReqs[3 * ind + 1], // Send request for sending the second message 00987 sendReqs[3 * ind + 2], // This is for sending the ack 00988 done );MB_CHK_SET_ERR( error, "Failed to resize recv buffer" ); 00989 00990 if( done ) 00991 { 00992 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 00993 00994 int from_proc = status.MPI_SOURCE; 00995 int idx = std::find( send_procs.begin(), send_procs.end(), from_proc ) - send_procs.begin(); 00996 00997 int msg = msgsizes[idx].size(); 00998 std::vector< int > recvmsg( msg ); 00999 int ndata = senddata[idx].size(); 01000 std::vector< EntityHandle > dum_vec( ndata ); 01001 01002 UNPACK_INTS( remoteOwnedBuffs[ind]->buff_ptr, &recvmsg[0], msg ); 01003 UNPACK_EH( remoteOwnedBuffs[ind]->buff_ptr, &dum_vec[0], ndata ); 01004 01005 recvdata[idx].insert( recvdata[idx].end(), dum_vec.begin(), dum_vec.end() ); 01006 } 01007 } 01008 01009 #ifdef USE_MPE 01010 if( myDebug->get_verbosity() == 2 ) 01011 { 01012 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_recv_entities." ); 01013 } 01014 #endif 01015 01016 return MB_SUCCESS; 01017 } 01018 01019 ErrorCode ParallelComm::update_remote_data( EntityHandle entity, 01020 std::vector< int >& procs, 01021 std::vector< EntityHandle >& handles ) 01022 { 01023 ErrorCode error; 01024 unsigned char pstatus = PSTATUS_INTERFACE; 01025 01026 int procmin = *std::min_element( procs.begin(), procs.end() ); 01027 01028 if( (int)rank() > procmin ) 01029 pstatus |= PSTATUS_NOT_OWNED; 01030 else 01031 procmin = rank(); 01032 01033 // DBG 01034 // std::cout<<"entity = "<<entity<<std::endl; 01035 // for (int j=0; j<procs.size(); j++) 01036 // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl; 01037 // DBG 01038 01039 if( (int)procs.size() > 1 ) 01040 { 01041 procs.push_back( rank() ); 01042 handles.push_back( entity ); 01043 01044 int idx = std::find( procs.begin(), procs.end(), procmin ) - procs.begin(); 01045 01046 std::iter_swap( procs.begin(), procs.begin() + idx ); 01047 std::iter_swap( handles.begin(), handles.begin() + idx ); 01048 01049 // DBG 01050 // std::cout<<"entity = "<<entity<<std::endl; 01051 // for (int j=0; j<procs.size(); j++) 01052 // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl; 01053 // DBG 01054 } 01055 01056 // if ((entity == 10388) && (rank()==1)) 01057 // std::cout<<"Here"<<std::endl; 01058 01059 error = update_remote_data( entity, &procs[0], &handles[0], procs.size(), pstatus );MB_CHK_ERR( error ); 01060 01061 return MB_SUCCESS; 01062 } 01063 01064 ErrorCode ParallelComm::get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc ) 01065 { 01066 ErrorCode error; 01067 std::vector< EntityHandle > newents; 01068 error = get_remote_handles( true, local_vec, rem_vec, num_ents, to_proc, newents );MB_CHK_ERR( error ); 01069 01070 return MB_SUCCESS; 01071 } 01072 01073 ////////////////////////////////////////////////////////////////// 01074 01075 ErrorCode ParallelComm::recv_entities( const int from_proc, 01076 const bool store_remote_handles, 01077 const bool is_iface, 01078 Range& final_ents, 01079 int& incoming1, 01080 int& incoming2, 01081 std::vector< std::vector< EntityHandle > >& L1hloc, 01082 std::vector< std::vector< EntityHandle > >& L1hrem, 01083 std::vector< std::vector< int > >& L1p, 01084 std::vector< EntityHandle >& L2hloc, 01085 std::vector< EntityHandle >& L2hrem, 01086 std::vector< unsigned int >& L2p, 01087 std::vector< MPI_Request >& recv_remoteh_reqs, 01088 bool /*wait_all*/ ) 01089 { 01090 #ifndef MOAB_HAVE_MPI 01091 return MB_FAILURE; 01092 #else 01093 // Non-blocking receive for the first message (having size info) 01094 int ind1 = get_buffers( from_proc ); 01095 incoming1++; 01096 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, 01097 MB_MESG_ENTS_SIZE, incoming1 ); 01098 int success = MPI_Irecv( remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, 01099 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind1] ); 01100 if( success != MPI_SUCCESS ) 01101 { 01102 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 01103 } 01104 01105 // Receive messages in while loop 01106 return recv_messages( from_proc, store_remote_handles, is_iface, final_ents, incoming1, incoming2, L1hloc, L1hrem, 01107 L1p, L2hloc, L2hrem, L2p, recv_remoteh_reqs ); 01108 #endif 01109 } 01110 01111 ErrorCode ParallelComm::recv_entities( std::set< unsigned int >& recv_procs, 01112 int incoming1, 01113 int incoming2, 01114 const bool store_remote_handles, 01115 const bool migrate ) 01116 { 01117 //=========================================== 01118 // Receive/unpack new entities 01119 //=========================================== 01120 // Number of incoming messages is the number of procs we communicate with 01121 int success, ind, i; 01122 ErrorCode result; 01123 MPI_Status status; 01124 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() ); 01125 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() ); 01126 std::vector< std::vector< int > > L1p( buffProcs.size() ); 01127 std::vector< EntityHandle > L2hloc, L2hrem; 01128 std::vector< unsigned int > L2p; 01129 std::vector< EntityHandle > new_ents; 01130 01131 while( incoming1 ) 01132 { 01133 // Wait for all recvs of ents before proceeding to sending remote handles, 01134 // b/c some procs may have sent to a 3rd proc ents owned by me; 01135 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 01136 01137 success = MPI_Waitany( 2 * buffProcs.size(), &recvReqs[0], &ind, &status ); 01138 if( MPI_SUCCESS != success ) 01139 { 01140 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 01141 } 01142 01143 PRINT_DEBUG_RECD( status ); 01144 01145 // OK, received something; decrement incoming counter 01146 incoming1--; 01147 bool done = false; 01148 01149 // In case ind is for ack, we need index of one before it 01150 unsigned int base_ind = 2 * ( ind / 2 ); 01151 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 2], recvReqs[ind], recvReqs[ind + 1], 01152 incoming1, localOwnedBuffs[ind / 2], sendReqs[base_ind], sendReqs[base_ind + 1], done, 01153 ( store_remote_handles ? localOwnedBuffs[ind / 2] : NULL ), MB_MESG_REMOTEH_SIZE, 01154 &recvRemotehReqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 01155 01156 if( done ) 01157 { 01158 if( myDebug->get_verbosity() == 4 ) 01159 { 01160 msgs.resize( msgs.size() + 1 ); 01161 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 2] ); 01162 } 01163 01164 // Message completely received - process buffer that was sent 01165 remoteOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) ); 01166 result = unpack_buffer( remoteOwnedBuffs[ind / 2]->buff_ptr, store_remote_handles, buffProcs[ind / 2], 01167 ind / 2, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true ); 01168 if( MB_SUCCESS != result ) 01169 { 01170 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl; 01171 print_buffer( remoteOwnedBuffs[ind / 2]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 2], false ); 01172 return result; 01173 } 01174 01175 if( recvReqs.size() != 2 * buffProcs.size() ) 01176 { 01177 // Post irecv's for remote handles from new proc 01178 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01179 for( i = recvReqs.size(); i < (int)( 2 * buffProcs.size() ); i += 2 ) 01180 { 01181 localOwnedBuffs[i / 2]->reset_buffer(); 01182 incoming2++; 01183 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 2], localOwnedBuffs[i / 2]->mem_ptr, 01184 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 ); 01185 success = MPI_Irecv( localOwnedBuffs[i / 2]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, 01186 buffProcs[i / 2], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(), 01187 &recvRemotehReqs[i] ); 01188 if( success != MPI_SUCCESS ) 01189 { 01190 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" ); 01191 } 01192 } 01193 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01194 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 01195 } 01196 } 01197 } 01198 01199 // Assign and remove newly created elements from/to receive processor 01200 result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" ); 01201 if( migrate ) 01202 { 01203 // result = remove_entities_part(allsent, procConfig.proc_rank());MB_CHK_SET_ERR(ressult, 01204 // "Failed to remove entities to part"); 01205 } 01206 01207 // Add requests for any new addl procs 01208 if( recvReqs.size() != 2 * buffProcs.size() ) 01209 { 01210 // Shouldn't get here... 01211 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" ); 01212 } 01213 01214 #ifdef MOAB_HAVE_MPE 01215 if( myDebug->get_verbosity() == 2 ) 01216 { 01217 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending recv entities." ); 01218 } 01219 #endif 01220 01221 //=========================================== 01222 // Send local handles for new entity to owner 01223 //=========================================== 01224 std::set< unsigned int >::iterator it = recv_procs.begin(); 01225 std::set< unsigned int >::iterator eit = recv_procs.end(); 01226 for( ; it != eit; ++it ) 01227 { 01228 ind = get_buffers( *it ); 01229 // Reserve space on front for size and for initial buff size 01230 remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 01231 01232 result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 01233 remoteOwnedBuffs[ind]->set_stored_size(); 01234 01235 if( myDebug->get_verbosity() == 4 ) 01236 { 01237 msgs.resize( msgs.size() + 1 ); 01238 msgs.back() = new Buffer( *remoteOwnedBuffs[ind] ); 01239 } 01240 result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[2 * ind], 01241 recvRemotehReqs[2 * ind + 1], &ackbuff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 01242 } 01243 01244 //=========================================== 01245 // Process remote handles of my ghosteds 01246 //=========================================== 01247 while( incoming2 ) 01248 { 01249 PRINT_DEBUG_WAITANY( recvRemotehReqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 01250 success = MPI_Waitany( 2 * buffProcs.size(), &recvRemotehReqs[0], &ind, &status ); 01251 if( MPI_SUCCESS != success ) 01252 { 01253 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 01254 } 01255 01256 // OK, received something; decrement incoming counter 01257 incoming2--; 01258 01259 PRINT_DEBUG_RECD( status ); 01260 bool done = false; 01261 unsigned int base_ind = 2 * ( ind / 2 ); 01262 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 2], recvRemotehReqs[ind], 01263 recvRemotehReqs[ind + 1], incoming2, remoteOwnedBuffs[ind / 2], sendReqs[base_ind], 01264 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 01265 if( done ) 01266 { 01267 // Incoming remote handles 01268 if( myDebug->get_verbosity() == 4 ) 01269 { 01270 msgs.resize( msgs.size() + 1 ); 01271 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 01272 } 01273 01274 localOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) ); 01275 result = 01276 unpack_remote_handles( buffProcs[ind / 2], localOwnedBuffs[ind / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 01277 } 01278 } 01279 01280 #ifdef MOAB_HAVE_MPE 01281 if( myDebug->get_verbosity() == 2 ) 01282 { 01283 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." ); 01284 MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending recv entities (still doing checks)." ); 01285 } 01286 #endif 01287 myDebug->tprintf( 1, "Exiting recv_entities.\n" ); 01288 01289 return MB_SUCCESS; 01290 } 01291 01292 ErrorCode ParallelComm::recv_messages( const int from_proc, 01293 const bool store_remote_handles, 01294 const bool is_iface, 01295 Range& final_ents, 01296 int& incoming1, 01297 int& incoming2, 01298 std::vector< std::vector< EntityHandle > >& L1hloc, 01299 std::vector< std::vector< EntityHandle > >& L1hrem, 01300 std::vector< std::vector< int > >& L1p, 01301 std::vector< EntityHandle >& L2hloc, 01302 std::vector< EntityHandle >& L2hrem, 01303 std::vector< unsigned int >& L2p, 01304 std::vector< MPI_Request >& recv_remoteh_reqs ) 01305 { 01306 #ifndef MOAB_HAVE_MPI 01307 return MB_FAILURE; 01308 #else 01309 MPI_Status status; 01310 ErrorCode result; 01311 int ind1 = get_buffers( from_proc ); 01312 int success, ind2; 01313 std::vector< EntityHandle > new_ents; 01314 01315 // Wait and receive messages 01316 while( incoming1 ) 01317 { 01318 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() ); 01319 success = MPI_Waitany( 2, &recvReqs[2 * ind1], &ind2, &status ); 01320 if( MPI_SUCCESS != success ) 01321 { 01322 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_messages" ); 01323 } 01324 01325 PRINT_DEBUG_RECD( status ); 01326 01327 // OK, received something; decrement incoming counter 01328 incoming1--; 01329 bool done = false; 01330 01331 // In case ind is for ack, we need index of one before it 01332 ind2 += 2 * ind1; 01333 unsigned int base_ind = 2 * ( ind2 / 2 ); 01334 01335 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind2 / 2], 01336 // recvbuff, 01337 recvReqs[ind2], recvReqs[ind2 + 1], incoming1, localOwnedBuffs[ind2 / 2], 01338 sendReqs[base_ind], sendReqs[base_ind + 1], done, 01339 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind2 / 2] : NULL ), 01340 MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 01341 01342 if( done ) 01343 { 01344 // If it is done, unpack buffer 01345 remoteOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) ); 01346 result = unpack_buffer( remoteOwnedBuffs[ind2 / 2]->buff_ptr, store_remote_handles, from_proc, ind2 / 2, 01347 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );MB_CHK_SET_ERR( result, "Failed to unpack buffer in recev_messages" ); 01348 01349 std::copy( new_ents.begin(), new_ents.end(), range_inserter( final_ents ) ); 01350 01351 // Send local handles for new elements to owner 01352 // Reserve space on front for size and for initial buff size 01353 remoteOwnedBuffs[ind2 / 2]->reset_buffer( sizeof( int ) ); 01354 01355 result = pack_remote_handles( L1hloc[ind2 / 2], L1hrem[ind2 / 2], L1p[ind2 / 2], from_proc, 01356 remoteOwnedBuffs[ind2 / 2] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 01357 remoteOwnedBuffs[ind2 / 2]->set_stored_size(); 01358 01359 result = send_buffer( buffProcs[ind2 / 2], remoteOwnedBuffs[ind2 / 2], MB_MESG_REMOTEH_SIZE, sendReqs[ind2], 01360 recv_remoteh_reqs[ind2 + 1], (int*)( localOwnedBuffs[ind2 / 2]->mem_ptr ), 01361 //&ackbuff, 01362 incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 01363 } 01364 } 01365 01366 return MB_SUCCESS; 01367 #endif 01368 } 01369 01370 ErrorCode ParallelComm::recv_remote_handle_messages( const int from_proc, 01371 int& incoming2, 01372 std::vector< EntityHandle >& L2hloc, 01373 std::vector< EntityHandle >& L2hrem, 01374 std::vector< unsigned int >& L2p, 01375 std::vector< MPI_Request >& recv_remoteh_reqs ) 01376 { 01377 #ifndef MOAB_HAVE_MPI 01378 return MB_FAILURE; 01379 #else 01380 MPI_Status status; 01381 ErrorCode result; 01382 int ind1 = get_buffers( from_proc ); 01383 int success, ind2; 01384 01385 while( incoming2 ) 01386 { 01387 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 01388 success = MPI_Waitany( 2, &recv_remoteh_reqs[2 * ind1], &ind2, &status ); 01389 if( MPI_SUCCESS != success ) 01390 { 01391 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_remote_handle_messages" ); 01392 } 01393 01394 // OK, received something; decrement incoming counter 01395 incoming2--; 01396 01397 PRINT_DEBUG_RECD( status ); 01398 01399 bool done = false; 01400 ind2 += 2 * ind1; 01401 unsigned int base_ind = 2 * ( ind2 / 2 ); 01402 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind2 / 2], recv_remoteh_reqs[ind2], 01403 recv_remoteh_reqs[ind2 + 1], incoming2, remoteOwnedBuffs[ind2 / 2], sendReqs[base_ind], 01404 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 01405 if( done ) 01406 { 01407 // Incoming remote handles 01408 localOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) ); 01409 result = 01410 unpack_remote_handles( buffProcs[ind2 / 2], localOwnedBuffs[ind2 / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 01411 } 01412 } 01413 01414 return MB_SUCCESS; 01415 #endif 01416 } 01417 01418 ErrorCode ParallelComm::pack_buffer( Range& orig_ents, 01419 const bool /*adjacencies*/, 01420 const bool tags, 01421 const bool store_remote_handles, 01422 const int to_proc, 01423 Buffer* buff, 01424 TupleList* entprocs, 01425 Range* allsent ) 01426 { 01427 // Pack the buffer with the entity ranges, adjacencies, and tags sections 01428 // 01429 // Note: new entities used in subsequent connectivity lists, sets, or tags, 01430 // are referred to as (MBMAXTYPE + index), where index is into vector 01431 // of new entities, 0-based 01432 ErrorCode result; 01433 01434 Range set_range; 01435 std::vector< Tag > all_tags; 01436 std::vector< Range > tag_ranges; 01437 01438 Range::const_iterator rit; 01439 01440 // Entities 01441 result = pack_entities( orig_ents, buff, store_remote_handles, to_proc, false, entprocs, allsent );MB_CHK_SET_ERR( result, "Packing entities failed" ); 01442 01443 // Sets 01444 result = pack_sets( orig_ents, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing sets (count) failed" ); 01445 01446 // Tags 01447 Range final_ents; 01448 if( tags ) 01449 { 01450 result = get_tag_send_list( orig_ents, all_tags, tag_ranges );MB_CHK_SET_ERR( result, "Failed to get tagged entities" ); 01451 result = pack_tags( orig_ents, all_tags, all_tags, tag_ranges, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing tags (count) failed" ); 01452 } 01453 else 01454 { // Set tag size to 0 01455 buff->check_space( sizeof( int ) ); 01456 PACK_INT( buff->buff_ptr, 0 ); 01457 buff->set_stored_size(); 01458 } 01459 01460 return result; 01461 } 01462 01463 ErrorCode ParallelComm::unpack_buffer( unsigned char* buff_ptr, 01464 const bool store_remote_handles, 01465 const int from_proc, 01466 const int ind, 01467 std::vector< std::vector< EntityHandle > >& L1hloc, 01468 std::vector< std::vector< EntityHandle > >& L1hrem, 01469 std::vector< std::vector< int > >& L1p, 01470 std::vector< EntityHandle >& L2hloc, 01471 std::vector< EntityHandle >& L2hrem, 01472 std::vector< unsigned int >& L2p, 01473 std::vector< EntityHandle >& new_ents, 01474 const bool created_iface ) 01475 { 01476 unsigned char* tmp_buff = buff_ptr; 01477 ErrorCode result; 01478 result = unpack_entities( buff_ptr, store_remote_handles, ind, false, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, 01479 new_ents, created_iface );MB_CHK_SET_ERR( result, "Unpacking entities failed" ); 01480 if( myDebug->get_verbosity() == 3 ) 01481 { 01482 myDebug->tprintf( 4, "unpack_entities buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01483 tmp_buff = buff_ptr; 01484 } 01485 result = unpack_sets( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking sets failed" ); 01486 if( myDebug->get_verbosity() == 3 ) 01487 { 01488 myDebug->tprintf( 4, "unpack_sets buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01489 tmp_buff = buff_ptr; 01490 } 01491 result = unpack_tags( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking tags failed" ); 01492 if( myDebug->get_verbosity() == 3 ) 01493 { 01494 myDebug->tprintf( 4, "unpack_tags buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) ); 01495 // tmp_buff = buff_ptr; 01496 } 01497 01498 if( myDebug->get_verbosity() == 3 ) myDebug->print( 4, "\n" ); 01499 01500 return MB_SUCCESS; 01501 } 01502 01503 int ParallelComm::estimate_ents_buffer_size( Range& entities, const bool store_remote_handles ) 01504 { 01505 int buff_size = 0; 01506 std::vector< EntityHandle > dum_connect_vec; 01507 const EntityHandle* connect; 01508 int num_connect; 01509 01510 int num_verts = entities.num_of_type( MBVERTEX ); 01511 // # verts + coords + handles 01512 buff_size += 2 * sizeof( int ) + 3 * sizeof( double ) * num_verts; 01513 if( store_remote_handles ) buff_size += sizeof( EntityHandle ) * num_verts; 01514 01515 // Do a rough count by looking at first entity of each type 01516 for( EntityType t = MBEDGE; t < MBENTITYSET; t++ ) 01517 { 01518 const Range::iterator rit = entities.lower_bound( t ); 01519 if( TYPE_FROM_HANDLE( *rit ) != t ) continue; 01520 01521 ErrorCode result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect_vec );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get connectivity to estimate buffer size", -1 ); 01522 01523 // Number, type, nodes per entity 01524 buff_size += 3 * sizeof( int ); 01525 int num_ents = entities.num_of_type( t ); 01526 // Connectivity, handle for each ent 01527 buff_size += ( num_connect + 1 ) * sizeof( EntityHandle ) * num_ents; 01528 } 01529 01530 // Extra entity type at end, passed as int 01531 buff_size += sizeof( int ); 01532 01533 return buff_size; 01534 } 01535 01536 int ParallelComm::estimate_sets_buffer_size( Range& entities, const bool /*store_remote_handles*/ ) 01537 { 01538 // Number of sets 01539 int buff_size = sizeof( int ); 01540 01541 // Do a rough count by looking at first entity of each type 01542 Range::iterator rit = entities.lower_bound( MBENTITYSET ); 01543 ErrorCode result; 01544 01545 for( ; rit != entities.end(); ++rit ) 01546 { 01547 unsigned int options; 01548 result = mbImpl->get_meshset_options( *rit, options );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get meshset options", -1 ); 01549 01550 buff_size += sizeof( int ); 01551 01552 Range set_range; 01553 if( options & MESHSET_SET ) 01554 { 01555 // Range-based set; count the subranges 01556 result = mbImpl->get_entities_by_handle( *rit, set_range );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get set entities", -1 ); 01557 01558 // Set range 01559 buff_size += RANGE_SIZE( set_range ); 01560 } 01561 else if( options & MESHSET_ORDERED ) 01562 { 01563 // Just get the number of entities in the set 01564 int num_ents; 01565 result = mbImpl->get_number_entities_by_handle( *rit, num_ents );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get number entities in ordered set", -1 ); 01566 01567 // Set vec 01568 buff_size += sizeof( EntityHandle ) * num_ents + sizeof( int ); 01569 } 01570 01571 // Get numbers of parents/children 01572 int num_par, num_ch; 01573 result = mbImpl->num_child_meshsets( *rit, &num_ch );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num children", -1 ); 01574 result = mbImpl->num_parent_meshsets( *rit, &num_par );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num parents", -1 ); 01575 01576 buff_size += ( num_ch + num_par ) * sizeof( EntityHandle ) + 2 * sizeof( int ); 01577 } 01578 01579 return buff_size; 01580 } 01581 01582 ErrorCode ParallelComm::pack_entities( Range& entities, 01583 Buffer* buff, 01584 const bool store_remote_handles, 01585 const int to_proc, 01586 const bool /*is_iface*/, 01587 TupleList* entprocs, 01588 Range* /*allsent*/ ) 01589 { 01590 // Packed information: 01591 // 1. # entities = E 01592 // 2. for e in E 01593 // a. # procs sharing e, incl. sender and receiver = P 01594 // b. for p in P (procs sharing e) 01595 // c. for p in P (handle for e on p) (Note1) 01596 // 3. vertex/entity info 01597 01598 // Get an estimate of the buffer size & pre-allocate buffer size 01599 int buff_size = estimate_ents_buffer_size( entities, store_remote_handles ); 01600 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate ents buffer size" ); 01601 buff->check_space( buff_size ); 01602 myDebug->tprintf( 3, "estimate buffer size for %d entities: %d \n", (int)entities.size(), buff_size ); 01603 01604 unsigned int num_ents; 01605 ErrorCode result; 01606 01607 std::vector< EntityHandle > entities_vec( entities.size() ); 01608 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 01609 01610 // First pack procs/handles sharing this ent, not including this dest but including 01611 // others (with zero handles) 01612 if( store_remote_handles ) 01613 { 01614 // Buff space is at least proc + handle for each entity; use avg of 4 other procs 01615 // to estimate buff size, but check later 01616 buff->check_space( sizeof( int ) + ( 5 * sizeof( int ) + sizeof( EntityHandle ) ) * entities.size() ); 01617 01618 // 1. # entities = E 01619 PACK_INT( buff->buff_ptr, entities.size() ); 01620 01621 Range::iterator rit; 01622 01623 // Pre-fetch sharedp and pstatus 01624 std::vector< int > sharedp_vals( entities.size() ); 01625 result = mbImpl->tag_get_data( sharedp_tag(), entities, &sharedp_vals[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 01626 std::vector< char > pstatus_vals( entities.size() ); 01627 result = mbImpl->tag_get_data( pstatus_tag(), entities, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 01628 01629 unsigned int i; 01630 int tmp_procs[MAX_SHARING_PROCS]; 01631 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 01632 std::set< unsigned int > dumprocs; 01633 01634 // 2. for e in E 01635 for( rit = entities.begin(), i = 0; rit != entities.end(); ++rit, i++ ) 01636 { 01637 unsigned int ind = 01638 std::lower_bound( entprocs->vul_rd, entprocs->vul_rd + entprocs->get_n(), *rit ) - entprocs->vul_rd; 01639 assert( ind < entprocs->get_n() ); 01640 01641 while( ind < entprocs->get_n() && entprocs->vul_rd[ind] == *rit ) 01642 dumprocs.insert( entprocs->vi_rd[ind++] ); 01643 01644 result = build_sharedhps_list( *rit, pstatus_vals[i], sharedp_vals[i], dumprocs, num_ents, tmp_procs, 01645 tmp_handles );MB_CHK_SET_ERR( result, "Failed to build sharedhps" ); 01646 01647 dumprocs.clear(); 01648 01649 // Now pack them 01650 buff->check_space( ( num_ents + 1 ) * sizeof( int ) + num_ents * sizeof( EntityHandle ) ); 01651 PACK_INT( buff->buff_ptr, num_ents ); 01652 PACK_INTS( buff->buff_ptr, tmp_procs, num_ents ); 01653 PACK_EH( buff->buff_ptr, tmp_handles, num_ents ); 01654 01655 #ifndef NDEBUG 01656 // Check for duplicates in proc list 01657 unsigned int dp = 0; 01658 for( ; dp < MAX_SHARING_PROCS && -1 != tmp_procs[dp]; dp++ ) 01659 dumprocs.insert( tmp_procs[dp] ); 01660 assert( dumprocs.size() == dp ); 01661 dumprocs.clear(); 01662 #endif 01663 } 01664 } 01665 01666 // Pack vertices 01667 Range these_ents = entities.subset_by_type( MBVERTEX ); 01668 num_ents = these_ents.size(); 01669 01670 if( num_ents ) 01671 { 01672 buff_size = 2 * sizeof( int ) + 3 * num_ents * sizeof( double ); 01673 buff->check_space( buff_size ); 01674 01675 // Type, # ents 01676 PACK_INT( buff->buff_ptr, ( (int)MBVERTEX ) ); 01677 PACK_INT( buff->buff_ptr, ( (int)num_ents ) ); 01678 01679 std::vector< double > tmp_coords( 3 * num_ents ); 01680 result = mbImpl->get_coords( these_ents, &tmp_coords[0] );MB_CHK_SET_ERR( result, "Failed to get vertex coordinates" ); 01681 PACK_DBLS( buff->buff_ptr, &tmp_coords[0], 3 * num_ents ); 01682 01683 myDebug->tprintf( 4, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(), 01684 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01685 } 01686 01687 // Now entities; go through range, packing by type and equal # verts per element 01688 Range::iterator start_rit = entities.find( *these_ents.rbegin() ); 01689 ++start_rit; 01690 int last_nodes = -1; 01691 EntityType last_type = MBMAXTYPE; 01692 these_ents.clear(); 01693 Range::iterator end_rit = start_rit; 01694 EntitySequence* seq; 01695 ElementSequence* eseq; 01696 01697 while( start_rit != entities.end() || !these_ents.empty() ) 01698 { 01699 // Cases: 01700 // A: !end, last_type == MBMAXTYPE, seq: save contig sequence in these_ents 01701 // B: !end, last type & nodes same, seq: save contig sequence in these_ents 01702 // C: !end, last type & nodes different: pack these_ents, then save contig sequence in 01703 // these_ents D: end: pack these_ents 01704 01705 // Find the sequence holding current start entity, if we're not at end 01706 eseq = NULL; 01707 if( start_rit != entities.end() ) 01708 { 01709 result = sequenceManager->find( *start_rit, seq );MB_CHK_SET_ERR( result, "Failed to find entity sequence" ); 01710 if( NULL == seq ) return MB_FAILURE; 01711 eseq = dynamic_cast< ElementSequence* >( seq ); 01712 } 01713 01714 // Pack the last batch if at end or next one is different 01715 if( !these_ents.empty() && 01716 ( !eseq || eseq->type() != last_type || last_nodes != (int)eseq->nodes_per_element() ) ) 01717 { 01718 result = pack_entity_seq( last_nodes, store_remote_handles, to_proc, these_ents, entities_vec, buff );MB_CHK_SET_ERR( result, "Failed to pack entities from a sequence" ); 01719 these_ents.clear(); 01720 } 01721 01722 if( eseq ) 01723 { 01724 // Continuation of current range, just save these entities 01725 // Get position in entities list one past end of this sequence 01726 end_rit = entities.lower_bound( start_rit, entities.end(), eseq->end_handle() + 1 ); 01727 01728 // Put these entities in the range 01729 std::copy( start_rit, end_rit, range_inserter( these_ents ) ); 01730 01731 last_type = eseq->type(); 01732 last_nodes = eseq->nodes_per_element(); 01733 } 01734 else if( start_rit != entities.end() && TYPE_FROM_HANDLE( *start_rit ) == MBENTITYSET ) 01735 break; 01736 01737 start_rit = end_rit; 01738 } 01739 01740 // Pack MBMAXTYPE to indicate end of ranges 01741 buff->check_space( sizeof( int ) ); 01742 PACK_INT( buff->buff_ptr, ( (int)MBMAXTYPE ) ); 01743 01744 buff->set_stored_size(); 01745 return MB_SUCCESS; 01746 } 01747 01748 ErrorCode ParallelComm::build_sharedhps_list( const EntityHandle entity, 01749 const unsigned char pstatus, 01750 const int 01751 #ifndef NDEBUG 01752 sharedp 01753 #endif 01754 , 01755 const std::set< unsigned int >& procs, 01756 unsigned int& num_ents, 01757 int* tmp_procs, 01758 EntityHandle* tmp_handles ) 01759 { 01760 num_ents = 0; 01761 unsigned char pstat; 01762 ErrorCode result = get_sharing_data( entity, tmp_procs, tmp_handles, pstat, num_ents );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 01763 assert( pstat == pstatus ); 01764 01765 // Build shared proc/handle lists 01766 // Start with multi-shared, since if it is the owner will be first 01767 if( pstatus & PSTATUS_MULTISHARED ) 01768 { 01769 } 01770 else if( pstatus & PSTATUS_NOT_OWNED ) 01771 { 01772 // If not multishared and not owned, other sharing proc is owner, put that 01773 // one first 01774 assert( "If not owned, I should be shared too" && pstatus & PSTATUS_SHARED && 1 == num_ents ); 01775 tmp_procs[1] = procConfig.proc_rank(); 01776 tmp_handles[1] = entity; 01777 num_ents = 2; 01778 } 01779 else if( pstatus & PSTATUS_SHARED ) 01780 { 01781 // If not multishared and owned, I'm owner 01782 assert( "shared and owned, should be only 1 sharing proc" && 1 == num_ents ); 01783 tmp_procs[1] = tmp_procs[0]; 01784 tmp_procs[0] = procConfig.proc_rank(); 01785 tmp_handles[1] = tmp_handles[0]; 01786 tmp_handles[0] = entity; 01787 num_ents = 2; 01788 } 01789 else 01790 { 01791 // Not shared yet, just add owner (me) 01792 tmp_procs[0] = procConfig.proc_rank(); 01793 tmp_handles[0] = entity; 01794 num_ents = 1; 01795 } 01796 01797 #ifndef NDEBUG 01798 int tmp_ps = num_ents; 01799 #endif 01800 01801 // Now add others, with zero handle for now 01802 for( std::set< unsigned int >::iterator sit = procs.begin(); sit != procs.end(); ++sit ) 01803 { 01804 #ifndef NDEBUG 01805 if( tmp_ps && std::find( tmp_procs, tmp_procs + tmp_ps, *sit ) != tmp_procs + tmp_ps ) 01806 { 01807 std::cerr << "Trouble with something already in shared list on proc " << procConfig.proc_rank() 01808 << ". Entity:" << std::endl; 01809 list_entities( &entity, 1 ); 01810 std::cerr << "pstatus = " << (int)pstatus << ", sharedp = " << sharedp << std::endl; 01811 std::cerr << "tmp_ps = "; 01812 for( int i = 0; i < tmp_ps; i++ ) 01813 std::cerr << tmp_procs[i] << " "; 01814 std::cerr << std::endl; 01815 std::cerr << "procs = "; 01816 for( std::set< unsigned int >::iterator sit2 = procs.begin(); sit2 != procs.end(); ++sit2 ) 01817 std::cerr << *sit2 << " "; 01818 assert( false ); 01819 } 01820 #endif 01821 tmp_procs[num_ents] = *sit; 01822 tmp_handles[num_ents] = 0; 01823 num_ents++; 01824 } 01825 01826 // Put -1 after procs and 0 after handles 01827 if( MAX_SHARING_PROCS > num_ents ) 01828 { 01829 tmp_procs[num_ents] = -1; 01830 tmp_handles[num_ents] = 0; 01831 } 01832 01833 return MB_SUCCESS; 01834 } 01835 01836 ErrorCode ParallelComm::pack_entity_seq( const int nodes_per_entity, 01837 const bool store_remote_handles, 01838 const int to_proc, 01839 Range& these_ents, 01840 std::vector< EntityHandle >& entities_vec, 01841 Buffer* buff ) 01842 { 01843 int tmp_space = 3 * sizeof( int ) + nodes_per_entity * these_ents.size() * sizeof( EntityHandle ); 01844 buff->check_space( tmp_space ); 01845 01846 // Pack the entity type 01847 PACK_INT( buff->buff_ptr, ( (int)TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01848 01849 // Pack # ents 01850 PACK_INT( buff->buff_ptr, these_ents.size() ); 01851 01852 // Pack the nodes per entity 01853 PACK_INT( buff->buff_ptr, nodes_per_entity ); 01854 myDebug->tprintf( 3, "after some pack int %d \n", buff->get_current_size() ); 01855 01856 // Pack the connectivity 01857 std::vector< EntityHandle > connect; 01858 ErrorCode result = MB_SUCCESS; 01859 for( Range::const_iterator rit = these_ents.begin(); rit != these_ents.end(); ++rit ) 01860 { 01861 connect.clear(); 01862 result = mbImpl->get_connectivity( &( *rit ), 1, connect, false );MB_CHK_SET_ERR( result, "Failed to get connectivity" ); 01863 assert( (int)connect.size() == nodes_per_entity ); 01864 result = 01865 get_remote_handles( store_remote_handles, &connect[0], &connect[0], connect.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" ); 01866 PACK_EH( buff->buff_ptr, &connect[0], connect.size() ); 01867 } 01868 01869 myDebug->tprintf( 3, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(), 01870 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) ); 01871 01872 return result; 01873 } 01874 01875 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 01876 EntityHandle* from_vec, 01877 EntityHandle* to_vec_tmp, 01878 int num_ents, 01879 int to_proc, 01880 const std::vector< EntityHandle >& new_ents ) 01881 { 01882 // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE RANGE-BASED VERSION, NO REUSE 01883 // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE 01884 // OTHER VERSION TOO!!! 01885 if( 0 == num_ents ) return MB_SUCCESS; 01886 01887 // Use a local destination ptr in case we're doing an in-place copy 01888 std::vector< EntityHandle > tmp_vector; 01889 EntityHandle* to_vec = to_vec_tmp; 01890 if( to_vec == from_vec ) 01891 { 01892 tmp_vector.resize( num_ents ); 01893 to_vec = &tmp_vector[0]; 01894 } 01895 01896 if( !store_remote_handles ) 01897 { 01898 int err; 01899 // In this case, substitute position in new_ents list 01900 for( int i = 0; i < num_ents; i++ ) 01901 { 01902 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin(); 01903 assert( new_ents[ind] == from_vec[i] ); 01904 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err ); 01905 assert( to_vec[i] != 0 && !err && -1 != ind ); 01906 } 01907 } 01908 else 01909 { 01910 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 01911 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 01912 01913 // Get single-proc destination handles and shared procs 01914 std::vector< int > sharing_procs( num_ents ); 01915 result = mbImpl->tag_get_data( shh_tag, from_vec, num_ents, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" ); 01916 result = mbImpl->tag_get_data( shp_tag, from_vec, num_ents, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" ); 01917 for( int j = 0; j < num_ents; j++ ) 01918 { 01919 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0; 01920 } 01921 01922 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 01923 int tmp_procs[MAX_SHARING_PROCS]; 01924 int i; 01925 // Go through results, and for 0-valued ones, look for multiple shared proc 01926 for( i = 0; i < num_ents; i++ ) 01927 { 01928 if( !to_vec[i] ) 01929 { 01930 result = mbImpl->tag_get_data( shps_tag, from_vec + i, 1, tmp_procs ); 01931 if( MB_SUCCESS == result ) 01932 { 01933 for( int j = 0; j < MAX_SHARING_PROCS; j++ ) 01934 { 01935 if( -1 == tmp_procs[j] ) 01936 break; 01937 else if( tmp_procs[j] == to_proc ) 01938 { 01939 result = mbImpl->tag_get_data( shhs_tag, from_vec + i, 1, tmp_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 01940 to_vec[i] = tmp_handles[j]; 01941 assert( to_vec[i] ); 01942 break; 01943 } 01944 } 01945 } 01946 if( !to_vec[i] ) 01947 { 01948 int j = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin(); 01949 if( (int)new_ents.size() == j ) 01950 { 01951 std::cout << "Failed to find new entity in send list, proc " << procConfig.proc_rank() 01952 << std::endl; 01953 for( int k = 0; k <= num_ents; k++ ) 01954 std::cout << k << ": " << from_vec[k] << " " << to_vec[k] << std::endl; 01955 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" ); 01956 } 01957 int err; 01958 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err ); 01959 if( err ) 01960 { 01961 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" ); 01962 } 01963 } 01964 } 01965 } 01966 } 01967 01968 // memcpy over results if from_vec and to_vec are the same 01969 if( to_vec_tmp == from_vec ) memcpy( from_vec, to_vec, num_ents * sizeof( EntityHandle ) ); 01970 01971 return MB_SUCCESS; 01972 } 01973 01974 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 01975 const Range& from_range, 01976 EntityHandle* to_vec, 01977 int to_proc, 01978 const std::vector< EntityHandle >& new_ents ) 01979 { 01980 // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE VECTOR-BASED VERSION, NO REUSE 01981 // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE 01982 // OTHER VERSION TOO!!! 01983 if( from_range.empty() ) return MB_SUCCESS; 01984 01985 if( !store_remote_handles ) 01986 { 01987 int err; 01988 // In this case, substitute position in new_ents list 01989 Range::iterator rit; 01990 unsigned int i; 01991 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ ) 01992 { 01993 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin(); 01994 assert( new_ents[ind] == *rit ); 01995 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err ); 01996 assert( to_vec[i] != 0 && !err && -1 != ind ); 01997 } 01998 } 01999 else 02000 { 02001 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 02002 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 02003 02004 // Get single-proc destination handles and shared procs 02005 std::vector< int > sharing_procs( from_range.size() ); 02006 result = mbImpl->tag_get_data( shh_tag, from_range, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" ); 02007 result = mbImpl->tag_get_data( shp_tag, from_range, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" ); 02008 for( unsigned int j = 0; j < from_range.size(); j++ ) 02009 { 02010 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0; 02011 } 02012 02013 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 02014 int tmp_procs[MAX_SHARING_PROCS]; 02015 // Go through results, and for 0-valued ones, look for multiple shared proc 02016 Range::iterator rit; 02017 unsigned int i; 02018 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ ) 02019 { 02020 if( !to_vec[i] ) 02021 { 02022 result = mbImpl->tag_get_data( shhs_tag, &( *rit ), 1, tmp_handles ); 02023 if( MB_SUCCESS == result ) 02024 { 02025 result = mbImpl->tag_get_data( shps_tag, &( *rit ), 1, tmp_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" ); 02026 for( int j = 0; j < MAX_SHARING_PROCS; j++ ) 02027 if( tmp_procs[j] == to_proc ) 02028 { 02029 to_vec[i] = tmp_handles[j]; 02030 break; 02031 } 02032 } 02033 02034 if( !to_vec[i] ) 02035 { 02036 int j = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin(); 02037 if( (int)new_ents.size() == j ) 02038 { 02039 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" ); 02040 } 02041 int err; 02042 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err ); 02043 if( err ) 02044 { 02045 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" ); 02046 } 02047 } 02048 } 02049 } 02050 } 02051 02052 return MB_SUCCESS; 02053 } 02054 02055 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles, 02056 const Range& from_range, 02057 Range& to_range, 02058 int to_proc, 02059 const std::vector< EntityHandle >& new_ents ) 02060 { 02061 std::vector< EntityHandle > to_vector( from_range.size() ); 02062 02063 ErrorCode result = get_remote_handles( store_remote_handles, from_range, &to_vector[0], to_proc, new_ents );MB_CHK_SET_ERR( result, "Failed to get remote handles" ); 02064 std::copy( to_vector.begin(), to_vector.end(), range_inserter( to_range ) ); 02065 return result; 02066 } 02067 02068 ErrorCode ParallelComm::unpack_entities( unsigned char*& buff_ptr, 02069 const bool store_remote_handles, 02070 const int /*from_ind*/, 02071 const bool is_iface, 02072 std::vector< std::vector< EntityHandle > >& L1hloc, 02073 std::vector< std::vector< EntityHandle > >& L1hrem, 02074 std::vector< std::vector< int > >& L1p, 02075 std::vector< EntityHandle >& L2hloc, 02076 std::vector< EntityHandle >& L2hrem, 02077 std::vector< unsigned int >& L2p, 02078 std::vector< EntityHandle >& new_ents, 02079 const bool created_iface ) 02080 { 02081 // General algorithm: 02082 // - unpack # entities 02083 // - save start of remote handle info, then scan forward to entity definition data 02084 // - for all vertices or entities w/ same # verts: 02085 // . get entity type, num ents, and (if !vert) # verts 02086 // . for each ent: 02087 // o get # procs/handles in remote handle info 02088 // o if # procs/handles > 2, check for already-created entity: 02089 // x get index of owner proc (1st in proc list), resize L1 list if nec 02090 // x look for already-arrived entity in L2 by owner handle 02091 // o if no existing entity: 02092 // x if iface, look for existing entity with same connect & type 02093 // x if none found, create vertex or element 02094 // x if !iface & multi-shared, save on L2 02095 // x if !iface, put new entity on new_ents list 02096 // o update proc/handle, pstatus tags, adjusting to put owner first if iface 02097 // o if !iface, save new handle on L1 for all sharing procs 02098 02099 // Lists of handles/procs to return to sending/other procs 02100 // L1hloc[p], L1hrem[p]: handle pairs [h, h'], where h is the local proc handle 02101 // and h' is either the remote proc handle (if that is known) or 02102 // the owner proc handle (otherwise); 02103 // L1p[p]: indicates whether h is remote handle (= -1) or owner (rank of owner) 02104 // L2hloc, L2hrem: local/remote handles for entities shared by > 2 procs; 02105 // remote handles are on owning proc 02106 // L2p: owning procs for handles in L2hrem 02107 02108 ErrorCode result; 02109 bool done = false; 02110 ReadUtilIface* ru = NULL; 02111 02112 result = mbImpl->query_interface( ru );MB_CHK_SET_ERR( result, "Failed to get ReadUtilIface" ); 02113 02114 // 1. # entities = E 02115 int num_ents = 0; 02116 unsigned char* buff_save = buff_ptr; 02117 int i, j; 02118 02119 if( store_remote_handles ) 02120 { 02121 UNPACK_INT( buff_ptr, num_ents ); 02122 02123 buff_save = buff_ptr; 02124 02125 // Save place where remote handle info starts, then scan forward to ents 02126 for( i = 0; i < num_ents; i++ ) 02127 { 02128 UNPACK_INT( buff_ptr, j ); 02129 if( j < 0 ) 02130 { 02131 std::cout << "Should be non-negative # proc/handles."; 02132 return MB_FAILURE; 02133 } 02134 02135 buff_ptr += j * ( sizeof( int ) + sizeof( EntityHandle ) ); 02136 } 02137 } 02138 02139 std::vector< EntityHandle > msg_ents; 02140 02141 while( !done ) 02142 { 02143 EntityType this_type = MBMAXTYPE; 02144 UNPACK_TYPE( buff_ptr, this_type ); 02145 assert( this_type != MBENTITYSET ); 02146 02147 // MBMAXTYPE signifies end of entities data 02148 if( MBMAXTYPE == this_type ) break; 02149 02150 // Get the number of ents 02151 int num_ents2, verts_per_entity = 0; 02152 UNPACK_INT( buff_ptr, num_ents2 ); 02153 02154 // Unpack the nodes per entity 02155 if( MBVERTEX != this_type && num_ents2 ) 02156 { 02157 UNPACK_INT( buff_ptr, verts_per_entity ); 02158 } 02159 02160 std::vector< int > ps( MAX_SHARING_PROCS, -1 ); 02161 std::vector< EntityHandle > hs( MAX_SHARING_PROCS, 0 ); 02162 for( int e = 0; e < num_ents2; e++ ) 02163 { 02164 // Check for existing entity, otherwise make new one 02165 EntityHandle new_h = 0; 02166 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT]; 02167 double coords[3]; 02168 int num_ps = -1; 02169 02170 //======================================= 02171 // Unpack all the data at once, to make sure the buffer pointers 02172 // are tracked correctly 02173 //======================================= 02174 if( store_remote_handles ) 02175 { 02176 // Pointers to other procs/handles 02177 UNPACK_INT( buff_save, num_ps ); 02178 if( 0 >= num_ps ) 02179 { 02180 std::cout << "Shouldn't ever be fewer than 1 procs here." << std::endl; 02181 return MB_FAILURE; 02182 } 02183 02184 UNPACK_INTS( buff_save, &ps[0], num_ps ); 02185 UNPACK_EH( buff_save, &hs[0], num_ps ); 02186 } 02187 02188 if( MBVERTEX == this_type ) 02189 { 02190 UNPACK_DBLS( buff_ptr, coords, 3 ); 02191 } 02192 else 02193 { 02194 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT ); 02195 UNPACK_EH( buff_ptr, connect, verts_per_entity ); 02196 02197 // Update connectivity to local handles 02198 result = get_local_handles( connect, verts_per_entity, msg_ents );MB_CHK_SET_ERR( result, "Failed to get local handles" ); 02199 } 02200 02201 //======================================= 02202 // Now, process that data; begin by finding an identical 02203 // entity, if there is one 02204 //======================================= 02205 if( store_remote_handles ) 02206 { 02207 result = find_existing_entity( is_iface, ps[0], hs[0], num_ps, connect, verts_per_entity, this_type, 02208 L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Failed to get existing entity" ); 02209 } 02210 02211 //======================================= 02212 // If we didn't find one, we'll have to create one 02213 //======================================= 02214 bool created_here = false; 02215 if( !new_h && !is_iface ) 02216 { 02217 if( MBVERTEX == this_type ) 02218 { 02219 // Create a vertex 02220 result = mbImpl->create_vertex( coords, new_h );MB_CHK_SET_ERR( result, "Failed to make new vertex" ); 02221 } 02222 else 02223 { 02224 // Create the element 02225 result = mbImpl->create_element( this_type, connect, verts_per_entity, new_h );MB_CHK_SET_ERR( result, "Failed to make new element" ); 02226 02227 // Update adjacencies 02228 result = ru->update_adjacencies( new_h, 1, verts_per_entity, connect );MB_CHK_SET_ERR( result, "Failed to update adjacencies" ); 02229 } 02230 02231 // Should have a new handle now 02232 assert( new_h ); 02233 02234 created_here = true; 02235 } 02236 02237 //======================================= 02238 // Take care of sharing data 02239 //======================================= 02240 02241 // Need to save entities found in order, for interpretation of 02242 // later parts of this message 02243 if( !is_iface ) 02244 { 02245 assert( new_h ); 02246 msg_ents.push_back( new_h ); 02247 } 02248 02249 if( created_here ) new_ents.push_back( new_h ); 02250 02251 if( new_h && store_remote_handles ) 02252 { 02253 unsigned char new_pstat = 0x0; 02254 if( is_iface ) 02255 { 02256 new_pstat = PSTATUS_INTERFACE; 02257 // Here, lowest rank proc should be first 02258 int idx = std::min_element( &ps[0], &ps[0] + num_ps ) - &ps[0]; 02259 if( idx ) 02260 { 02261 std::swap( ps[0], ps[idx] ); 02262 std::swap( hs[0], hs[idx] ); 02263 } 02264 // Set ownership based on lowest rank; can't be in update_remote_data, because 02265 // there we don't know whether it resulted from ghosting or not 02266 if( ( num_ps > 1 && ps[0] != (int)rank() ) ) new_pstat |= PSTATUS_NOT_OWNED; 02267 } 02268 else if( created_here ) 02269 { 02270 if( created_iface ) 02271 new_pstat = PSTATUS_NOT_OWNED; 02272 else 02273 new_pstat = PSTATUS_GHOST | PSTATUS_NOT_OWNED; 02274 } 02275 02276 // Update sharing data and pstatus, adjusting order if iface 02277 result = update_remote_data( new_h, &ps[0], &hs[0], num_ps, new_pstat );MB_CHK_SET_ERR( result, "unpack_entities" ); 02278 02279 // If a new multi-shared entity, save owner for subsequent lookup in L2 lists 02280 if( store_remote_handles && !is_iface && num_ps > 2 ) 02281 { 02282 L2hrem.push_back( hs[0] ); 02283 L2hloc.push_back( new_h ); 02284 L2p.push_back( ps[0] ); 02285 } 02286 02287 // Need to send this new handle to all sharing procs 02288 if( !is_iface ) 02289 { 02290 for( j = 0; j < num_ps; j++ ) 02291 { 02292 if( ps[j] == (int)procConfig.proc_rank() ) continue; 02293 int idx = get_buffers( ps[j] ); 02294 if( idx == (int)L1hloc.size() ) 02295 { 02296 L1hloc.resize( idx + 1 ); 02297 L1hrem.resize( idx + 1 ); 02298 L1p.resize( idx + 1 ); 02299 } 02300 02301 // Don't bother adding if it's already in the list 02302 std::vector< EntityHandle >::iterator vit = 02303 std::find( L1hloc[idx].begin(), L1hloc[idx].end(), new_h ); 02304 if( vit != L1hloc[idx].end() ) 02305 { 02306 // If it's in the list but remote handle isn't known but we know 02307 // it, replace in the list 02308 if( L1p[idx][vit - L1hloc[idx].begin()] != -1 && hs[j] ) 02309 { 02310 L1hrem[idx][vit - L1hloc[idx].begin()] = hs[j]; 02311 L1p[idx][vit - L1hloc[idx].begin()] = -1; 02312 } 02313 else 02314 continue; 02315 } 02316 else 02317 { 02318 if( !hs[j] ) 02319 { 02320 assert( -1 != ps[0] && num_ps > 2 ); 02321 L1p[idx].push_back( ps[0] ); 02322 L1hrem[idx].push_back( hs[0] ); 02323 } 02324 else 02325 { 02326 assert( 02327 "either this remote handle isn't in the remote list, or " 02328 "it's for another proc" && 02329 ( std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) == L1hrem[idx].end() || 02330 L1p[idx][std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) - 02331 L1hrem[idx].begin()] != -1 ) ); 02332 L1p[idx].push_back( -1 ); 02333 L1hrem[idx].push_back( hs[j] ); 02334 } 02335 L1hloc[idx].push_back( new_h ); 02336 } 02337 } 02338 } 02339 02340 assert( "Shouldn't be here for non-shared entities" && -1 != num_ps ); 02341 std::fill( &ps[0], &ps[num_ps], -1 ); 02342 std::fill( &hs[0], &hs[num_ps], 0 ); 02343 } 02344 } 02345 02346 myDebug->tprintf( 4, "Unpacked %d ents of type %s", num_ents2, CN::EntityTypeName( this_type ) ); 02347 } 02348 02349 myDebug->tprintf( 4, "Done unpacking entities.\n" ); 02350 02351 // Need to sort here, to enable searching 02352 std::sort( new_ents.begin(), new_ents.end() ); 02353 02354 return MB_SUCCESS; 02355 } 02356 02357 ErrorCode ParallelComm::print_buffer( unsigned char* buff_ptr, int mesg_tag, int from_proc, bool sent ) 02358 { 02359 std::cerr << procConfig.proc_rank(); 02360 if( sent ) 02361 std::cerr << " sent"; 02362 else 02363 std::cerr << " received"; 02364 std::cerr << " message type " << mesg_tag << " to/from proc " << from_proc << "; contents:" << std::endl; 02365 02366 int msg_length, num_ents; 02367 unsigned char* orig_ptr = buff_ptr; 02368 UNPACK_INT( buff_ptr, msg_length ); 02369 std::cerr << msg_length << " bytes..." << std::endl; 02370 02371 if( MB_MESG_ENTS_SIZE == mesg_tag || MB_MESG_ENTS_LARGE == mesg_tag ) 02372 { 02373 // 1. # entities = E 02374 int i, j, k; 02375 std::vector< int > ps; 02376 std::vector< EntityHandle > hs; 02377 02378 UNPACK_INT( buff_ptr, num_ents ); 02379 std::cerr << num_ents << " entities..." << std::endl; 02380 02381 // Save place where remote handle info starts, then scan forward to ents 02382 for( i = 0; i < num_ents; i++ ) 02383 { 02384 UNPACK_INT( buff_ptr, j ); 02385 if( 0 > j ) return MB_FAILURE; 02386 ps.resize( j ); 02387 hs.resize( j ); 02388 std::cerr << "Entity " << i << ", # procs = " << j << std::endl; 02389 UNPACK_INTS( buff_ptr, &ps[0], j ); 02390 UNPACK_EH( buff_ptr, &hs[0], j ); 02391 std::cerr << " Procs: "; 02392 for( k = 0; k < j; k++ ) 02393 std::cerr << ps[k] << " "; 02394 std::cerr << std::endl; 02395 std::cerr << " Handles: "; 02396 for( k = 0; k < j; k++ ) 02397 std::cerr << hs[k] << " "; 02398 std::cerr << std::endl; 02399 02400 if( buff_ptr - orig_ptr > msg_length ) 02401 { 02402 std::cerr << "End of buffer..." << std::endl; 02403 std::cerr.flush(); 02404 return MB_FAILURE; 02405 } 02406 } 02407 02408 while( true ) 02409 { 02410 EntityType this_type = MBMAXTYPE; 02411 UNPACK_TYPE( buff_ptr, this_type ); 02412 assert( this_type != MBENTITYSET ); 02413 02414 // MBMAXTYPE signifies end of entities data 02415 if( MBMAXTYPE == this_type ) break; 02416 02417 // Get the number of ents 02418 int num_ents2, verts_per_entity = 0; 02419 UNPACK_INT( buff_ptr, num_ents2 ); 02420 02421 // Unpack the nodes per entity 02422 if( MBVERTEX != this_type && num_ents2 ) 02423 { 02424 UNPACK_INT( buff_ptr, verts_per_entity ); 02425 } 02426 02427 std::cerr << "Type: " << CN::EntityTypeName( this_type ) << "; num_ents = " << num_ents2; 02428 if( MBVERTEX != this_type ) std::cerr << "; verts_per_ent = " << verts_per_entity; 02429 std::cerr << std::endl; 02430 if( num_ents2 < 0 || num_ents2 > msg_length ) 02431 { 02432 std::cerr << "Wrong number of entities, returning." << std::endl; 02433 return MB_FAILURE; 02434 } 02435 02436 for( int e = 0; e < num_ents2; e++ ) 02437 { 02438 // Check for existing entity, otherwise make new one 02439 if( MBVERTEX == this_type ) 02440 { 02441 double coords[3]; 02442 UNPACK_DBLS( buff_ptr, coords, 3 ); 02443 std::cerr << "xyz = " << coords[0] << ", " << coords[1] << ", " << coords[2] << std::endl; 02444 } 02445 else 02446 { 02447 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT]; 02448 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT ); 02449 UNPACK_EH( buff_ptr, connect, verts_per_entity ); 02450 02451 // Update connectivity to local handles 02452 std::cerr << "Connectivity: "; 02453 for( k = 0; k < verts_per_entity; k++ ) 02454 std::cerr << connect[k] << " "; 02455 std::cerr << std::endl; 02456 } 02457 02458 if( buff_ptr - orig_ptr > msg_length ) 02459 { 02460 std::cerr << "End of buffer..." << std::endl; 02461 std::cerr.flush(); 02462 return MB_FAILURE; 02463 } 02464 } 02465 } 02466 } 02467 else if( MB_MESG_REMOTEH_SIZE == mesg_tag || MB_MESG_REMOTEH_LARGE == mesg_tag ) 02468 { 02469 UNPACK_INT( buff_ptr, num_ents ); 02470 std::cerr << num_ents << " entities..." << std::endl; 02471 if( 0 > num_ents || num_ents > msg_length ) 02472 { 02473 std::cerr << "Wrong number of entities, returning." << std::endl; 02474 return MB_FAILURE; 02475 } 02476 std::vector< EntityHandle > L1hloc( num_ents ), L1hrem( num_ents ); 02477 std::vector< int > L1p( num_ents ); 02478 UNPACK_INTS( buff_ptr, &L1p[0], num_ents ); 02479 UNPACK_EH( buff_ptr, &L1hrem[0], num_ents ); 02480 UNPACK_EH( buff_ptr, &L1hloc[0], num_ents ); 02481 std::cerr << num_ents << " Entity pairs; hremote/hlocal/proc: " << std::endl; 02482 for( int i = 0; i < num_ents; i++ ) 02483 { 02484 EntityType etype = TYPE_FROM_HANDLE( L1hloc[i] ); 02485 std::cerr << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hrem[i] ) << ", " 02486 << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hloc[i] ) << ", " << L1p[i] << std::endl; 02487 } 02488 02489 if( buff_ptr - orig_ptr > msg_length ) 02490 { 02491 std::cerr << "End of buffer..." << std::endl; 02492 std::cerr.flush(); 02493 return MB_FAILURE; 02494 } 02495 } 02496 else if( mesg_tag == MB_MESG_TAGS_SIZE || mesg_tag == MB_MESG_TAGS_LARGE ) 02497 { 02498 int num_tags, dum1, data_type, tag_size; 02499 UNPACK_INT( buff_ptr, num_tags ); 02500 std::cerr << "Number of tags = " << num_tags << std::endl; 02501 for( int i = 0; i < num_tags; i++ ) 02502 { 02503 std::cerr << "Tag " << i << ":" << std::endl; 02504 UNPACK_INT( buff_ptr, tag_size ); 02505 UNPACK_INT( buff_ptr, dum1 ); 02506 UNPACK_INT( buff_ptr, data_type ); 02507 std::cerr << "Tag size, type, data type = " << tag_size << ", " << dum1 << ", " << data_type << std::endl; 02508 UNPACK_INT( buff_ptr, dum1 ); 02509 std::cerr << "Default value size = " << dum1 << std::endl; 02510 buff_ptr += dum1; 02511 UNPACK_INT( buff_ptr, dum1 ); 02512 std::string name( (char*)buff_ptr, dum1 ); 02513 std::cerr << "Tag name = " << name.c_str() << std::endl; 02514 buff_ptr += dum1; 02515 UNPACK_INT( buff_ptr, num_ents ); 02516 std::cerr << "Number of ents = " << num_ents << std::endl; 02517 std::vector< EntityHandle > tmp_buff( num_ents ); 02518 UNPACK_EH( buff_ptr, &tmp_buff[0], num_ents ); 02519 int tot_length = 0; 02520 for( int j = 0; j < num_ents; j++ ) 02521 { 02522 EntityType etype = TYPE_FROM_HANDLE( tmp_buff[j] ); 02523 std::cerr << CN::EntityTypeName( etype ) << " " << ID_FROM_HANDLE( tmp_buff[j] ) << ", tag = "; 02524 if( tag_size == MB_VARIABLE_LENGTH ) 02525 { 02526 UNPACK_INT( buff_ptr, dum1 ); 02527 tot_length += dum1; 02528 std::cerr << "(variable, length = " << dum1 << ")" << std::endl; 02529 } 02530 else if( data_type == MB_TYPE_DOUBLE ) 02531 { 02532 double dum_dbl; 02533 UNPACK_DBL( buff_ptr, dum_dbl ); 02534 std::cerr << dum_dbl << std::endl; 02535 } 02536 else if( data_type == MB_TYPE_INTEGER ) 02537 { 02538 int dum_int; 02539 UNPACK_INT( buff_ptr, dum_int ); 02540 std::cerr << dum_int << std::endl; 02541 } 02542 else if( data_type == MB_TYPE_OPAQUE ) 02543 { 02544 std::cerr << "(opaque)" << std::endl; 02545 buff_ptr += tag_size; 02546 } 02547 else if( data_type == MB_TYPE_HANDLE ) 02548 { 02549 EntityHandle dum_eh; 02550 UNPACK_EH( buff_ptr, &dum_eh, 1 ); 02551 std::cerr << dum_eh << std::endl; 02552 } 02553 else if( data_type == MB_TYPE_BIT ) 02554 { 02555 std::cerr << "(bit)" << std::endl; 02556 buff_ptr += tag_size; 02557 } 02558 } 02559 if( tag_size == MB_VARIABLE_LENGTH ) buff_ptr += tot_length; 02560 } 02561 } 02562 else 02563 { 02564 assert( false ); 02565 return MB_FAILURE; 02566 } 02567 02568 std::cerr.flush(); 02569 02570 return MB_SUCCESS; 02571 } 02572 02573 ErrorCode ParallelComm::list_entities( const EntityHandle* ents, int num_ents ) 02574 { 02575 if( NULL == ents ) 02576 { 02577 Range shared_ents; 02578 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) ); 02579 shared_ents.print( "Shared entities:\n" ); 02580 return MB_SUCCESS; 02581 } 02582 02583 unsigned char pstat; 02584 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 02585 int tmp_procs[MAX_SHARING_PROCS]; 02586 unsigned int num_ps; 02587 ErrorCode result; 02588 02589 for( int i = 0; i < num_ents; i++ ) 02590 { 02591 result = mbImpl->list_entities( ents + i, 1 );MB_CHK_ERR( result ); 02592 double coords[3]; 02593 result = mbImpl->get_coords( ents + i, 1, coords ); 02594 std::cout << " coords: " << coords[0] << " " << coords[1] << " " << coords[2] << "\n"; 02595 02596 result = get_sharing_data( ents[i], tmp_procs, tmp_handles, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 02597 02598 std::cout << "Pstatus: "; 02599 if( !num_ps ) 02600 std::cout << "local " << std::endl; 02601 else 02602 { 02603 if( pstat & PSTATUS_NOT_OWNED ) std::cout << "NOT_OWNED; "; 02604 if( pstat & PSTATUS_SHARED ) std::cout << "SHARED; "; 02605 if( pstat & PSTATUS_MULTISHARED ) std::cout << "MULTISHARED; "; 02606 if( pstat & PSTATUS_INTERFACE ) std::cout << "INTERFACE; "; 02607 if( pstat & PSTATUS_GHOST ) std::cout << "GHOST; "; 02608 std::cout << std::endl; 02609 for( unsigned int j = 0; j < num_ps; j++ ) 02610 { 02611 std::cout << " proc " << tmp_procs[j] << " id (handle) " << mbImpl->id_from_handle( tmp_handles[j] ) 02612 << "(" << tmp_handles[j] << ")" << std::endl; 02613 } 02614 } 02615 std::cout << std::endl; 02616 } 02617 02618 return MB_SUCCESS; 02619 } 02620 02621 ErrorCode ParallelComm::list_entities( const Range& ents ) 02622 { 02623 for( Range::iterator rit = ents.begin(); rit != ents.end(); ++rit ) 02624 list_entities( &( *rit ), 1 ); 02625 02626 return MB_SUCCESS; 02627 } 02628 02629 ErrorCode ParallelComm::update_remote_data( Range& local_range, 02630 Range& remote_range, 02631 int other_proc, 02632 const unsigned char add_pstat ) 02633 { 02634 Range::iterator rit, rit2; 02635 ErrorCode result = MB_SUCCESS; 02636 02637 // For each pair of local/remote handles: 02638 for( rit = local_range.begin(), rit2 = remote_range.begin(); rit != local_range.end(); ++rit, ++rit2 ) 02639 { 02640 result = update_remote_data( *rit, &other_proc, &( *rit2 ), 1, add_pstat );MB_CHK_ERR( result ); 02641 } 02642 02643 return MB_SUCCESS; 02644 } 02645 02646 ErrorCode ParallelComm::update_remote_data( const EntityHandle new_h, 02647 const int* ps, 02648 const EntityHandle* hs, 02649 const int num_ps, 02650 const unsigned char add_pstat 02651 // The following lines left in for future debugging, at least until I trust 02652 // this function; tjt, 10/4/2013 02653 // , int *new_ps, 02654 // EntityHandle *new_hs, 02655 // int &new_numps, 02656 // unsigned char &new_pstat 02657 ) 02658 { 02659 // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0 02660 // in this function, so no need to initialize; sharing data does not include 02661 // this proc if shared with only one other 02662 02663 // Following variables declared here to avoid compiler errors 02664 int new_numps; 02665 unsigned char new_pstat; 02666 std::vector< int > new_ps( MAX_SHARING_PROCS, -1 ); 02667 std::vector< EntityHandle > new_hs( MAX_SHARING_PROCS, 0 ); 02668 02669 new_numps = 0; 02670 ErrorCode result = get_sharing_data( new_h, &new_ps[0], &new_hs[0], new_pstat, new_numps );MB_CHK_SET_ERR( result, "Failed to get sharing data in update_remote_data" ); 02671 int num_exist = new_numps; 02672 02673 // Add new pstat info to the flag 02674 new_pstat |= add_pstat; 02675 02676 /* 02677 #define plist(str, lst, siz) \ 02678 std::cout << str << "("; \ 02679 for (int i = 0; i < (int)siz; i++) std::cout << lst[i] << " "; \ 02680 std::cout << ") "; \ 02681 02682 std::cout << "update_remote_data: rank = " << rank() << ", new_h = " << new_h << std::endl; 02683 std::string ostr; 02684 plist("ps", ps, num_ps); 02685 plist("hs", hs, num_ps); 02686 print_pstatus(add_pstat, ostr); 02687 std::cout << ", add_pstat = " << ostr.c_str() << std::endl; 02688 plist("tag_ps", new_ps, new_numps); 02689 plist("tag_hs", new_hs, new_numps); 02690 assert(new_numps <= size()); 02691 print_pstatus(new_pstat, ostr); 02692 std::cout << ", tag_pstat=" << ostr.c_str() << std::endl; 02693 */ 02694 02695 #ifndef NDEBUG 02696 { 02697 // Check for duplicates in proc list 02698 std::set< unsigned int > dumprocs; 02699 unsigned int dp = 0; 02700 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ ) 02701 dumprocs.insert( ps[dp] ); 02702 assert( dp == dumprocs.size() ); 02703 } 02704 #endif 02705 02706 // If only one sharer and I'm the owner, insert myself in the list; 02707 // otherwise, my data is checked at the end 02708 if( 1 == new_numps && !( new_pstat & PSTATUS_NOT_OWNED ) ) 02709 { 02710 new_hs[1] = new_hs[0]; 02711 new_ps[1] = new_ps[0]; 02712 new_hs[0] = new_h; 02713 new_ps[0] = rank(); 02714 new_numps = 2; 02715 } 02716 02717 // Now put passed-in data onto lists 02718 int idx; 02719 for( int i = 0; i < num_ps; i++ ) 02720 { 02721 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, ps[i] ) - &new_ps[0]; 02722 if( idx < new_numps ) 02723 { 02724 if( !new_hs[idx] && hs[i] ) 02725 // h on list is 0 and passed-in h is non-zero, replace it 02726 new_hs[idx] = hs[i]; 02727 else 02728 assert( !hs[i] || new_hs[idx] == hs[i] ); 02729 } 02730 else 02731 { 02732 if( new_numps + 1 == MAX_SHARING_PROCS ) 02733 { 02734 MB_SET_ERR( MB_FAILURE, "Exceeded MAX_SHARING_PROCS for " 02735 << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) << ' ' 02736 << ID_FROM_HANDLE( new_h ) << " in process " << rank() ); 02737 } 02738 new_ps[new_numps] = ps[i]; 02739 new_hs[new_numps] = hs[i]; 02740 new_numps++; 02741 } 02742 } 02743 02744 // Add myself, if it isn't there already 02745 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, rank() ) - &new_ps[0]; 02746 if( idx == new_numps ) 02747 { 02748 new_ps[new_numps] = rank(); 02749 new_hs[new_numps] = new_h; 02750 new_numps++; 02751 } 02752 else if( !new_hs[idx] && new_numps > 2 ) 02753 new_hs[idx] = new_h; 02754 02755 // Proc list is complete; update for shared, multishared 02756 if( new_numps > 1 ) 02757 { 02758 if( new_numps > 2 ) new_pstat |= PSTATUS_MULTISHARED; 02759 new_pstat |= PSTATUS_SHARED; 02760 } 02761 02762 /* 02763 plist("new_ps", new_ps, new_numps); 02764 plist("new_hs", new_hs, new_numps); 02765 print_pstatus(new_pstat, ostr); 02766 std::cout << ", new_pstat=" << ostr.c_str() << std::endl; 02767 std::cout << std::endl; 02768 */ 02769 02770 result = set_sharing_data( new_h, new_pstat, num_exist, new_numps, &new_ps[0], &new_hs[0] );MB_CHK_SET_ERR( result, "Failed to set sharing data in update_remote_data" ); 02771 02772 if( new_pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h ); 02773 02774 return MB_SUCCESS; 02775 } 02776 02777 ErrorCode ParallelComm::update_remote_data_old( const EntityHandle new_h, 02778 const int* ps, 02779 const EntityHandle* hs, 02780 const int num_ps, 02781 const unsigned char add_pstat ) 02782 { 02783 EntityHandle tag_hs[MAX_SHARING_PROCS]; 02784 int tag_ps[MAX_SHARING_PROCS]; 02785 unsigned char pstat; 02786 // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0 02787 // in this function, so no need to initialize 02788 unsigned int num_exist; 02789 ErrorCode result = get_sharing_data( new_h, tag_ps, tag_hs, pstat, num_exist );MB_CHK_ERR( result ); 02790 02791 #ifndef NDEBUG 02792 { 02793 // Check for duplicates in proc list 02794 std::set< unsigned int > dumprocs; 02795 unsigned int dp = 0; 02796 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ ) 02797 dumprocs.insert( ps[dp] ); 02798 assert( dp == dumprocs.size() ); 02799 } 02800 #endif 02801 02802 // Add any new sharing data 02803 bool changed = false; 02804 int idx; 02805 if( !num_exist ) 02806 { 02807 // Just take what caller passed 02808 memcpy( tag_ps, ps, num_ps * sizeof( int ) ); 02809 memcpy( tag_hs, hs, num_ps * sizeof( EntityHandle ) ); 02810 num_exist = num_ps; 02811 // If it's only one, hopefully I'm not there yet... 02812 assert( "I shouldn't be the only proc there." && ( 1 != num_exist || ps[0] != (int)procConfig.proc_rank() ) ); 02813 changed = true; 02814 } 02815 else 02816 { 02817 for( int i = 0; i < num_ps; i++ ) 02818 { 02819 idx = std::find( tag_ps, tag_ps + num_exist, ps[i] ) - tag_ps; 02820 if( idx == (int)num_exist ) 02821 { 02822 if( num_exist == MAX_SHARING_PROCS ) 02823 { 02824 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) 02825 << ' ' << ID_FROM_HANDLE( new_h ) << " in process " << proc_config().proc_rank() 02826 << std::endl; 02827 std::cerr.flush(); 02828 MPI_Abort( proc_config().proc_comm(), 66 ); 02829 } 02830 02831 // If there's only 1 sharing proc, and it's not me, then 02832 // we'll end up with 3; add me to the front 02833 if( !i && num_ps == 1 && num_exist == 1 && ps[0] != (int)procConfig.proc_rank() ) 02834 { 02835 int j = 1; 02836 // If I own this entity, put me at front, otherwise after first 02837 if( !( pstat & PSTATUS_NOT_OWNED ) ) 02838 { 02839 tag_ps[1] = tag_ps[0]; 02840 tag_hs[1] = tag_hs[0]; 02841 j = 0; 02842 } 02843 tag_ps[j] = procConfig.proc_rank(); 02844 tag_hs[j] = new_h; 02845 num_exist++; 02846 } 02847 02848 tag_ps[num_exist] = ps[i]; 02849 tag_hs[num_exist] = hs[i]; 02850 num_exist++; 02851 changed = true; 02852 } 02853 else if( 0 == tag_hs[idx] ) 02854 { 02855 tag_hs[idx] = hs[i]; 02856 changed = true; 02857 } 02858 else if( 0 != hs[i] ) 02859 { 02860 assert( hs[i] == tag_hs[idx] ); 02861 } 02862 } 02863 } 02864 02865 // Adjust for interface layer if necessary 02866 if( add_pstat & PSTATUS_INTERFACE ) 02867 { 02868 idx = std::min_element( tag_ps, tag_ps + num_exist ) - tag_ps; 02869 if( idx ) 02870 { 02871 int tag_proc = tag_ps[idx]; 02872 tag_ps[idx] = tag_ps[0]; 02873 tag_ps[0] = tag_proc; 02874 EntityHandle tag_h = tag_hs[idx]; 02875 tag_hs[idx] = tag_hs[0]; 02876 tag_hs[0] = tag_h; 02877 changed = true; 02878 if( tag_ps[0] != (int)procConfig.proc_rank() ) pstat |= PSTATUS_NOT_OWNED; 02879 } 02880 } 02881 02882 if( !changed ) return MB_SUCCESS; 02883 02884 assert( "interface entities should have > 1 proc" && ( !( add_pstat & PSTATUS_INTERFACE ) || num_exist > 1 ) ); 02885 assert( "ghost entities should have > 1 proc" && ( !( add_pstat & PSTATUS_GHOST ) || num_exist > 1 ) ); 02886 02887 // If it's multi-shared and we created the entity in this unpack, 02888 // local handle probably isn't in handle list yet 02889 if( num_exist > 2 ) 02890 { 02891 idx = std::find( tag_ps, tag_ps + num_exist, procConfig.proc_rank() ) - tag_ps; 02892 assert( idx < (int)num_exist ); 02893 if( !tag_hs[idx] ) tag_hs[idx] = new_h; 02894 } 02895 02896 int tag_p; 02897 EntityHandle tag_h; 02898 02899 // Update pstat 02900 pstat |= add_pstat; 02901 02902 if( num_exist > 2 ) 02903 pstat |= ( PSTATUS_MULTISHARED | PSTATUS_SHARED ); 02904 else if( num_exist > 0 ) 02905 pstat |= PSTATUS_SHARED; 02906 02907 // compare_remote_data(new_h, num_ps, hs, ps, add_pstat, 02908 // num_exist, tag_hs, tag_ps, pstat); 02909 02910 // Reset single shared proc/handle if was shared and moving to multi-shared 02911 if( num_exist > 2 && !( pstat & PSTATUS_MULTISHARED ) && ( pstat & PSTATUS_SHARED ) ) 02912 { 02913 // Must remove sharedp/h first, which really means set to default value 02914 tag_p = -1; 02915 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, &tag_p );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" ); 02916 tag_h = 0; 02917 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, &tag_h );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" ); 02918 } 02919 02920 // Set sharing tags 02921 if( num_exist > 2 ) 02922 { 02923 std::fill( tag_ps + num_exist, tag_ps + MAX_SHARING_PROCS, -1 ); 02924 std::fill( tag_hs + num_exist, tag_hs + MAX_SHARING_PROCS, 0 ); 02925 result = mbImpl->tag_set_data( sharedps_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" ); 02926 result = mbImpl->tag_set_data( sharedhs_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" ); 02927 02928 #ifndef NDEBUG 02929 { 02930 // Check for duplicates in proc list 02931 std::set< unsigned int > dumprocs; 02932 unsigned int dp = 0; 02933 for( ; dp < num_exist && -1 != tag_ps[dp]; dp++ ) 02934 dumprocs.insert( tag_ps[dp] ); 02935 assert( dp == dumprocs.size() ); 02936 } 02937 #endif 02938 } 02939 else if( num_exist == 2 || num_exist == 1 ) 02940 { 02941 if( tag_ps[0] == (int)procConfig.proc_rank() ) 02942 { 02943 assert( 2 == num_exist && tag_ps[1] != (int)procConfig.proc_rank() ); 02944 tag_ps[0] = tag_ps[1]; 02945 tag_hs[0] = tag_hs[1]; 02946 } 02947 assert( tag_ps[0] != -1 && tag_hs[0] != 0 ); 02948 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" ); 02949 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" ); 02950 } 02951 02952 // Now set new pstatus 02953 result = mbImpl->tag_set_data( pstatus_tag(), &new_h, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 02954 02955 if( pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h ); 02956 02957 return MB_SUCCESS; 02958 } 02959 02960 ErrorCode ParallelComm::get_sharing_data( const Range& entities, std::set< int >& procs, int operation ) 02961 { 02962 // Get the union or intersection of sharing data for multiple entities 02963 ErrorCode result; 02964 int sp2[MAX_SHARING_PROCS]; 02965 int num_ps; 02966 unsigned char pstat; 02967 std::set< int > tmp_procs; 02968 procs.clear(); 02969 02970 for( Range::const_iterator rit = entities.begin(); rit != entities.end(); ++rit ) 02971 { 02972 // Get sharing procs 02973 result = get_sharing_data( *rit, sp2, NULL, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_sharing_data" ); 02974 if( !( pstat & PSTATUS_SHARED ) && Interface::INTERSECT == operation ) 02975 { 02976 procs.clear(); 02977 return MB_SUCCESS; 02978 } 02979 02980 if( rit == entities.begin() ) 02981 { 02982 std::copy( sp2, sp2 + num_ps, std::inserter( procs, procs.begin() ) ); 02983 } 02984 else 02985 { 02986 std::sort( sp2, sp2 + num_ps ); 02987 tmp_procs.clear(); 02988 if( Interface::UNION == operation ) 02989 std::set_union( procs.begin(), procs.end(), sp2, sp2 + num_ps, 02990 std::inserter( tmp_procs, tmp_procs.end() ) ); 02991 else if( Interface::INTERSECT == operation ) 02992 std::set_intersection( procs.begin(), procs.end(), sp2, sp2 + num_ps, 02993 std::inserter( tmp_procs, tmp_procs.end() ) ); 02994 else 02995 { 02996 assert( "Unknown operation." && false ); 02997 return MB_FAILURE; 02998 } 02999 procs.swap( tmp_procs ); 03000 } 03001 if( Interface::INTERSECT == operation && procs.empty() ) return MB_SUCCESS; 03002 } 03003 03004 return MB_SUCCESS; 03005 } 03006 03007 ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity, 03008 int* ps, 03009 EntityHandle* hs, 03010 unsigned char& pstat, 03011 unsigned int& num_ps ) 03012 { 03013 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 03014 if( pstat & PSTATUS_MULTISHARED ) 03015 { 03016 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" ); 03017 if( hs ) 03018 { 03019 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 03020 } 03021 num_ps = std::find( ps, ps + MAX_SHARING_PROCS, -1 ) - ps; 03022 } 03023 else if( pstat & PSTATUS_SHARED ) 03024 { 03025 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 03026 if( hs ) 03027 { 03028 result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" ); 03029 hs[1] = 0; 03030 } 03031 // Initialize past end of data 03032 ps[1] = -1; 03033 num_ps = 1; 03034 } 03035 else 03036 { 03037 ps[0] = -1; 03038 if( hs ) hs[0] = 0; 03039 num_ps = 0; 03040 } 03041 03042 assert( MAX_SHARING_PROCS >= num_ps ); 03043 03044 return MB_SUCCESS; 03045 } 03046 03047 ErrorCode ParallelComm::find_existing_entity( const bool is_iface, 03048 const int owner_p, 03049 const EntityHandle owner_h, 03050 const int num_ps, 03051 const EntityHandle* connect, 03052 const int num_connect, 03053 const EntityType this_type, 03054 std::vector< EntityHandle >& L2hloc, 03055 std::vector< EntityHandle >& L2hrem, 03056 std::vector< unsigned int >& L2p, 03057 EntityHandle& new_h ) 03058 { 03059 new_h = 0; 03060 if( !is_iface && num_ps > 2 ) 03061 { 03062 for( unsigned int i = 0; i < L2hrem.size(); i++ ) 03063 { 03064 if( L2hrem[i] == owner_h && owner_p == (int)L2p[i] ) 03065 { 03066 new_h = L2hloc[i]; 03067 return MB_SUCCESS; 03068 } 03069 } 03070 } 03071 03072 // If we got here and it's a vertex, we don't need to look further 03073 if( MBVERTEX == this_type || !connect || !num_connect ) return MB_SUCCESS; 03074 03075 Range tmp_range; 03076 ErrorCode result = mbImpl->get_adjacencies( connect, num_connect, CN::Dimension( this_type ), false, tmp_range );MB_CHK_SET_ERR( result, "Failed to get existing entity" ); 03077 if( !tmp_range.empty() ) 03078 { 03079 // Found a corresponding entity - return target 03080 new_h = *tmp_range.begin(); 03081 } 03082 else 03083 { 03084 new_h = 0; 03085 } 03086 03087 return MB_SUCCESS; 03088 } 03089 03090 ErrorCode ParallelComm::get_local_handles( const Range& remote_handles, 03091 Range& local_handles, 03092 const std::vector< EntityHandle >& new_ents ) 03093 { 03094 std::vector< EntityHandle > rh_vec; 03095 rh_vec.reserve( remote_handles.size() ); 03096 std::copy( remote_handles.begin(), remote_handles.end(), std::back_inserter( rh_vec ) ); 03097 ErrorCode result = get_local_handles( &rh_vec[0], remote_handles.size(), new_ents ); 03098 std::copy( rh_vec.begin(), rh_vec.end(), range_inserter( local_handles ) ); 03099 return result; 03100 } 03101 03102 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents ) 03103 { 03104 std::vector< EntityHandle > tmp_ents; 03105 std::copy( new_ents.begin(), new_ents.end(), std::back_inserter( tmp_ents ) ); 03106 return get_local_handles( from_vec, num_ents, tmp_ents ); 03107 } 03108 03109 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, 03110 int num_ents, 03111 const std::vector< EntityHandle >& new_ents ) 03112 { 03113 for( int i = 0; i < num_ents; i++ ) 03114 { 03115 if( TYPE_FROM_HANDLE( from_vec[i] ) == MBMAXTYPE ) 03116 { 03117 assert( ID_FROM_HANDLE( from_vec[i] ) < (int)new_ents.size() ); 03118 from_vec[i] = new_ents[ID_FROM_HANDLE( from_vec[i] )]; 03119 } 03120 } 03121 03122 return MB_SUCCESS; 03123 } 03124 03125 /* 03126 template <typename T> void 03127 insert_in_array(T* array, size_t array_size, size_t location, T value) 03128 { 03129 assert(location + 1 < array_size); 03130 for (size_t i = array_size - 1; i > location; i--) 03131 array[i] = array[i - 1]; 03132 array[location] = value; 03133 } 03134 */ 03135 03136 ErrorCode ParallelComm::pack_range_map( Range& key_range, EntityHandle val_start, HandleMap& handle_map ) 03137 { 03138 for( Range::const_pair_iterator key_it = key_range.const_pair_begin(); key_it != key_range.const_pair_end(); 03139 ++key_it ) 03140 { 03141 int tmp_num = ( *key_it ).second - ( *key_it ).first + 1; 03142 handle_map.insert( ( *key_it ).first, val_start, tmp_num ); 03143 val_start += tmp_num; 03144 } 03145 03146 return MB_SUCCESS; 03147 } 03148 03149 ErrorCode ParallelComm::pack_sets( Range& entities, Buffer* buff, const bool store_remote_handles, const int to_proc ) 03150 { 03151 // SETS: 03152 // . #sets 03153 // . for each set: 03154 // - options[#sets] (unsigned int) 03155 // - if (unordered) set range 03156 // - else if ordered 03157 // . #ents in set 03158 // . handles[#ents] 03159 // - #parents 03160 // - if (#parents) handles[#parents] 03161 // - #children 03162 // - if (#children) handles[#children] 03163 03164 // Now the sets; assume any sets the application wants to pass are in the entities list 03165 ErrorCode result; 03166 Range all_sets = entities.subset_by_type( MBENTITYSET ); 03167 03168 int buff_size = estimate_sets_buffer_size( all_sets, store_remote_handles ); 03169 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate sets buffer size" ); 03170 buff->check_space( buff_size ); 03171 03172 // Number of sets 03173 PACK_INT( buff->buff_ptr, all_sets.size() ); 03174 03175 // Options for all sets 03176 std::vector< unsigned int > options( all_sets.size() ); 03177 Range::iterator rit; 03178 std::vector< EntityHandle > members; 03179 int i; 03180 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03181 { 03182 result = mbImpl->get_meshset_options( *rit, options[i] );MB_CHK_SET_ERR( result, "Failed to get meshset options" ); 03183 } 03184 buff->check_space( all_sets.size() * sizeof( unsigned int ) ); 03185 PACK_VOID( buff->buff_ptr, &options[0], all_sets.size() * sizeof( unsigned int ) ); 03186 03187 // Pack parallel geometry unique id 03188 if( !all_sets.empty() ) 03189 { 03190 Tag uid_tag; 03191 int n_sets = all_sets.size(); 03192 bool b_pack = false; 03193 std::vector< int > id_data( n_sets ); 03194 result = 03195 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" ); 03196 03197 result = mbImpl->tag_get_data( uid_tag, all_sets, &id_data[0] ); 03198 if( MB_TAG_NOT_FOUND != result ) 03199 { 03200 if( MB_SUCCESS != result ) MB_SET_ERR( result, "Failed to get parallel geometry unique ids" ); 03201 for( i = 0; i < n_sets; i++ ) 03202 { 03203 if( id_data[i] != 0 ) 03204 { 03205 b_pack = true; 03206 break; 03207 } 03208 } 03209 } 03210 03211 if( b_pack ) 03212 { // If you find 03213 buff->check_space( ( n_sets + 1 ) * sizeof( int ) ); 03214 PACK_INT( buff->buff_ptr, n_sets ); 03215 PACK_INTS( buff->buff_ptr, &id_data[0], n_sets ); 03216 } 03217 else 03218 { 03219 buff->check_space( sizeof( int ) ); 03220 PACK_INT( buff->buff_ptr, 0 ); 03221 } 03222 } 03223 03224 // Vectors/ranges 03225 std::vector< EntityHandle > entities_vec( entities.size() ); 03226 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 03227 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03228 { 03229 members.clear(); 03230 result = mbImpl->get_entities_by_handle( *rit, members );MB_CHK_SET_ERR( result, "Failed to get entities in ordered set" ); 03231 result = 03232 get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" ); 03233 buff->check_space( members.size() * sizeof( EntityHandle ) + sizeof( int ) ); 03234 PACK_INT( buff->buff_ptr, members.size() ); 03235 PACK_EH( buff->buff_ptr, &members[0], members.size() ); 03236 } 03237 03238 // Pack parent/child sets 03239 if( !store_remote_handles ) 03240 { // Only works not store remote handles 03241 // Pack numbers of parents/children 03242 unsigned int tot_pch = 0; 03243 int num_pch; 03244 buff->check_space( 2 * all_sets.size() * sizeof( int ) ); 03245 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03246 { 03247 // Pack parents 03248 result = mbImpl->num_parent_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num parents" ); 03249 PACK_INT( buff->buff_ptr, num_pch ); 03250 tot_pch += num_pch; 03251 result = mbImpl->num_child_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num children" ); 03252 PACK_INT( buff->buff_ptr, num_pch ); 03253 tot_pch += num_pch; 03254 } 03255 03256 // Now pack actual parents/children 03257 members.clear(); 03258 members.reserve( tot_pch ); 03259 std::vector< EntityHandle > tmp_pch; 03260 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ ) 03261 { 03262 result = mbImpl->get_parent_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get parents" ); 03263 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) ); 03264 tmp_pch.clear(); 03265 result = mbImpl->get_child_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get children" ); 03266 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) ); 03267 tmp_pch.clear(); 03268 } 03269 assert( members.size() == tot_pch ); 03270 if( !members.empty() ) 03271 { 03272 result = get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, 03273 entities_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles for set parent/child sets" ); 03274 #ifndef NDEBUG 03275 // Check that all handles are either sets or maxtype 03276 for( unsigned int __j = 0; __j < members.size(); __j++ ) 03277 assert( ( TYPE_FROM_HANDLE( members[__j] ) == MBMAXTYPE && 03278 ID_FROM_HANDLE( members[__j] ) < (int)entities.size() ) || 03279 TYPE_FROM_HANDLE( members[__j] ) == MBENTITYSET ); 03280 #endif 03281 buff->check_space( members.size() * sizeof( EntityHandle ) ); 03282 PACK_EH( buff->buff_ptr, &members[0], members.size() ); 03283 } 03284 } 03285 else 03286 { 03287 buff->check_space( 2 * all_sets.size() * sizeof( int ) ); 03288 for( rit = all_sets.begin(); rit != all_sets.end(); ++rit ) 03289 { 03290 PACK_INT( buff->buff_ptr, 0 ); 03291 PACK_INT( buff->buff_ptr, 0 ); 03292 } 03293 } 03294 03295 // Pack the handles 03296 if( store_remote_handles && !all_sets.empty() ) 03297 { 03298 buff_size = RANGE_SIZE( all_sets ); 03299 buff->check_space( buff_size ); 03300 PACK_RANGE( buff->buff_ptr, all_sets ); 03301 } 03302 03303 myDebug->tprintf( 4, "Done packing sets.\n" ); 03304 03305 buff->set_stored_size(); 03306 03307 return MB_SUCCESS; 03308 } 03309 03310 ErrorCode ParallelComm::unpack_sets( unsigned char*& buff_ptr, 03311 std::vector< EntityHandle >& entities, 03312 const bool store_remote_handles, 03313 const int from_proc ) 03314 { 03315 // Now the sets; assume any sets the application wants to pass are in the entities list 03316 ErrorCode result; 03317 03318 bool no_sets = ( entities.empty() || ( mbImpl->type_from_handle( *entities.rbegin() ) == MBENTITYSET ) ); 03319 03320 Range new_sets; 03321 int num_sets; 03322 UNPACK_INT( buff_ptr, num_sets ); 03323 03324 if( !num_sets ) return MB_SUCCESS; 03325 03326 int i; 03327 Range::const_iterator rit; 03328 std::vector< EntityHandle > members; 03329 int num_ents; 03330 std::vector< unsigned int > options_vec( num_sets ); 03331 // Option value 03332 if( num_sets ) UNPACK_VOID( buff_ptr, &options_vec[0], num_sets * sizeof( unsigned int ) ); 03333 03334 // Unpack parallel geometry unique id 03335 int n_uid; 03336 UNPACK_INT( buff_ptr, n_uid ); 03337 if( n_uid > 0 && n_uid != num_sets ) 03338 { 03339 std::cerr << "The number of Parallel geometry unique ids should be same." << std::endl; 03340 } 03341 03342 if( n_uid > 0 ) 03343 { // If parallel geometry unique id is packed 03344 std::vector< int > uids( n_uid ); 03345 UNPACK_INTS( buff_ptr, &uids[0], n_uid ); 03346 03347 Tag uid_tag; 03348 result = 03349 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" ); 03350 03351 // Find existing sets 03352 for( i = 0; i < n_uid; i++ ) 03353 { 03354 EntityHandle set_handle; 03355 Range temp_sets; 03356 void* tag_vals[] = { &uids[i] }; 03357 if( uids[i] > 0 ) 03358 { 03359 result = mbImpl->get_entities_by_type_and_tag( 0, MBENTITYSET, &uid_tag, tag_vals, 1, temp_sets ); 03360 } 03361 if( !temp_sets.empty() ) 03362 { // Existing set 03363 set_handle = *temp_sets.begin(); 03364 } 03365 else 03366 { // Create a new set 03367 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" ); 03368 result = mbImpl->tag_set_data( uid_tag, &set_handle, 1, &uids[i] );MB_CHK_SET_ERR( result, "Failed to set parallel geometry unique ids" ); 03369 } 03370 new_sets.insert( set_handle ); 03371 } 03372 } 03373 else 03374 { 03375 // Create sets 03376 for( i = 0; i < num_sets; i++ ) 03377 { 03378 EntityHandle set_handle; 03379 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" ); 03380 03381 // Make sure new sets handles are monotonically increasing 03382 assert( set_handle > *new_sets.rbegin() ); 03383 new_sets.insert( set_handle ); 03384 } 03385 } 03386 03387 std::copy( new_sets.begin(), new_sets.end(), std::back_inserter( entities ) ); 03388 // Only need to sort if we came in with no sets on the end 03389 if( !no_sets ) std::sort( entities.begin(), entities.end() ); 03390 03391 for( rit = new_sets.begin(), i = 0; rit != new_sets.end(); ++rit, i++ ) 03392 { 03393 // Unpack entities as vector, with length 03394 UNPACK_INT( buff_ptr, num_ents ); 03395 members.resize( num_ents ); 03396 if( num_ents ) UNPACK_EH( buff_ptr, &members[0], num_ents ); 03397 result = get_local_handles( &members[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for ordered set contents" ); 03398 result = mbImpl->add_entities( *rit, &members[0], num_ents );MB_CHK_SET_ERR( result, "Failed to add ents to ordered set in unpack" ); 03399 } 03400 03401 std::vector< int > num_pch( 2 * new_sets.size() ); 03402 std::vector< int >::iterator vit; 03403 int tot_pch = 0; 03404 for( vit = num_pch.begin(); vit != num_pch.end(); ++vit ) 03405 { 03406 UNPACK_INT( buff_ptr, *vit ); 03407 tot_pch += *vit; 03408 } 03409 03410 members.resize( tot_pch ); 03411 UNPACK_EH( buff_ptr, &members[0], tot_pch ); 03412 result = get_local_handles( &members[0], tot_pch, entities );MB_CHK_SET_ERR( result, "Failed to get local handle for parent/child sets" ); 03413 03414 int num = 0; 03415 EntityHandle* mem_ptr = &members[0]; 03416 for( rit = new_sets.begin(); rit != new_sets.end(); ++rit ) 03417 { 03418 // Unpack parents/children 03419 int num_par = num_pch[num++], num_child = num_pch[num++]; 03420 if( num_par + num_child ) 03421 { 03422 for( i = 0; i < num_par; i++ ) 03423 { 03424 assert( 0 != mem_ptr[i] ); 03425 result = mbImpl->add_parent_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add parent to set in unpack" ); 03426 } 03427 mem_ptr += num_par; 03428 for( i = 0; i < num_child; i++ ) 03429 { 03430 assert( 0 != mem_ptr[i] ); 03431 result = mbImpl->add_child_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add child to set in unpack" ); 03432 } 03433 mem_ptr += num_child; 03434 } 03435 } 03436 03437 // Unpack source handles 03438 Range dum_range; 03439 if( store_remote_handles && !new_sets.empty() ) 03440 { 03441 UNPACK_RANGE( buff_ptr, dum_range ); 03442 result = update_remote_data( new_sets, dum_range, from_proc, 0 );MB_CHK_SET_ERR( result, "Failed to set sharing data for sets" ); 03443 } 03444 03445 myDebug->tprintf( 4, "Done unpacking sets." ); 03446 03447 return MB_SUCCESS; 03448 } 03449 03450 ErrorCode ParallelComm::pack_adjacencies( Range& /*entities*/, 03451 Range::const_iterator& /*start_rit*/, 03452 Range& /*whole_range*/, 03453 unsigned char*& /*buff_ptr*/, 03454 int& /*count*/, 03455 const bool /*just_count*/, 03456 const bool /*store_handles*/, 03457 const int /*to_proc*/ ) 03458 { 03459 return MB_FAILURE; 03460 } 03461 03462 ErrorCode ParallelComm::unpack_adjacencies( unsigned char*& /*buff_ptr*/, 03463 Range& /*entities*/, 03464 const bool /*store_handles*/, 03465 const int /*from_proc*/ ) 03466 { 03467 return MB_FAILURE; 03468 } 03469 03470 ErrorCode ParallelComm::pack_tags( Range& entities, 03471 const std::vector< Tag >& src_tags, 03472 const std::vector< Tag >& dst_tags, 03473 const std::vector< Range >& tag_ranges, 03474 Buffer* buff, 03475 const bool store_remote_handles, 03476 const int to_proc ) 03477 { 03478 ErrorCode result; 03479 std::vector< Tag >::const_iterator tag_it, dst_it; 03480 std::vector< Range >::const_iterator rit; 03481 int count = 0; 03482 03483 for( tag_it = src_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); ++tag_it, ++rit ) 03484 { 03485 result = packed_tag_size( *tag_it, *rit, count ); 03486 if( MB_SUCCESS != result ) return result; 03487 } 03488 03489 // Number of tags 03490 count += sizeof( int ); 03491 03492 buff->check_space( count ); 03493 03494 PACK_INT( buff->buff_ptr, src_tags.size() ); 03495 03496 std::vector< EntityHandle > entities_vec( entities.size() ); 03497 std::copy( entities.begin(), entities.end(), entities_vec.begin() ); 03498 03499 for( tag_it = src_tags.begin(), dst_it = dst_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); 03500 ++tag_it, ++dst_it, ++rit ) 03501 { 03502 result = pack_tag( *tag_it, *dst_it, *rit, entities_vec, buff, store_remote_handles, to_proc ); 03503 if( MB_SUCCESS != result ) return result; 03504 } 03505 03506 myDebug->tprintf( 4, "Done packing tags." ); 03507 03508 buff->set_stored_size(); 03509 03510 return MB_SUCCESS; 03511 } 03512 03513 ErrorCode ParallelComm::packed_tag_size( Tag tag, const Range& tagged_entities, int& count ) 03514 { 03515 // For dense tags, compute size assuming all entities have that tag 03516 // For sparse tags, get number of entities w/ that tag to compute size 03517 03518 std::vector< int > var_len_sizes; 03519 std::vector< const void* > var_len_values; 03520 03521 // Default value 03522 count += sizeof( int ); 03523 if( NULL != tag->get_default_value() ) count += tag->get_default_value_size(); 03524 03525 // Size, type, data type 03526 count += 3 * sizeof( int ); 03527 03528 // Name 03529 count += sizeof( int ); 03530 count += tag->get_name().size(); 03531 03532 // Range of tag 03533 count += sizeof( int ) + tagged_entities.size() * sizeof( EntityHandle ); 03534 03535 if( tag->get_size() == MB_VARIABLE_LENGTH ) 03536 { 03537 const int num_ent = tagged_entities.size(); 03538 // Send a tag size for each entity 03539 count += num_ent * sizeof( int ); 03540 // Send tag data for each entity 03541 var_len_sizes.resize( num_ent ); 03542 var_len_values.resize( num_ent ); 03543 ErrorCode result = 03544 tag->get_data( sequenceManager, errorHandler, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get lenghts of variable-length tag values" ); 03545 count += std::accumulate( var_len_sizes.begin(), var_len_sizes.end(), 0 ); 03546 } 03547 else 03548 { 03549 // Tag data values for range or vector 03550 count += tagged_entities.size() * tag->get_size(); 03551 } 03552 03553 return MB_SUCCESS; 03554 } 03555 03556 ErrorCode ParallelComm::pack_tag( Tag src_tag, 03557 Tag dst_tag, 03558 const Range& tagged_entities, 03559 const std::vector< EntityHandle >& whole_vec, 03560 Buffer* buff, 03561 const bool store_remote_handles, 03562 const int to_proc ) 03563 { 03564 ErrorCode result; 03565 std::vector< int > var_len_sizes; 03566 std::vector< const void* > var_len_values; 03567 03568 if( src_tag != dst_tag ) 03569 { 03570 if( dst_tag->get_size() != src_tag->get_size() ) return MB_TYPE_OUT_OF_RANGE; 03571 if( dst_tag->get_data_type() != src_tag->get_data_type() && dst_tag->get_data_type() != MB_TYPE_OPAQUE && 03572 src_tag->get_data_type() != MB_TYPE_OPAQUE ) 03573 return MB_TYPE_OUT_OF_RANGE; 03574 } 03575 03576 // Size, type, data type 03577 buff->check_space( 3 * sizeof( int ) ); 03578 PACK_INT( buff->buff_ptr, src_tag->get_size() ); 03579 TagType this_type; 03580 result = mbImpl->tag_get_type( dst_tag, this_type ); 03581 PACK_INT( buff->buff_ptr, (int)this_type ); 03582 DataType data_type = src_tag->get_data_type(); 03583 PACK_INT( buff->buff_ptr, (int)data_type ); 03584 int type_size = TagInfo::size_from_data_type( data_type ); 03585 03586 // Default value 03587 if( NULL == src_tag->get_default_value() ) 03588 { 03589 buff->check_space( sizeof( int ) ); 03590 PACK_INT( buff->buff_ptr, 0 ); 03591 } 03592 else 03593 { 03594 buff->check_space( src_tag->get_default_value_size() ); 03595 PACK_BYTES( buff->buff_ptr, src_tag->get_default_value(), src_tag->get_default_value_size() ); 03596 } 03597 03598 // Name 03599 buff->check_space( src_tag->get_name().size() ); 03600 PACK_BYTES( buff->buff_ptr, dst_tag->get_name().c_str(), dst_tag->get_name().size() ); 03601 03602 myDebug->tprintf( 4, "Packing tag \"%s\"", src_tag->get_name().c_str() ); 03603 if( src_tag != dst_tag ) myDebug->tprintf( 4, " (as tag \"%s\")", dst_tag->get_name().c_str() ); 03604 myDebug->tprintf( 4, "\n" ); 03605 03606 // Pack entities 03607 buff->check_space( tagged_entities.size() * sizeof( EntityHandle ) + sizeof( int ) ); 03608 PACK_INT( buff->buff_ptr, tagged_entities.size() ); 03609 std::vector< EntityHandle > dum_tagged_entities( tagged_entities.size() ); 03610 result = get_remote_handles( store_remote_handles, tagged_entities, &dum_tagged_entities[0], to_proc, whole_vec ); 03611 if( MB_SUCCESS != result ) 03612 { 03613 if( myDebug->get_verbosity() == 3 ) 03614 { 03615 std::cerr << "Failed to get remote handles for tagged entities:" << std::endl; 03616 tagged_entities.print( " " ); 03617 } 03618 MB_SET_ERR( result, "Failed to get remote handles for tagged entities" ); 03619 } 03620 03621 PACK_EH( buff->buff_ptr, &dum_tagged_entities[0], dum_tagged_entities.size() ); 03622 03623 const size_t num_ent = tagged_entities.size(); 03624 if( src_tag->get_size() == MB_VARIABLE_LENGTH ) 03625 { 03626 var_len_sizes.resize( num_ent, 0 ); 03627 var_len_values.resize( num_ent, 0 ); 03628 result = mbImpl->tag_get_by_ptr( src_tag, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get variable-length tag data in pack_tags" ); 03629 buff->check_space( num_ent * sizeof( int ) ); 03630 PACK_INTS( buff->buff_ptr, &var_len_sizes[0], num_ent ); 03631 for( unsigned int i = 0; i < num_ent; i++ ) 03632 { 03633 buff->check_space( var_len_sizes[i] ); 03634 PACK_VOID( buff->buff_ptr, var_len_values[i], type_size * var_len_sizes[i] ); 03635 } 03636 } 03637 else 03638 { 03639 buff->check_space( num_ent * src_tag->get_size() ); 03640 // Should be OK to read directly into buffer, since tags are untyped and 03641 // handled by memcpy 03642 result = mbImpl->tag_get_data( src_tag, tagged_entities, buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get tag data in pack_tags" ); 03643 buff->buff_ptr += num_ent * src_tag->get_size(); 03644 PC( num_ent * src_tag->get_size(), " void" ); 03645 } 03646 03647 return MB_SUCCESS; 03648 } 03649 03650 ErrorCode ParallelComm::get_tag_send_list( const Range& whole_range, 03651 std::vector< Tag >& all_tags, 03652 std::vector< Range >& tag_ranges ) 03653 { 03654 std::vector< Tag > tmp_tags; 03655 ErrorCode result = mbImpl->tag_get_tags( tmp_tags );MB_CHK_SET_ERR( result, "Failed to get tags in pack_tags" ); 03656 03657 std::vector< Tag >::iterator tag_it; 03658 for( tag_it = tmp_tags.begin(); tag_it != tmp_tags.end(); ++tag_it ) 03659 { 03660 std::string tag_name; 03661 result = mbImpl->tag_get_name( *tag_it, tag_name ); 03662 if( tag_name.c_str()[0] == '_' && tag_name.c_str()[1] == '_' ) continue; 03663 03664 Range tmp_range; 03665 result = ( *tag_it )->get_tagged_entities( sequenceManager, tmp_range );MB_CHK_SET_ERR( result, "Failed to get entities for tag in pack_tags" ); 03666 tmp_range = intersect( tmp_range, whole_range ); 03667 03668 if( tmp_range.empty() ) continue; 03669 03670 // OK, we'll be sending this tag 03671 all_tags.push_back( *tag_it ); 03672 tag_ranges.push_back( Range() ); 03673 tag_ranges.back().swap( tmp_range ); 03674 } 03675 03676 return MB_SUCCESS; 03677 } 03678 03679 ErrorCode ParallelComm::unpack_tags( unsigned char*& buff_ptr, 03680 std::vector< EntityHandle >& entities, 03681 const bool /*store_remote_handles*/, 03682 const int /*from_proc*/, 03683 const MPI_Op* const mpi_op ) 03684 { 03685 // Tags 03686 // Get all the tags 03687 // For dense tags, compute size assuming all entities have that tag 03688 // For sparse tags, get number of entities w/ that tag to compute size 03689 03690 ErrorCode result; 03691 03692 int num_tags; 03693 UNPACK_INT( buff_ptr, num_tags ); 03694 std::vector< const void* > var_len_vals; 03695 std::vector< unsigned char > dum_vals; 03696 std::vector< EntityHandle > dum_ehvals; 03697 03698 for( int i = 0; i < num_tags; i++ ) 03699 { 03700 // Tag handle 03701 Tag tag_handle; 03702 03703 // Size, data type 03704 int tag_size, tag_data_type, tag_type; 03705 UNPACK_INT( buff_ptr, tag_size ); 03706 UNPACK_INT( buff_ptr, tag_type ); 03707 UNPACK_INT( buff_ptr, tag_data_type ); 03708 03709 // Default value 03710 int def_val_size; 03711 UNPACK_INT( buff_ptr, def_val_size ); 03712 void* def_val_ptr = NULL; 03713 if( def_val_size ) 03714 { 03715 def_val_ptr = buff_ptr; 03716 buff_ptr += def_val_size; 03717 UPC( tag_size, " void" ); 03718 } 03719 03720 // Name 03721 int name_len; 03722 UNPACK_INT( buff_ptr, name_len ); 03723 std::string tag_name( reinterpret_cast< char* >( buff_ptr ), name_len ); 03724 buff_ptr += name_len; 03725 UPC( 64, " chars" ); 03726 03727 myDebug->tprintf( 4, "Unpacking tag %s\n", tag_name.c_str() ); 03728 03729 // Create the tag 03730 if( tag_size == MB_VARIABLE_LENGTH ) 03731 result = mbImpl->tag_get_handle( tag_name.c_str(), def_val_size, (DataType)tag_data_type, tag_handle, 03732 MB_TAG_VARLEN | MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr ); 03733 else 03734 result = mbImpl->tag_get_handle( tag_name.c_str(), tag_size, (DataType)tag_data_type, tag_handle, 03735 MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr ); 03736 if( MB_SUCCESS != result ) return result; 03737 03738 // Get handles and convert to local handles 03739 int num_ents; 03740 UNPACK_INT( buff_ptr, num_ents ); 03741 std::vector< EntityHandle > dum_ents( num_ents ); 03742 UNPACK_EH( buff_ptr, &dum_ents[0], num_ents ); 03743 03744 // In this case handles are indices into new entity range; need to convert 03745 // to local handles 03746 result = get_local_handles( &dum_ents[0], num_ents, entities );MB_CHK_SET_ERR( result, "Unable to convert to local handles" ); 03747 03748 // If it's a handle type, also convert tag vals in-place in buffer 03749 if( MB_TYPE_HANDLE == tag_type ) 03750 { 03751 dum_ehvals.resize( num_ents ); 03752 UNPACK_EH( buff_ptr, &dum_ehvals[0], num_ents ); 03753 result = get_local_handles( &dum_ehvals[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for tag vals" ); 03754 } 03755 03756 DataType data_type; 03757 mbImpl->tag_get_data_type( tag_handle, data_type ); 03758 int type_size = TagInfo::size_from_data_type( data_type ); 03759 03760 if( !dum_ents.empty() ) 03761 { 03762 if( tag_size == MB_VARIABLE_LENGTH ) 03763 { 03764 // Be careful of alignment here. If the integers are aligned 03765 // in the buffer, we can use them directly. Otherwise we must 03766 // copy them. 03767 std::vector< int > var_lengths( num_ents ); 03768 UNPACK_INTS( buff_ptr, &var_lengths[0], num_ents ); 03769 UPC( sizeof( int ) * num_ents, " void" ); 03770 03771 // Get pointers into buffer for each tag value 03772 var_len_vals.resize( num_ents ); 03773 for( std::vector< EntityHandle >::size_type j = 0; j < (std::vector< EntityHandle >::size_type)num_ents; 03774 j++ ) 03775 { 03776 var_len_vals[j] = buff_ptr; 03777 buff_ptr += var_lengths[j] * type_size; 03778 UPC( var_lengths[j], " void" ); 03779 } 03780 result = 03781 mbImpl->tag_set_by_ptr( tag_handle, &dum_ents[0], num_ents, &var_len_vals[0], &var_lengths[0] );MB_CHK_SET_ERR( result, "Failed to set tag data when unpacking variable-length tag" ); 03782 } 03783 else 03784 { 03785 // Get existing values of dst tag 03786 dum_vals.resize( tag_size * num_ents ); 03787 if( mpi_op ) 03788 { 03789 int tag_length; 03790 result = mbImpl->tag_get_length( tag_handle, tag_length );MB_CHK_SET_ERR( result, "Failed to get tag length" ); 03791 result = mbImpl->tag_get_data( tag_handle, &dum_ents[0], num_ents, &dum_vals[0] );MB_CHK_SET_ERR( result, "Failed to get existing value of dst tag on entities" ); 03792 result = reduce_void( tag_data_type, *mpi_op, tag_length * num_ents, &dum_vals[0], buff_ptr );MB_CHK_SET_ERR( result, "Failed to perform mpi op on dst tags" ); 03793 } 03794 result = mbImpl->tag_set_data( tag_handle, &dum_ents[0], num_ents, buff_ptr );MB_CHK_SET_ERR( result, "Failed to set range-based tag data when unpacking tag" ); 03795 buff_ptr += num_ents * tag_size; 03796 UPC( num_ents * tag_size, " void" ); 03797 } 03798 } 03799 } 03800 03801 myDebug->tprintf( 4, "Done unpacking tags.\n" ); 03802 03803 return MB_SUCCESS; 03804 } 03805 03806 template < class T > 03807 T LAND( const T& arg1, const T& arg2 ) 03808 { 03809 return arg1 && arg2; 03810 } 03811 template < class T > 03812 T LOR( const T& arg1, const T& arg2 ) 03813 { 03814 return arg1 || arg2; 03815 } 03816 template < class T > 03817 T LXOR( const T& arg1, const T& arg2 ) 03818 { 03819 return ( ( arg1 && !arg2 ) || ( !arg1 && arg2 ) ); 03820 } 03821 template < class T > 03822 T MAX( const T& arg1, const T& arg2 ) 03823 { 03824 return ( arg1 > arg2 ? arg1 : arg2 ); 03825 } 03826 template < class T > 03827 T MIN( const T& arg1, const T& arg2 ) 03828 { 03829 return ( arg1 < arg2 ? arg1 : arg2 ); 03830 } 03831 template < class T > 03832 T ADD( const T& arg1, const T& arg2 ) 03833 { 03834 return arg1 + arg2; 03835 } 03836 template < class T > 03837 T MULT( const T& arg1, const T& arg2 ) 03838 { 03839 return arg1 * arg2; 03840 } 03841 03842 template < class T > 03843 ErrorCode ParallelComm::reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals ) 03844 { 03845 T* old_tmp = reinterpret_cast< T* >( old_vals ); 03846 // T *new_tmp = reinterpret_cast<T*>(new_vals); 03847 // new vals pointer needs to be aligned , some compilers will optimize and will shift 03848 03849 std::vector< T > new_values; 03850 new_values.resize( num_ents ); 03851 memcpy( &new_values[0], new_vals, num_ents * sizeof( T ) ); 03852 T* new_tmp = &new_values[0]; 03853 03854 if( mpi_op == MPI_SUM ) 03855 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, ADD< T > ); 03856 else if( mpi_op == MPI_PROD ) 03857 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MULT< T > ); 03858 else if( mpi_op == MPI_MAX ) 03859 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MAX< T > ); 03860 else if( mpi_op == MPI_MIN ) 03861 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MIN< T > ); 03862 else if( mpi_op == MPI_LAND ) 03863 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LAND< T > ); 03864 else if( mpi_op == MPI_LOR ) 03865 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LOR< T > ); 03866 else if( mpi_op == MPI_LXOR ) 03867 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LXOR< T > ); 03868 else if( mpi_op == MPI_BAND || mpi_op == MPI_BOR || mpi_op == MPI_BXOR ) 03869 { 03870 std::cerr << "Bitwise operations not allowed in tag reductions." << std::endl; 03871 return MB_FAILURE; 03872 } 03873 else if( mpi_op != MPI_OP_NULL ) 03874 { 03875 std::cerr << "Unknown MPI operation type." << std::endl; 03876 return MB_TYPE_OUT_OF_RANGE; 03877 } 03878 03879 // copy now the result back where it should be 03880 memcpy( new_vals, new_tmp, num_ents * sizeof( T ) ); 03881 std::vector< T >().swap( new_values ); // way to release allocated vector 03882 03883 return MB_SUCCESS; 03884 } 03885 03886 ErrorCode ParallelComm::reduce_void( int tag_data_type, 03887 const MPI_Op mpi_op, 03888 int num_ents, 03889 void* old_vals, 03890 void* new_vals ) 03891 { 03892 ErrorCode result; 03893 switch( tag_data_type ) 03894 { 03895 case MB_TYPE_INTEGER: 03896 result = reduce< int >( mpi_op, num_ents, old_vals, new_vals ); 03897 break; 03898 case MB_TYPE_DOUBLE: 03899 result = reduce< double >( mpi_op, num_ents, old_vals, new_vals ); 03900 break; 03901 case MB_TYPE_BIT: 03902 result = reduce< unsigned char >( mpi_op, num_ents, old_vals, new_vals ); 03903 break; 03904 default: 03905 result = MB_SUCCESS; 03906 break; 03907 } 03908 03909 return result; 03910 } 03911 03912 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, int resolve_dim, int shared_dim, const Tag* id_tag ) 03913 { 03914 ErrorCode result; 03915 Range proc_ents; 03916 03917 // Check for structured mesh, and do it differently if it is 03918 ScdInterface* scdi; 03919 result = mbImpl->query_interface( scdi ); 03920 if( scdi ) 03921 { 03922 result = scdi->tag_shared_vertices( this, this_set ); 03923 if( MB_SUCCESS == result ) 03924 { 03925 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 03926 return result; 03927 } 03928 } 03929 03930 if( 0 == this_set ) 03931 { 03932 // Get the entities in the partition sets 03933 for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit ) 03934 { 03935 Range tmp_ents; 03936 result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true ); 03937 if( MB_SUCCESS != result ) return result; 03938 proc_ents.merge( tmp_ents ); 03939 } 03940 } 03941 else 03942 { 03943 result = mbImpl->get_entities_by_handle( this_set, proc_ents, true ); 03944 if( MB_SUCCESS != result ) return result; 03945 } 03946 03947 // Resolve dim is maximal dim of entities in proc_ents 03948 if( -1 == resolve_dim ) 03949 { 03950 if( !proc_ents.empty() ) resolve_dim = mbImpl->dimension_from_handle( *proc_ents.rbegin() ); 03951 } 03952 03953 // proc_ents should all be of same dimension 03954 if( resolve_dim > shared_dim && 03955 mbImpl->dimension_from_handle( *proc_ents.rbegin() ) != mbImpl->dimension_from_handle( *proc_ents.begin() ) ) 03956 { 03957 Range::iterator lower = proc_ents.lower_bound( CN::TypeDimensionMap[0].first ), 03958 upper = proc_ents.upper_bound( CN::TypeDimensionMap[resolve_dim - 1].second ); 03959 proc_ents.erase( lower, upper ); 03960 } 03961 03962 // Must call even if we don't have any entities, to make sure 03963 // collective comm'n works 03964 return resolve_shared_ents( this_set, proc_ents, resolve_dim, shared_dim, NULL, id_tag ); 03965 } 03966 03967 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, 03968 Range& proc_ents, 03969 int resolve_dim, 03970 int shared_dim, 03971 Range* skin_ents, 03972 const Tag* id_tag ) 03973 { 03974 #ifdef MOAB_HAVE_MPE 03975 if( myDebug->get_verbosity() == 2 ) 03976 { 03977 define_mpe(); 03978 MPE_Log_event( RESOLVE_START, procConfig.proc_rank(), "Entering resolve_shared_ents." ); 03979 } 03980 #endif 03981 03982 ErrorCode result; 03983 myDebug->tprintf( 1, "Resolving shared entities.\n" ); 03984 03985 if( resolve_dim < shared_dim ) 03986 { 03987 MB_SET_ERR( MB_FAILURE, "MOAB does not support vertex-based partitions, only element-based ones" ); 03988 } 03989 03990 if( -1 == shared_dim ) 03991 { 03992 if( !proc_ents.empty() ) 03993 shared_dim = mbImpl->dimension_from_handle( *proc_ents.begin() ) - 1; 03994 else if( resolve_dim == 3 ) 03995 shared_dim = 2; 03996 } 03997 int max_global_resolve_dim = -1; 03998 int err = MPI_Allreduce( &resolve_dim, &max_global_resolve_dim, 1, MPI_INT, MPI_MAX, proc_config().proc_comm() ); 03999 if( MPI_SUCCESS != err ) 04000 { 04001 MB_SET_ERR( MB_FAILURE, "Unable to guess global resolve_dim" ); 04002 } 04003 if( shared_dim < 0 || resolve_dim < 0 ) 04004 { 04005 // MB_SET_ERR(MB_FAILURE, "Unable to guess shared_dim or resolve_dim"); 04006 resolve_dim = max_global_resolve_dim; 04007 shared_dim = resolve_dim - 1; 04008 } 04009 04010 if( resolve_dim < 0 || shared_dim < 0 ) return MB_SUCCESS; 04011 // no task has any mesh, get out 04012 04013 // Get the skin entities by dimension 04014 Range tmp_skin_ents[4]; 04015 04016 // Get the entities to be skinned 04017 // Find the skin 04018 int skin_dim = resolve_dim - 1; 04019 if( !skin_ents ) 04020 { 04021 skin_ents = tmp_skin_ents; 04022 skin_ents[resolve_dim] = proc_ents; 04023 Skinner skinner( mbImpl ); 04024 result = 04025 skinner.find_skin( this_set, skin_ents[skin_dim + 1], false, skin_ents[skin_dim], NULL, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" ); 04026 myDebug->tprintf( 1, "Found skin: skin_dim: %d resolve_dim: %d , now resolving.\n", skin_dim, resolve_dim ); 04027 myDebug->tprintf( 3, "skin_ents[0].size(): %d skin_ents[1].size(): %d \n", (int)skin_ents[0].size(), 04028 (int)skin_ents[1].size() ); 04029 // Get entities adjacent to skin ents from shared_dim down to zero 04030 for( int this_dim = skin_dim - 1; this_dim >= 0; this_dim-- ) 04031 { 04032 result = 04033 mbImpl->get_adjacencies( skin_ents[skin_dim], this_dim, true, skin_ents[this_dim], Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" ); 04034 04035 if( this_set && skin_dim == 2 && this_dim == 1 ) 04036 { 04037 result = mbImpl->add_entities( this_set, skin_ents[this_dim] );MB_CHK_ERR( result ); 04038 } 04039 } 04040 } 04041 else if( skin_ents[resolve_dim].empty() ) 04042 skin_ents[resolve_dim] = proc_ents; 04043 04044 // Global id tag 04045 Tag gid_tag; 04046 if( id_tag ) 04047 gid_tag = *id_tag; 04048 else 04049 { 04050 bool tag_created = false; 04051 int def_val = -1; 04052 result = mbImpl->tag_get_handle( GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE | MB_TAG_CREAT, 04053 &def_val, &tag_created ); 04054 if( MB_ALREADY_ALLOCATED != result && MB_SUCCESS != result ) 04055 { 04056 MB_SET_ERR( result, "Failed to create/get gid tag handle" ); 04057 } 04058 else if( tag_created ) 04059 { 04060 // Just created it, so we need global ids 04061 result = assign_global_ids( this_set, skin_dim + 1, true, true, true );MB_CHK_SET_ERR( result, "Failed to assign global ids" ); 04062 } 04063 } 04064 04065 DataType tag_type; 04066 result = mbImpl->tag_get_data_type( gid_tag, tag_type );MB_CHK_SET_ERR( result, "Failed to get tag data type" ); 04067 int bytes_per_tag; 04068 result = mbImpl->tag_get_bytes( gid_tag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed to get number of bytes per tag" ); 04069 // On 64 bits, long and int are different 04070 // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?) 04071 04072 // Get gids for skin ents in a vector, to pass to gs 04073 std::vector< long > lgid_data( skin_ents[0].size() ); 04074 // Size is either long or int 04075 // On 64 bit is 8 or 4 04076 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) ) 04077 { // It is a special id tag 04078 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &lgid_data[0] );MB_CHK_SET_ERR( result, "Couldn't get gid tag for skin vertices" ); 04079 } 04080 else if( 4 == bytes_per_tag ) 04081 { // Must be GLOBAL_ID tag or 32 bits ... 04082 std::vector< int > gid_data( lgid_data.size() ); 04083 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &gid_data[0] );MB_CHK_SET_ERR( result, "Failed to get gid tag for skin vertices" ); 04084 std::copy( gid_data.begin(), gid_data.end(), lgid_data.begin() ); 04085 } 04086 else 04087 { 04088 // Not supported flag 04089 MB_SET_ERR( MB_FAILURE, "Unsupported id tag" ); 04090 } 04091 04092 // Put handles in vector for passing to gs setup 04093 std::vector< Ulong > handle_vec; // Assumes that we can do conversion from Ulong to EntityHandle 04094 std::copy( skin_ents[0].begin(), skin_ents[0].end(), std::back_inserter( handle_vec ) ); 04095 04096 #ifdef MOAB_HAVE_MPE 04097 if( myDebug->get_verbosity() == 2 ) 04098 { 04099 MPE_Log_event( SHAREDV_START, procConfig.proc_rank(), "Creating crystal router." ); 04100 } 04101 #endif 04102 04103 // Get a crystal router 04104 gs_data::crystal_data* cd = procConfig.crystal_router(); 04105 04106 /* 04107 // Get total number of entities; will overshoot highest global id, but 04108 // that's OK 04109 int num_total[2] = {0, 0}, num_local[2] = {0, 0}; 04110 result = mbImpl->get_number_entities_by_dimension(this_set, 0, num_local); 04111 if (MB_SUCCESS != result)return result; 04112 int failure = MPI_Allreduce(num_local, num_total, 1, 04113 MPI_INT, MPI_SUM, procConfig.proc_comm()); 04114 if (failure) { 04115 MB_SET_ERR(MB_FAILURE, "Allreduce for total number of shared ents failed"); 04116 } 04117 */ 04118 // Call gather-scatter to get shared ids & procs 04119 gs_data* gsd = new gs_data(); 04120 // assert(sizeof(ulong_) == sizeof(EntityHandle)); 04121 result = gsd->initialize( skin_ents[0].size(), &lgid_data[0], &handle_vec[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" ); 04122 04123 // Get shared proc tags 04124 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 04125 result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" ); 04126 04127 // Load shared verts into a tuple, then sort by index 04128 TupleList shared_verts; 04129 shared_verts.initialize( 2, 0, 1, 0, skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ) ); 04130 shared_verts.enableWriteAccess(); 04131 04132 unsigned int i = 0, j = 0; 04133 for( unsigned int p = 0; p < gsd->nlinfo->_np; p++ ) 04134 for( unsigned int np = 0; np < gsd->nlinfo->_nshared[p]; np++ ) 04135 { 04136 shared_verts.vi_wr[i++] = gsd->nlinfo->_sh_ind[j]; 04137 shared_verts.vi_wr[i++] = gsd->nlinfo->_target[p]; 04138 shared_verts.vul_wr[j] = gsd->nlinfo->_ulabels[j]; 04139 j++; 04140 shared_verts.inc_n(); 04141 } 04142 04143 myDebug->tprintf( 3, " shared verts size %d \n", (int)shared_verts.get_n() ); 04144 04145 int max_size = skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ); 04146 moab::TupleList::buffer sort_buffer; 04147 sort_buffer.buffer_init( max_size ); 04148 shared_verts.sort( 0, &sort_buffer ); 04149 sort_buffer.reset(); 04150 04151 // Set sharing procs and handles tags on skin ents 04152 int maxp = -1; 04153 std::vector< int > sharing_procs( MAX_SHARING_PROCS ); 04154 std::fill( sharing_procs.begin(), sharing_procs.end(), maxp ); 04155 j = 0; 04156 i = 0; 04157 04158 // Get ents shared by 1 or n procs 04159 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs; 04160 Range proc_verts; 04161 result = mbImpl->get_adjacencies( proc_ents, 0, false, proc_verts, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get proc_verts" ); 04162 04163 myDebug->print( 3, " resolve shared ents: proc verts ", proc_verts ); 04164 result = tag_shared_verts( shared_verts, skin_ents, proc_nvecs, proc_verts );MB_CHK_SET_ERR( result, "Failed to tag shared verts" ); 04165 04166 #ifdef MOAB_HAVE_MPE 04167 if( myDebug->get_verbosity() == 2 ) 04168 { 04169 MPE_Log_event( SHAREDV_END, procConfig.proc_rank(), "Finished tag_shared_verts." ); 04170 } 04171 #endif 04172 04173 // Get entities shared by 1 or n procs 04174 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );MB_CHK_SET_ERR( result, "Failed to tag shared entities" ); 04175 04176 shared_verts.reset(); 04177 04178 if( myDebug->get_verbosity() > 0 ) 04179 { 04180 for( std::map< std::vector< int >, std::vector< EntityHandle > >::const_iterator mit = proc_nvecs.begin(); 04181 mit != proc_nvecs.end(); ++mit ) 04182 { 04183 myDebug->tprintf( 1, "Iface: " ); 04184 for( std::vector< int >::const_iterator vit = ( mit->first ).begin(); vit != ( mit->first ).end(); ++vit ) 04185 myDebug->printf( 1, " %d", *vit ); 04186 myDebug->print( 1, "\n" ); 04187 } 04188 } 04189 04190 // Create the sets for each interface; store them as tags on 04191 // the interface instance 04192 Range iface_sets; 04193 result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" ); 04194 04195 // Establish comm procs and buffers for them 04196 std::set< unsigned int > procs; 04197 result = get_interface_procs( procs, true );MB_CHK_SET_ERR( result, "Failed to get interface procs" ); 04198 04199 #ifndef NDEBUG 04200 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Shared handle check failed after interface vertex exchange" ); 04201 #endif 04202 04203 // Resolve shared entity remote handles; implemented in ghost cell exchange 04204 // code because it's so similar 04205 result = exchange_ghost_cells( -1, -1, 0, 0, true, true );MB_CHK_SET_ERR( result, "Failed to resolve shared entity remote handles" ); 04206 04207 // Now build parent/child links for interface sets 04208 result = create_iface_pc_links();MB_CHK_SET_ERR( result, "Failed to create interface parent/child links" ); 04209 04210 gsd->reset(); 04211 delete gsd; 04212 04213 #ifdef MOAB_HAVE_MPE 04214 if( myDebug->get_verbosity() == 2 ) 04215 { 04216 MPE_Log_event( RESOLVE_END, procConfig.proc_rank(), "Exiting resolve_shared_ents." ); 04217 } 04218 #endif 04219 04220 // std::ostringstream ent_str; 04221 // ent_str << "mesh." << procConfig.proc_rank() << ".h5m"; 04222 // mbImpl->write_mesh(ent_str.str().c_str()); 04223 04224 // Done 04225 return result; 04226 } 04227 04228 void ParallelComm::define_mpe() 04229 { 04230 #ifdef MOAB_HAVE_MPE 04231 if( myDebug->get_verbosity() == 2 ) 04232 { 04233 // Define mpe states used for logging 04234 int success; 04235 MPE_Log_get_state_eventIDs( &IFACE_START, &IFACE_END ); 04236 MPE_Log_get_state_eventIDs( &GHOST_START, &GHOST_END ); 04237 MPE_Log_get_state_eventIDs( &SHAREDV_START, &SHAREDV_END ); 04238 MPE_Log_get_state_eventIDs( &RESOLVE_START, &RESOLVE_END ); 04239 MPE_Log_get_state_eventIDs( &ENTITIES_START, &ENTITIES_END ); 04240 MPE_Log_get_state_eventIDs( &RHANDLES_START, &RHANDLES_END ); 04241 MPE_Log_get_state_eventIDs( &OWNED_START, &OWNED_END ); 04242 success = MPE_Describe_state( IFACE_START, IFACE_END, "Resolve interface ents", "green" ); 04243 assert( MPE_LOG_OK == success ); 04244 success = MPE_Describe_state( GHOST_START, GHOST_END, "Exchange ghost ents", "red" ); 04245 assert( MPE_LOG_OK == success ); 04246 success = MPE_Describe_state( SHAREDV_START, SHAREDV_END, "Resolve interface vertices", "blue" ); 04247 assert( MPE_LOG_OK == success ); 04248 success = MPE_Describe_state( RESOLVE_START, RESOLVE_END, "Resolve shared ents", "purple" ); 04249 assert( MPE_LOG_OK == success ); 04250 success = MPE_Describe_state( ENTITIES_START, ENTITIES_END, "Exchange shared ents", "yellow" ); 04251 assert( MPE_LOG_OK == success ); 04252 success = MPE_Describe_state( RHANDLES_START, RHANDLES_END, "Remote handles", "cyan" ); 04253 assert( MPE_LOG_OK == success ); 04254 success = MPE_Describe_state( OWNED_START, OWNED_END, "Exchange owned ents", "black" ); 04255 assert( MPE_LOG_OK == success ); 04256 } 04257 #endif 04258 } 04259 04260 ErrorCode ParallelComm::resolve_shared_ents( ParallelComm** pc, 04261 const unsigned int np, 04262 EntityHandle this_set, 04263 const int part_dim ) 04264 { 04265 std::vector< Range > verts( np ); 04266 int tot_verts = 0; 04267 unsigned int p, i, j, v; 04268 ErrorCode rval; 04269 for( p = 0; p < np; p++ ) 04270 { 04271 Skinner skinner( pc[p]->get_moab() ); 04272 Range part_ents, skin_ents; 04273 rval = pc[p]->get_moab()->get_entities_by_dimension( this_set, part_dim, part_ents ); 04274 if( MB_SUCCESS != rval ) return rval; 04275 rval = skinner.find_skin( this_set, part_ents, false, skin_ents, 0, true, true, true ); 04276 if( MB_SUCCESS != rval ) return rval; 04277 rval = pc[p]->get_moab()->get_adjacencies( skin_ents, 0, true, verts[p], Interface::UNION ); 04278 if( MB_SUCCESS != rval ) return rval; 04279 tot_verts += verts[p].size(); 04280 } 04281 04282 TupleList shared_ents; 04283 shared_ents.initialize( 2, 0, 1, 0, tot_verts ); 04284 shared_ents.enableWriteAccess(); 04285 04286 i = 0; 04287 j = 0; 04288 std::vector< int > gids; 04289 Range::iterator rit; 04290 Tag gid_tag; 04291 for( p = 0; p < np; p++ ) 04292 { 04293 gid_tag = pc[p]->get_moab()->globalId_tag(); 04294 04295 gids.resize( verts[p].size() ); 04296 rval = pc[p]->get_moab()->tag_get_data( gid_tag, verts[p], &gids[0] ); 04297 if( MB_SUCCESS != rval ) return rval; 04298 04299 for( v = 0, rit = verts[p].begin(); v < gids.size(); v++, ++rit ) 04300 { 04301 shared_ents.vi_wr[i++] = gids[v]; 04302 shared_ents.vi_wr[i++] = p; 04303 shared_ents.vul_wr[j] = *rit; 04304 j++; 04305 shared_ents.inc_n(); 04306 } 04307 } 04308 04309 moab::TupleList::buffer sort_buffer; 04310 sort_buffer.buffer_init( tot_verts ); 04311 shared_ents.sort( 0, &sort_buffer ); 04312 sort_buffer.reset(); 04313 04314 j = 0; 04315 i = 0; 04316 std::vector< EntityHandle > handles; 04317 std::vector< int > procs; 04318 04319 while( i < shared_ents.get_n() ) 04320 { 04321 handles.clear(); 04322 procs.clear(); 04323 04324 // Count & accumulate sharing procs 04325 int this_gid = shared_ents.vi_rd[j]; 04326 while( i < shared_ents.get_n() && shared_ents.vi_rd[j] == this_gid ) 04327 { 04328 j++; 04329 procs.push_back( shared_ents.vi_rd[j++] ); 04330 handles.push_back( shared_ents.vul_rd[i++] ); 04331 } 04332 if( 1 == procs.size() ) continue; 04333 04334 for( v = 0; v < procs.size(); v++ ) 04335 { 04336 rval = pc[procs[v]]->update_remote_data( handles[v], &procs[0], &handles[0], procs.size(), 04337 ( procs[0] == (int)pc[procs[v]]->rank() 04338 ? PSTATUS_INTERFACE 04339 : ( PSTATUS_NOT_OWNED | PSTATUS_INTERFACE ) ) ); 04340 if( MB_SUCCESS != rval ) return rval; 04341 } 04342 } 04343 04344 std::set< unsigned int > psets; 04345 for( p = 0; p < np; p++ ) 04346 { 04347 rval = pc[p]->create_interface_sets( this_set, part_dim, part_dim - 1 ); 04348 if( MB_SUCCESS != rval ) return rval; 04349 // Establish comm procs and buffers for them 04350 psets.clear(); 04351 rval = pc[p]->get_interface_procs( psets, true ); 04352 if( MB_SUCCESS != rval ) return rval; 04353 } 04354 04355 shared_ents.reset(); 04356 04357 return MB_SUCCESS; 04358 } 04359 04360 ErrorCode ParallelComm::tag_iface_entities() 04361 { 04362 ErrorCode result = MB_SUCCESS; 04363 Range iface_ents, tmp_ents, rmv_ents; 04364 std::vector< unsigned char > pstat; 04365 unsigned char set_pstat; 04366 Range::iterator rit2; 04367 unsigned int i; 04368 04369 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 04370 { 04371 iface_ents.clear(); 04372 04373 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get interface set contents" ); 04374 pstat.resize( iface_ents.size() ); 04375 result = mbImpl->tag_get_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set entities" ); 04376 result = mbImpl->tag_get_data( pstatus_tag(), &( *rit ), 1, &set_pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set" ); 04377 rmv_ents.clear(); 04378 for( rit2 = iface_ents.begin(), i = 0; rit2 != iface_ents.end(); ++rit2, i++ ) 04379 { 04380 if( !( pstat[i] & PSTATUS_INTERFACE ) ) 04381 { 04382 rmv_ents.insert( *rit2 ); 04383 pstat[i] = 0x0; 04384 } 04385 } 04386 result = mbImpl->remove_entities( *rit, rmv_ents );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" ); 04387 04388 if( !( set_pstat & PSTATUS_NOT_OWNED ) ) continue; 04389 // If we're here, we need to set the notowned status on (remaining) set contents 04390 04391 // Remove rmv_ents from the contents list 04392 iface_ents = subtract( iface_ents, rmv_ents ); 04393 // Compress the pstat vector (removing 0x0's) 04394 std::remove_if( pstat.begin(), pstat.end(), 04395 std::bind( std::equal_to< unsigned char >(), std::placeholders::_1, 0x0 ) ); 04396 // std::bind2nd(std::equal_to<unsigned char>(), 0x0)); 04397 // https://stackoverflow.com/questions/32739018/a-replacement-for-stdbind2nd 04398 // Fold the not_owned bit into remaining values 04399 unsigned int sz = iface_ents.size(); 04400 for( i = 0; i < sz; i++ ) 04401 pstat[i] |= PSTATUS_NOT_OWNED; 04402 04403 // Set the tag on the entities 04404 result = mbImpl->tag_set_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus values for interface set entities" ); 04405 } 04406 04407 return MB_SUCCESS; 04408 } 04409 04410 ErrorCode ParallelComm::set_pstatus_entities( Range& pstatus_ents, 04411 unsigned char pstatus_val, 04412 bool lower_dim_ents, 04413 bool verts_too, 04414 int operation ) 04415 { 04416 std::vector< unsigned char > pstatus_vals( pstatus_ents.size() ); 04417 Range all_ents, *range_ptr = &pstatus_ents; 04418 ErrorCode result; 04419 if( lower_dim_ents || verts_too ) 04420 { 04421 all_ents = pstatus_ents; 04422 range_ptr = &all_ents; 04423 int start_dim = ( lower_dim_ents ? mbImpl->dimension_from_handle( *pstatus_ents.rbegin() ) - 1 : 0 ); 04424 for( ; start_dim >= 0; start_dim-- ) 04425 { 04426 result = mbImpl->get_adjacencies( all_ents, start_dim, true, all_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get adjacencies for pstatus entities" ); 04427 } 04428 } 04429 if( Interface::UNION == operation ) 04430 { 04431 result = mbImpl->tag_get_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 04432 for( unsigned int i = 0; i < pstatus_vals.size(); i++ ) 04433 pstatus_vals[i] |= pstatus_val; 04434 } 04435 else 04436 { 04437 for( unsigned int i = 0; i < pstatus_vals.size(); i++ ) 04438 pstatus_vals[i] = pstatus_val; 04439 } 04440 result = mbImpl->tag_set_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 04441 04442 return MB_SUCCESS; 04443 } 04444 04445 ErrorCode ParallelComm::set_pstatus_entities( EntityHandle* pstatus_ents, 04446 int num_ents, 04447 unsigned char pstatus_val, 04448 bool lower_dim_ents, 04449 bool verts_too, 04450 int operation ) 04451 { 04452 std::vector< unsigned char > pstatus_vals( num_ents ); 04453 ErrorCode result; 04454 if( lower_dim_ents || verts_too ) 04455 { 04456 // In this case, call the range-based version 04457 Range tmp_range; 04458 std::copy( pstatus_ents, pstatus_ents + num_ents, range_inserter( tmp_range ) ); 04459 return set_pstatus_entities( tmp_range, pstatus_val, lower_dim_ents, verts_too, operation ); 04460 } 04461 04462 if( Interface::UNION == operation ) 04463 { 04464 result = mbImpl->tag_get_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 04465 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ ) 04466 pstatus_vals[i] |= pstatus_val; 04467 } 04468 else 04469 { 04470 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ ) 04471 pstatus_vals[i] = pstatus_val; 04472 } 04473 result = mbImpl->tag_set_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 04474 04475 return MB_SUCCESS; 04476 } 04477 04478 static size_t choose_owner_idx( const std::vector< unsigned >& proc_list ) 04479 { 04480 // Try to assign owners randomly so we get a good distribution, 04481 // (note: specifying the same seed on all procs is essential) 04482 unsigned val = 0; 04483 for( size_t i = 0; i < proc_list.size(); i++ ) 04484 val ^= proc_list[i]; 04485 srand( (int)( val ) ); 04486 return rand() % proc_list.size(); 04487 } 04488 04489 struct set_tuple 04490 { 04491 unsigned idx; 04492 unsigned proc; 04493 EntityHandle handle; 04494 inline bool operator<( set_tuple other ) const 04495 { 04496 return ( idx == other.idx ) ? ( proc < other.proc ) : ( idx < other.idx ); 04497 } 04498 }; 04499 04500 ErrorCode ParallelComm::resolve_shared_sets( EntityHandle file, const Tag* idtag ) 04501 { 04502 // Find all sets with any of the following tags: 04503 const char* const shared_set_tag_names[] = { GEOM_DIMENSION_TAG_NAME, MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, 04504 NEUMANN_SET_TAG_NAME, PARALLEL_PARTITION_TAG_NAME }; 04505 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] ); 04506 Range candidate_sets; 04507 ErrorCode result = MB_FAILURE; 04508 04509 // If we're not given an ID tag to use to globally identify sets, 04510 // then fall back to using known tag values 04511 if( !idtag ) 04512 { 04513 Tag gid, tag; 04514 gid = mbImpl->globalId_tag(); 04515 if( NULL != gid ) result = mbImpl->tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tag ); 04516 if( MB_SUCCESS == result ) 04517 { 04518 for( int d = 0; d < 4; d++ ) 04519 { 04520 candidate_sets.clear(); 04521 const void* vals[] = { &d }; 04522 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, vals, 1, candidate_sets ); 04523 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, gid ); 04524 } 04525 } 04526 04527 for( int i = 1; i < num_tags; i++ ) 04528 { 04529 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag ); 04530 if( MB_SUCCESS == result ) 04531 { 04532 candidate_sets.clear(); 04533 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets ); 04534 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, tag ); 04535 } 04536 } 04537 04538 return MB_SUCCESS; 04539 } 04540 04541 for( int i = 0; i < num_tags; i++ ) 04542 { 04543 Tag tag; 04544 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag, MB_TAG_ANY ); 04545 if( MB_SUCCESS != result ) continue; 04546 04547 mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets, Interface::UNION ); 04548 } 04549 04550 // Find any additional sets that contain shared entities 04551 Range::iterator hint = candidate_sets.begin(); 04552 Range all_sets; 04553 mbImpl->get_entities_by_type( file, MBENTITYSET, all_sets ); 04554 all_sets = subtract( all_sets, candidate_sets ); 04555 Range::iterator it = all_sets.begin(); 04556 while( it != all_sets.end() ) 04557 { 04558 Range contents; 04559 mbImpl->get_entities_by_handle( *it, contents ); 04560 contents.erase( contents.lower_bound( MBENTITYSET ), contents.end() ); 04561 filter_pstatus( contents, PSTATUS_SHARED, PSTATUS_OR ); 04562 if( contents.empty() ) 04563 { 04564 ++it; 04565 } 04566 else 04567 { 04568 hint = candidate_sets.insert( hint, *it ); 04569 it = all_sets.erase( it ); 04570 } 04571 } 04572 04573 // Find any additionl sets that contain or are parents of potential shared sets 04574 Range prev_list = candidate_sets; 04575 while( !prev_list.empty() ) 04576 { 04577 it = all_sets.begin(); 04578 Range new_list; 04579 hint = new_list.begin(); 04580 while( it != all_sets.end() ) 04581 { 04582 Range contents; 04583 mbImpl->get_entities_by_type( *it, MBENTITYSET, contents ); 04584 if( !intersect( prev_list, contents ).empty() ) 04585 { 04586 hint = new_list.insert( hint, *it ); 04587 it = all_sets.erase( it ); 04588 } 04589 else 04590 { 04591 new_list.clear(); 04592 mbImpl->get_child_meshsets( *it, contents ); 04593 if( !intersect( prev_list, contents ).empty() ) 04594 { 04595 hint = new_list.insert( hint, *it ); 04596 it = all_sets.erase( it ); 04597 } 04598 else 04599 { 04600 ++it; 04601 } 04602 } 04603 } 04604 04605 candidate_sets.merge( new_list ); 04606 prev_list.swap( new_list ); 04607 } 04608 04609 return resolve_shared_sets( candidate_sets, *idtag ); 04610 } 04611 04612 #ifndef NDEBUG 04613 bool is_sorted_unique( std::vector< unsigned >& v ) 04614 { 04615 for( size_t i = 1; i < v.size(); i++ ) 04616 if( v[i - 1] >= v[i] ) return false; 04617 return true; 04618 } 04619 #endif 04620 04621 ErrorCode ParallelComm::resolve_shared_sets( Range& sets, Tag idtag ) 04622 { 04623 ErrorCode result; 04624 const unsigned rk = proc_config().proc_rank(); 04625 MPI_Comm cm = proc_config().proc_comm(); 04626 04627 // Build sharing list for all sets 04628 04629 // Get ids for sets in a vector, to pass to gs 04630 std::vector< long > larray; // Allocate sufficient space for longs 04631 std::vector< Ulong > handles; 04632 Range tmp_sets; 04633 // The id tag can be size 4 or size 8 04634 // Based on that, convert to int or to long, similarly to what we do 04635 // for resolving shared vertices; 04636 // This code must work on 32 bit too, where long is 4 bytes, also 04637 // so test first size 4, then we should be fine 04638 DataType tag_type; 04639 result = mbImpl->tag_get_data_type( idtag, tag_type );MB_CHK_SET_ERR( result, "Failed getting tag data type" ); 04640 int bytes_per_tag; 04641 result = mbImpl->tag_get_bytes( idtag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed getting number of bytes per tag" ); 04642 // On 64 bits, long and int are different 04643 // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?) 04644 04645 for( Range::iterator rit = sets.begin(); rit != sets.end(); ++rit ) 04646 { 04647 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) ) 04648 { // It is a special id tag 04649 long dum; 04650 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum ); 04651 if( MB_SUCCESS == result ) 04652 { 04653 larray.push_back( dum ); 04654 handles.push_back( *rit ); 04655 tmp_sets.insert( tmp_sets.end(), *rit ); 04656 } 04657 } 04658 else if( 4 == bytes_per_tag ) 04659 { // Must be GLOBAL_ID tag or MATERIAL_ID, etc 04660 int dum; 04661 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum ); 04662 if( MB_SUCCESS == result ) 04663 { 04664 larray.push_back( dum ); 04665 handles.push_back( *rit ); 04666 tmp_sets.insert( tmp_sets.end(), *rit ); 04667 } 04668 } 04669 } 04670 04671 const size_t nsets = handles.size(); 04672 04673 // Get handle array for sets 04674 // This is not true on windows machine, 64 bits: entity handle is 64 bit, long is 32 04675 // assert(sizeof(EntityHandle) <= sizeof(unsigned long)); 04676 04677 // Do communication of data 04678 gs_data::crystal_data* cd = procConfig.crystal_router(); 04679 gs_data* gsd = new gs_data(); 04680 result = gsd->initialize( nsets, &larray[0], &handles[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" ); 04681 04682 // Convert from global IDs grouped by process rank to list 04683 // of <idx, rank> pairs so that we can sort primarily 04684 // by idx and secondarily by rank (we want lists of procs for each 04685 // idx, not lists if indices for each proc). 04686 size_t ntuple = 0; 04687 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ ) 04688 ntuple += gsd->nlinfo->_nshared[p]; 04689 std::vector< set_tuple > tuples; 04690 tuples.reserve( ntuple ); 04691 size_t j = 0; 04692 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ ) 04693 { 04694 for( unsigned np = 0; np < gsd->nlinfo->_nshared[p]; np++ ) 04695 { 04696 set_tuple t; 04697 t.idx = gsd->nlinfo->_sh_ind[j]; 04698 t.proc = gsd->nlinfo->_target[p]; 04699 t.handle = gsd->nlinfo->_ulabels[j]; 04700 tuples.push_back( t ); 04701 j++; 04702 } 04703 } 04704 std::sort( tuples.begin(), tuples.end() ); 04705 04706 // Release crystal router stuff 04707 gsd->reset(); 04708 delete gsd; 04709 04710 // Storing sharing data for each set 04711 size_t ti = 0; 04712 unsigned idx = 0; 04713 std::vector< unsigned > procs; 04714 Range::iterator si = tmp_sets.begin(); 04715 while( si != tmp_sets.end() && ti < tuples.size() ) 04716 { 04717 assert( idx <= tuples[ti].idx ); 04718 if( idx < tuples[ti].idx ) si += ( tuples[ti].idx - idx ); 04719 idx = tuples[ti].idx; 04720 04721 procs.clear(); 04722 size_t ti_init = ti; 04723 while( ti < tuples.size() && tuples[ti].idx == idx ) 04724 { 04725 procs.push_back( tuples[ti].proc ); 04726 ++ti; 04727 } 04728 assert( is_sorted_unique( procs ) ); 04729 04730 result = sharedSetData->set_sharing_procs( *si, procs ); 04731 if( MB_SUCCESS != result ) 04732 { 04733 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl; 04734 std::cerr.flush(); 04735 MPI_Abort( cm, 1 ); 04736 } 04737 04738 // Add this proc to list of sharing procs in correct position 04739 // so that all procs select owner based on same list 04740 std::vector< unsigned >::iterator it = std::lower_bound( procs.begin(), procs.end(), rk ); 04741 assert( it == procs.end() || *it > rk ); 04742 procs.insert( it, rk ); 04743 size_t owner_idx = choose_owner_idx( procs ); 04744 EntityHandle owner_handle; 04745 if( procs[owner_idx] == rk ) 04746 owner_handle = *si; 04747 else if( procs[owner_idx] > rk ) 04748 owner_handle = tuples[ti_init + owner_idx - 1].handle; 04749 else 04750 owner_handle = tuples[ti_init + owner_idx].handle; 04751 result = sharedSetData->set_owner( *si, procs[owner_idx], owner_handle ); 04752 if( MB_SUCCESS != result ) 04753 { 04754 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl; 04755 std::cerr.flush(); 04756 MPI_Abort( cm, 1 ); 04757 } 04758 04759 ++si; 04760 ++idx; 04761 } 04762 04763 return MB_SUCCESS; 04764 } 04765 // populate sets with ghost entities, if necessary 04766 ErrorCode ParallelComm::augment_default_sets_with_ghosts( EntityHandle file_set ) 04767 { 04768 // gather all default sets we are interested in, material, neumann, etc 04769 // we will skip geometry sets, because they are not uniquely identified with their tag value 04770 // maybe we will add another tag, like category 04771 04772 if( procConfig.proc_size() < 2 ) return MB_SUCCESS; // no reason to stop by 04773 const char* const shared_set_tag_names[] = { MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, NEUMANN_SET_TAG_NAME, 04774 PARALLEL_PARTITION_TAG_NAME }; 04775 04776 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] ); 04777 04778 Range* rangeSets = new Range[num_tags]; 04779 Tag* tags = new Tag[num_tags + 1]; // one extra for global id tag, which is an int, so far 04780 04781 int my_rank = rank(); 04782 int** tagVals = new int*[num_tags]; 04783 for( int i = 0; i < num_tags; i++ ) 04784 tagVals[i] = NULL; 04785 ErrorCode rval; 04786 04787 // for each tag, we keep a local map, from the value to the actual set with that value 04788 // we assume that the tag values are unique, for a given set, otherwise we 04789 // do not know to which set to add the entity 04790 04791 typedef std::map< int, EntityHandle > MVal; 04792 typedef std::map< int, EntityHandle >::iterator itMVal; 04793 MVal* localMaps = new MVal[num_tags]; 04794 04795 for( int i = 0; i < num_tags; i++ ) 04796 { 04797 04798 rval = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tags[i], MB_TAG_ANY ); 04799 if( MB_SUCCESS != rval ) continue; 04800 rval = mbImpl->get_entities_by_type_and_tag( file_set, MBENTITYSET, &( tags[i] ), 0, 1, rangeSets[i], 04801 Interface::UNION );MB_CHK_SET_ERR( rval, "can't get sets with a tag" ); 04802 04803 if( rangeSets[i].size() > 0 ) 04804 { 04805 tagVals[i] = new int[rangeSets[i].size()]; 04806 // fill up with the tag values 04807 rval = mbImpl->tag_get_data( tags[i], rangeSets[i], tagVals[i] );MB_CHK_SET_ERR( rval, "can't get set tag values" ); 04808 // now for inverse mapping: 04809 for( int j = 0; j < (int)rangeSets[i].size(); j++ ) 04810 { 04811 localMaps[i][tagVals[i][j]] = rangeSets[i][j]; 04812 } 04813 } 04814 } 04815 // get the global id tag too 04816 tags[num_tags] = mbImpl->globalId_tag(); 04817 04818 TupleList remoteEnts; 04819 // processor to send to, type of tag (0-mat,) tag value, remote handle 04820 // 1-diri 04821 // 2-neum 04822 // 3-part 04823 // 04824 int initialSize = (int)sharedEnts.size(); // estimate that on average, each shared ent 04825 // will be sent to one processor, for one tag 04826 // we will actually send only entities that are owned locally, and from those 04827 // only those that do have a special tag (material, neumann, etc) 04828 // if we exceed the capacity, we resize the tuple 04829 remoteEnts.initialize( 3, 0, 1, 0, initialSize ); 04830 remoteEnts.enableWriteAccess(); 04831 04832 // now, for each owned entity, get the remote handle(s) and Proc(s), and verify if it 04833 // belongs to one of the sets; if yes, create a tuple and append it 04834 04835 std::set< EntityHandle > own_and_sha; 04836 int ir = 0, jr = 0; 04837 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 04838 { 04839 // ghosted eh 04840 EntityHandle geh = *vit; 04841 if( own_and_sha.find( geh ) != own_and_sha.end() ) // already encountered 04842 continue; 04843 int procs[MAX_SHARING_PROCS]; 04844 EntityHandle handles[MAX_SHARING_PROCS]; 04845 int nprocs; 04846 unsigned char pstat; 04847 rval = get_sharing_data( geh, procs, handles, pstat, nprocs ); 04848 if( rval != MB_SUCCESS ) 04849 { 04850 for( int i = 0; i < num_tags; i++ ) 04851 delete[] tagVals[i]; 04852 delete[] tagVals; 04853 04854 MB_CHK_SET_ERR( rval, "Failed to get sharing data" ); 04855 } 04856 if( pstat & PSTATUS_NOT_OWNED ) continue; // we will send info only for entities that we own 04857 own_and_sha.insert( geh ); 04858 for( int i = 0; i < num_tags; i++ ) 04859 { 04860 for( int j = 0; j < (int)rangeSets[i].size(); j++ ) 04861 { 04862 EntityHandle specialSet = rangeSets[i][j]; // this set has tag i, value tagVals[i][j]; 04863 if( mbImpl->contains_entities( specialSet, &geh, 1 ) ) 04864 { 04865 // this ghosted entity is in a special set, so form the tuple 04866 // to send to the processors that do not own this 04867 for( int k = 0; k < nprocs; k++ ) 04868 { 04869 if( procs[k] != my_rank ) 04870 { 04871 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 ) 04872 { 04873 // resize, so we do not overflow 04874 int oldSize = remoteEnts.get_max(); 04875 // increase with 50% the capacity 04876 remoteEnts.resize( oldSize + oldSize / 2 + 1 ); 04877 } 04878 remoteEnts.vi_wr[ir++] = procs[k]; // send to proc 04879 remoteEnts.vi_wr[ir++] = i; // for the tags [i] (0-3) 04880 remoteEnts.vi_wr[ir++] = tagVals[i][j]; // actual value of the tag 04881 remoteEnts.vul_wr[jr++] = handles[k]; 04882 remoteEnts.inc_n(); 04883 } 04884 } 04885 } 04886 } 04887 } 04888 // if the local entity has a global id, send it too, so we avoid 04889 // another "exchange_tags" for global id 04890 int gid; 04891 rval = mbImpl->tag_get_data( tags[num_tags], &geh, 1, &gid );MB_CHK_SET_ERR( rval, "Failed to get global id" ); 04892 if( gid != 0 ) 04893 { 04894 for( int k = 0; k < nprocs; k++ ) 04895 { 04896 if( procs[k] != my_rank ) 04897 { 04898 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 ) 04899 { 04900 // resize, so we do not overflow 04901 int oldSize = remoteEnts.get_max(); 04902 // increase with 50% the capacity 04903 remoteEnts.resize( oldSize + oldSize / 2 + 1 ); 04904 } 04905 remoteEnts.vi_wr[ir++] = procs[k]; // send to proc 04906 remoteEnts.vi_wr[ir++] = num_tags; // for the tags [j] (4) 04907 remoteEnts.vi_wr[ir++] = gid; // actual value of the tag 04908 remoteEnts.vul_wr[jr++] = handles[k]; 04909 remoteEnts.inc_n(); 04910 } 04911 } 04912 } 04913 } 04914 04915 #ifndef NDEBUG 04916 if( my_rank == 1 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 1, before augment routing" ); 04917 MPI_Barrier( procConfig.proc_comm() ); 04918 int sentEnts = remoteEnts.get_n(); 04919 assert( ( sentEnts == jr ) && ( 3 * sentEnts == ir ) ); 04920 #endif 04921 // exchange the info now, and send to 04922 gs_data::crystal_data* cd = this->procConfig.crystal_router(); 04923 // All communication happens here; no other mpi calls 04924 // Also, this is a collective call 04925 rval = cd->gs_transfer( 1, remoteEnts, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" ); 04926 #ifndef NDEBUG 04927 if( my_rank == 0 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 0, after augment routing" ); 04928 MPI_Barrier( procConfig.proc_comm() ); 04929 #endif 04930 04931 // now process the data received from other processors 04932 int received = remoteEnts.get_n(); 04933 for( int i = 0; i < received; i++ ) 04934 { 04935 // int from = ents_to_delete.vi_rd[i]; 04936 EntityHandle geh = (EntityHandle)remoteEnts.vul_rd[i]; 04937 int from_proc = remoteEnts.vi_rd[3 * i]; 04938 if( my_rank == from_proc ) 04939 std::cout << " unexpected receive from my rank " << my_rank << " during augmenting with ghosts\n "; 04940 int tag_type = remoteEnts.vi_rd[3 * i + 1]; 04941 assert( ( 0 <= tag_type ) && ( tag_type <= num_tags ) ); 04942 int value = remoteEnts.vi_rd[3 * i + 2]; 04943 if( tag_type == num_tags ) 04944 { 04945 // it is global id 04946 rval = mbImpl->tag_set_data( tags[num_tags], &geh, 1, &value );MB_CHK_SET_ERR( rval, "Error in setting gid tag" ); 04947 } 04948 else 04949 { 04950 // now, based on value and tag type, see if we have that value in the map 04951 MVal& lmap = localMaps[tag_type]; 04952 itMVal itm = lmap.find( value ); 04953 if( itm == lmap.end() ) 04954 { 04955 // the value was not found yet in the local map, so we have to create the set 04956 EntityHandle newSet; 04957 rval = mbImpl->create_meshset( MESHSET_SET, newSet );MB_CHK_SET_ERR( rval, "can't create new set" ); 04958 lmap[value] = newSet; 04959 // set the tag value 04960 rval = mbImpl->tag_set_data( tags[tag_type], &newSet, 1, &value );MB_CHK_SET_ERR( rval, "can't set tag for new set" ); 04961 04962 // we also need to add the new created set to the file set, if not null 04963 if( file_set ) 04964 { 04965 rval = mbImpl->add_entities( file_set, &newSet, 1 );MB_CHK_SET_ERR( rval, "can't add new set to the file set" ); 04966 } 04967 } 04968 // add the entity to the set pointed to by the map 04969 rval = mbImpl->add_entities( lmap[value], &geh, 1 );MB_CHK_SET_ERR( rval, "can't add ghost ent to the set" ); 04970 } 04971 } 04972 04973 for( int i = 0; i < num_tags; i++ ) 04974 delete[] tagVals[i]; 04975 delete[] tagVals; 04976 delete[] rangeSets; 04977 delete[] tags; 04978 delete[] localMaps; 04979 return MB_SUCCESS; 04980 } 04981 ErrorCode ParallelComm::create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim ) 04982 { 04983 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs; 04984 04985 // Build up the list of shared entities 04986 int procs[MAX_SHARING_PROCS]; 04987 EntityHandle handles[MAX_SHARING_PROCS]; 04988 ErrorCode result; 04989 int nprocs; 04990 unsigned char pstat; 04991 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 04992 { 04993 if( shared_dim != -1 && mbImpl->dimension_from_handle( *vit ) > shared_dim ) continue; 04994 result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 04995 std::sort( procs, procs + nprocs ); 04996 std::vector< int > tmp_procs( procs, procs + nprocs ); 04997 assert( tmp_procs.size() != 2 ); 04998 proc_nvecs[tmp_procs].push_back( *vit ); 04999 } 05000 05001 Skinner skinner( mbImpl ); 05002 Range skin_ents[4]; 05003 result = mbImpl->get_entities_by_dimension( this_set, resolve_dim, skin_ents[resolve_dim] );MB_CHK_SET_ERR( result, "Failed to get skin entities by dimension" ); 05004 result = 05005 skinner.find_skin( this_set, skin_ents[resolve_dim], false, skin_ents[resolve_dim - 1], 0, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" ); 05006 if( shared_dim > 1 ) 05007 { 05008 result = mbImpl->get_adjacencies( skin_ents[resolve_dim - 1], resolve_dim - 2, true, skin_ents[resolve_dim - 2], 05009 Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" ); 05010 } 05011 05012 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs ); 05013 05014 return create_interface_sets( proc_nvecs ); 05015 } 05016 05017 ErrorCode ParallelComm::create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs ) 05018 { 05019 if( proc_nvecs.empty() ) return MB_SUCCESS; 05020 05021 int proc_ids[MAX_SHARING_PROCS]; 05022 EntityHandle proc_handles[MAX_SHARING_PROCS]; 05023 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05024 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in create_interface_sets" ); 05025 Range::iterator rit; 05026 05027 // Create interface sets, tag them, and tag their contents with iface set tag 05028 std::vector< unsigned char > pstatus; 05029 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator vit = proc_nvecs.begin(); 05030 vit != proc_nvecs.end(); ++vit ) 05031 { 05032 // Create the set 05033 EntityHandle new_set; 05034 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" ); 05035 interfaceSets.insert( new_set ); 05036 05037 // Add entities 05038 assert( !vit->second.empty() ); 05039 result = mbImpl->add_entities( new_set, &( vit->second )[0], ( vit->second ).size() );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 05040 // Tag set with the proc rank(s) 05041 if( vit->first.size() == 1 ) 05042 { 05043 assert( ( vit->first )[0] != (int)procConfig.proc_rank() ); 05044 result = mbImpl->tag_set_data( shp_tag, &new_set, 1, &( vit->first )[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05045 proc_handles[0] = 0; 05046 result = mbImpl->tag_set_data( shh_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05047 } 05048 else 05049 { 05050 // Pad tag data out to MAX_SHARING_PROCS with -1 05051 if( vit->first.size() > MAX_SHARING_PROCS ) 05052 { 05053 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_set ) ) 05054 << ' ' << ID_FROM_HANDLE( new_set ) << " on process " << proc_config().proc_rank() 05055 << std::endl; 05056 std::cerr.flush(); 05057 MPI_Abort( proc_config().proc_comm(), 66 ); 05058 } 05059 // assert(vit->first.size() <= MAX_SHARING_PROCS); 05060 std::copy( vit->first.begin(), vit->first.end(), proc_ids ); 05061 std::fill( proc_ids + vit->first.size(), proc_ids + MAX_SHARING_PROCS, -1 ); 05062 result = mbImpl->tag_set_data( shps_tag, &new_set, 1, proc_ids );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05063 unsigned int ind = std::find( proc_ids, proc_ids + vit->first.size(), procConfig.proc_rank() ) - proc_ids; 05064 assert( ind < vit->first.size() ); 05065 std::fill( proc_handles, proc_handles + MAX_SHARING_PROCS, 0 ); 05066 proc_handles[ind] = new_set; 05067 result = mbImpl->tag_set_data( shhs_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 05068 } 05069 05070 // Get the owning proc, then set the pstatus tag on iface set 05071 int min_proc = ( vit->first )[0]; 05072 unsigned char pval = ( PSTATUS_SHARED | PSTATUS_INTERFACE ); 05073 if( min_proc < (int)procConfig.proc_rank() ) pval |= PSTATUS_NOT_OWNED; 05074 if( vit->first.size() > 1 ) pval |= PSTATUS_MULTISHARED; 05075 result = mbImpl->tag_set_data( pstat_tag, &new_set, 1, &pval );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" ); 05076 05077 // Tag the vertices with the same thing 05078 pstatus.clear(); 05079 std::vector< EntityHandle > verts; 05080 for( std::vector< EntityHandle >::iterator v2it = ( vit->second ).begin(); v2it != ( vit->second ).end(); 05081 ++v2it ) 05082 if( mbImpl->type_from_handle( *v2it ) == MBVERTEX ) verts.push_back( *v2it ); 05083 pstatus.resize( verts.size(), pval ); 05084 if( !verts.empty() ) 05085 { 05086 result = mbImpl->tag_set_data( pstat_tag, &verts[0], verts.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set vertices with pstatus" ); 05087 } 05088 } 05089 05090 return MB_SUCCESS; 05091 } 05092 05093 ErrorCode ParallelComm::create_iface_pc_links() 05094 { 05095 // Now that we've resolved the entities in the iface sets, 05096 // set parent/child links between the iface sets 05097 05098 // First tag all entities in the iface sets 05099 Tag tmp_iface_tag; 05100 EntityHandle tmp_iface_set = 0; 05101 ErrorCode result = mbImpl->tag_get_handle( "__tmp_iface", 1, MB_TYPE_HANDLE, tmp_iface_tag, 05102 MB_TAG_DENSE | MB_TAG_CREAT, &tmp_iface_set );MB_CHK_SET_ERR( result, "Failed to create temporary interface set tag" ); 05103 05104 Range iface_ents; 05105 std::vector< EntityHandle > tag_vals; 05106 Range::iterator rit; 05107 05108 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 05109 { 05110 // tag entities with interface set 05111 iface_ents.clear(); 05112 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in interface set" ); 05113 05114 if( iface_ents.empty() ) continue; 05115 05116 tag_vals.resize( iface_ents.size() ); 05117 std::fill( tag_vals.begin(), tag_vals.end(), *rit ); 05118 result = mbImpl->tag_set_data( tmp_iface_tag, iface_ents, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to tag iface entities with interface set" ); 05119 } 05120 05121 // Now go back through interface sets and add parent/child links 05122 Range tmp_ents2; 05123 for( int d = 2; d >= 0; d-- ) 05124 { 05125 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 05126 { 05127 // Get entities on this interface 05128 iface_ents.clear(); 05129 result = mbImpl->get_entities_by_handle( *rit, iface_ents, true );MB_CHK_SET_ERR( result, "Failed to get entities by handle" ); 05130 if( iface_ents.empty() || mbImpl->dimension_from_handle( *iface_ents.rbegin() ) != d ) continue; 05131 05132 // Get higher-dimensional entities and their interface sets 05133 result = mbImpl->get_adjacencies( &( *iface_ents.begin() ), 1, d + 1, false, tmp_ents2 );MB_CHK_SET_ERR( result, "Failed to get adjacencies for interface sets" ); 05134 tag_vals.resize( tmp_ents2.size() ); 05135 result = mbImpl->tag_get_data( tmp_iface_tag, tmp_ents2, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to get tmp iface tag for interface sets" ); 05136 05137 // Go through and for any on interface make it a parent 05138 EntityHandle last_set = 0; 05139 for( unsigned int i = 0; i < tag_vals.size(); i++ ) 05140 { 05141 if( tag_vals[i] && tag_vals[i] != last_set ) 05142 { 05143 result = mbImpl->add_parent_child( tag_vals[i], *rit );MB_CHK_SET_ERR( result, "Failed to add parent/child link for interface set" ); 05144 last_set = tag_vals[i]; 05145 } 05146 } 05147 } 05148 } 05149 05150 // Delete the temporary tag 05151 result = mbImpl->tag_delete( tmp_iface_tag );MB_CHK_SET_ERR( result, "Failed to delete tmp iface tag" ); 05152 05153 return MB_SUCCESS; 05154 } 05155 05156 ErrorCode ParallelComm::get_proc_nvecs( int resolve_dim, 05157 int shared_dim, 05158 Range* skin_ents, 05159 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs ) 05160 { 05161 // Set sharing procs tags on other skin ents 05162 ErrorCode result; 05163 const EntityHandle* connect; 05164 int num_connect; 05165 std::set< int > sharing_procs; 05166 std::vector< EntityHandle > dum_connect; 05167 std::vector< int > sp_vec; 05168 05169 for( int d = 3; d > 0; d-- ) 05170 { 05171 if( resolve_dim == d ) continue; 05172 05173 for( Range::iterator rit = skin_ents[d].begin(); rit != skin_ents[d].end(); ++rit ) 05174 { 05175 // Get connectivity 05176 result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect );MB_CHK_SET_ERR( result, "Failed to get connectivity on non-vertex skin entities" ); 05177 05178 int op = ( resolve_dim < shared_dim ? Interface::UNION : Interface::INTERSECT ); 05179 result = get_sharing_data( connect, num_connect, sharing_procs, op );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_proc_nvecs" ); 05180 if( sharing_procs.empty() || 05181 ( sharing_procs.size() == 1 && *sharing_procs.begin() == (int)procConfig.proc_rank() ) ) 05182 continue; 05183 05184 // Need to specify sharing data correctly for entities or they will 05185 // end up in a different interface set than corresponding vertices 05186 if( sharing_procs.size() == 2 ) 05187 { 05188 std::set< int >::iterator it = sharing_procs.find( proc_config().proc_rank() ); 05189 assert( it != sharing_procs.end() ); 05190 sharing_procs.erase( it ); 05191 } 05192 05193 // Intersection is the owning proc(s) for this skin ent 05194 sp_vec.clear(); 05195 std::copy( sharing_procs.begin(), sharing_procs.end(), std::back_inserter( sp_vec ) ); 05196 assert( sp_vec.size() != 2 ); 05197 proc_nvecs[sp_vec].push_back( *rit ); 05198 } 05199 } 05200 05201 #ifndef NDEBUG 05202 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05203 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05204 mit != proc_nvecs.end(); ++mit ) 05205 { 05206 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05207 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05208 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05209 assert( vit == tmp_vec.end() ); 05210 } 05211 #endif 05212 05213 return MB_SUCCESS; 05214 } 05215 05216 // Overloaded form of tag_shared_verts 05217 // Tuple coming in is of form (arbitrary value, remoteProc, localHandle, remoteHandle) 05218 // Also will check for doubles in the list if the list is sorted 05219 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents, 05220 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs, 05221 Range& /*proc_verts*/, 05222 unsigned int i_extra ) 05223 { 05224 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05225 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" ); 05226 05227 unsigned int j = 0, i = 0; 05228 std::vector< int > sharing_procs, sharing_procs2, tag_procs; 05229 std::vector< EntityHandle > sharing_handles, sharing_handles2, tag_lhandles, tag_rhandles; 05230 std::vector< unsigned char > pstatus; 05231 05232 // Were on tuple j/2 05233 if( i_extra ) i += i_extra; 05234 while( j < 2 * shared_ents.get_n() ) 05235 { 05236 // Count & accumulate sharing procs 05237 EntityHandle this_ent = shared_ents.vul_rd[j], other_ent = 0; 05238 int other_proc = -1; 05239 while( j < 2 * shared_ents.get_n() && shared_ents.vul_rd[j] == this_ent ) 05240 { 05241 j++; 05242 // Shouldn't have same proc 05243 assert( shared_ents.vi_rd[i] != (int)procConfig.proc_rank() ); 05244 // Grab the remote data if its not a dublicate 05245 if( shared_ents.vul_rd[j] != other_ent || shared_ents.vi_rd[i] != other_proc ) 05246 { 05247 assert( 0 != shared_ents.vul_rd[j] ); 05248 sharing_procs.push_back( shared_ents.vi_rd[i] ); 05249 sharing_handles.push_back( shared_ents.vul_rd[j] ); 05250 } 05251 other_proc = shared_ents.vi_rd[i]; 05252 other_ent = shared_ents.vul_rd[j]; 05253 j++; 05254 i += 1 + i_extra; 05255 } 05256 05257 if( sharing_procs.size() > 1 ) 05258 { 05259 // Add current proc/handle to list 05260 sharing_procs.push_back( procConfig.proc_rank() ); 05261 sharing_handles.push_back( this_ent ); 05262 05263 // Sort sharing_procs and sharing_handles such that 05264 // sharing_procs is in ascending order. Use temporary 05265 // lists and binary search to re-order sharing_handles. 05266 sharing_procs2 = sharing_procs; 05267 std::sort( sharing_procs2.begin(), sharing_procs2.end() ); 05268 sharing_handles2.resize( sharing_handles.size() ); 05269 for( size_t k = 0; k < sharing_handles.size(); k++ ) 05270 { 05271 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) - 05272 sharing_procs2.begin(); 05273 sharing_handles2[idx] = sharing_handles[k]; 05274 } 05275 sharing_procs.swap( sharing_procs2 ); 05276 sharing_handles.swap( sharing_handles2 ); 05277 } 05278 05279 assert( sharing_procs.size() != 2 ); 05280 proc_nvecs[sharing_procs].push_back( this_ent ); 05281 05282 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED ); 05283 if( sharing_procs.size() == 1 ) 05284 { 05285 tag_procs.push_back( sharing_procs[0] ); 05286 tag_lhandles.push_back( this_ent ); 05287 tag_rhandles.push_back( sharing_handles[0] ); 05288 pstatus.push_back( share_flag ); 05289 } 05290 else 05291 { 05292 // Pad lists 05293 // assert(sharing_procs.size() <= MAX_SHARING_PROCS); 05294 if( sharing_procs.size() > MAX_SHARING_PROCS ) 05295 { 05296 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process " 05297 << proc_config().proc_rank() << std::endl; 05298 std::cerr.flush(); 05299 MPI_Abort( proc_config().proc_comm(), 66 ); 05300 } 05301 sharing_procs.resize( MAX_SHARING_PROCS, -1 ); 05302 sharing_handles.resize( MAX_SHARING_PROCS, 0 ); 05303 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" ); 05304 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" ); 05305 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05306 sharedEnts.insert( this_ent ); 05307 } 05308 05309 // Reset sharing proc(s) tags 05310 sharing_procs.clear(); 05311 sharing_handles.clear(); 05312 } 05313 05314 if( !tag_procs.empty() ) 05315 { 05316 result = mbImpl->tag_set_data( shp_tag, &tag_lhandles[0], tag_procs.size(), &tag_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" ); 05317 result = mbImpl->tag_set_data( shh_tag, &tag_lhandles[0], tag_procs.size(), &tag_rhandles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" ); 05318 result = mbImpl->tag_set_data( pstat_tag, &tag_lhandles[0], tag_procs.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05319 for( std::vector< EntityHandle >::iterator vvt = tag_lhandles.begin(); vvt != tag_lhandles.end(); vvt++ ) 05320 sharedEnts.insert( *vvt ); 05321 } 05322 05323 #ifndef NDEBUG 05324 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05325 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05326 mit != proc_nvecs.end(); ++mit ) 05327 { 05328 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05329 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05330 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05331 assert( vit == tmp_vec.end() ); 05332 } 05333 #endif 05334 05335 return MB_SUCCESS; 05336 } 05337 05338 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents, 05339 Range* skin_ents, 05340 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs, 05341 Range& /*proc_verts*/ ) 05342 { 05343 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag; 05344 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" ); 05345 05346 unsigned int j = 0, i = 0; 05347 std::vector< int > sharing_procs, sharing_procs2; 05348 std::vector< EntityHandle > sharing_handles, sharing_handles2, skin_verts( skin_ents[0].size() ); 05349 for( Range::iterator rit = skin_ents[0].begin(); rit != skin_ents[0].end(); ++rit, i++ ) 05350 skin_verts[i] = *rit; 05351 i = 0; 05352 05353 while( j < 2 * shared_ents.get_n() ) 05354 { 05355 // Count & accumulate sharing procs 05356 int this_idx = shared_ents.vi_rd[j]; 05357 EntityHandle this_ent = skin_verts[this_idx]; 05358 while( j < 2 * shared_ents.get_n() && shared_ents.vi_rd[j] == this_idx ) 05359 { 05360 j++; 05361 // Shouldn't have same proc 05362 assert( shared_ents.vi_rd[j] != (int)procConfig.proc_rank() ); 05363 sharing_procs.push_back( shared_ents.vi_rd[j++] ); 05364 sharing_handles.push_back( shared_ents.vul_rd[i++] ); 05365 } 05366 05367 if( sharing_procs.size() > 1 ) 05368 { 05369 // Add current proc/handle to list 05370 sharing_procs.push_back( procConfig.proc_rank() ); 05371 sharing_handles.push_back( this_ent ); 05372 } 05373 05374 // Sort sharing_procs and sharing_handles such that 05375 // sharing_procs is in ascending order. Use temporary 05376 // lists and binary search to re-order sharing_handles. 05377 sharing_procs2 = sharing_procs; 05378 std::sort( sharing_procs2.begin(), sharing_procs2.end() ); 05379 sharing_handles2.resize( sharing_handles.size() ); 05380 for( size_t k = 0; k < sharing_handles.size(); k++ ) 05381 { 05382 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) - 05383 sharing_procs2.begin(); 05384 sharing_handles2[idx] = sharing_handles[k]; 05385 } 05386 sharing_procs.swap( sharing_procs2 ); 05387 sharing_handles.swap( sharing_handles2 ); 05388 05389 assert( sharing_procs.size() != 2 ); 05390 proc_nvecs[sharing_procs].push_back( this_ent ); 05391 05392 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED ); 05393 if( sharing_procs.size() == 1 ) 05394 { 05395 result = mbImpl->tag_set_data( shp_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" ); 05396 result = mbImpl->tag_set_data( shh_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" ); 05397 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &share_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05398 sharedEnts.insert( this_ent ); 05399 } 05400 else 05401 { 05402 // Pad lists 05403 // assert(sharing_procs.size() <= MAX_SHARING_PROCS); 05404 if( sharing_procs.size() > MAX_SHARING_PROCS ) 05405 { 05406 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process " 05407 << proc_config().proc_rank() << std::endl; 05408 std::cerr.flush(); 05409 MPI_Abort( proc_config().proc_comm(), 66 ); 05410 } 05411 sharing_procs.resize( MAX_SHARING_PROCS, -1 ); 05412 sharing_handles.resize( MAX_SHARING_PROCS, 0 ); 05413 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" ); 05414 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" ); 05415 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" ); 05416 sharedEnts.insert( this_ent ); 05417 } 05418 05419 // Reset sharing proc(s) tags 05420 sharing_procs.clear(); 05421 sharing_handles.clear(); 05422 } 05423 05424 #ifndef NDEBUG 05425 // Shouldn't be any repeated entities in any of the vectors in proc_nvecs 05426 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin(); 05427 mit != proc_nvecs.end(); ++mit ) 05428 { 05429 std::vector< EntityHandle > tmp_vec = ( mit->second ); 05430 std::sort( tmp_vec.begin(), tmp_vec.end() ); 05431 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() ); 05432 assert( vit == tmp_vec.end() ); 05433 } 05434 #endif 05435 05436 return MB_SUCCESS; 05437 } 05438 05439 //! Get processors with which this processor communicates; sets are sorted by processor 05440 ErrorCode ParallelComm::get_interface_procs( std::set< unsigned int >& procs_set, bool get_buffs ) 05441 { 05442 // Make sure the sharing procs vector is empty 05443 procs_set.clear(); 05444 05445 // Pre-load vector of single-proc tag values 05446 unsigned int i, j; 05447 std::vector< int > iface_proc( interfaceSets.size() ); 05448 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), interfaceSets, &iface_proc[0] );MB_CHK_SET_ERR( result, "Failed to get iface_proc for iface sets" ); 05449 05450 // Get sharing procs either from single-proc vector or by getting 05451 // multi-proc tag value 05452 int tmp_iface_procs[MAX_SHARING_PROCS]; 05453 std::fill( tmp_iface_procs, tmp_iface_procs + MAX_SHARING_PROCS, -1 ); 05454 Range::iterator rit; 05455 for( rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); ++rit, i++ ) 05456 { 05457 if( -1 != iface_proc[i] ) 05458 { 05459 assert( iface_proc[i] != (int)procConfig.proc_rank() ); 05460 procs_set.insert( (unsigned int)iface_proc[i] ); 05461 } 05462 else 05463 { 05464 // Get the sharing_procs tag 05465 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, tmp_iface_procs );MB_CHK_SET_ERR( result, "Failed to get iface_procs for iface set" ); 05466 for( j = 0; j < MAX_SHARING_PROCS; j++ ) 05467 { 05468 if( -1 != tmp_iface_procs[j] && tmp_iface_procs[j] != (int)procConfig.proc_rank() ) 05469 procs_set.insert( (unsigned int)tmp_iface_procs[j] ); 05470 else if( -1 == tmp_iface_procs[j] ) 05471 { 05472 std::fill( tmp_iface_procs, tmp_iface_procs + j, -1 ); 05473 break; 05474 } 05475 } 05476 } 05477 } 05478 05479 if( get_buffs ) 05480 { 05481 for( std::set< unsigned int >::iterator sit = procs_set.begin(); sit != procs_set.end(); ++sit ) 05482 get_buffers( *sit ); 05483 } 05484 05485 return MB_SUCCESS; 05486 } 05487 05488 ErrorCode ParallelComm::get_pstatus( EntityHandle entity, unsigned char& pstatus_val ) 05489 { 05490 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstatus_val );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" ); 05491 return result; 05492 } 05493 05494 ErrorCode ParallelComm::get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents ) 05495 { 05496 Range ents; 05497 ErrorCode result; 05498 05499 if( -1 == dim ) 05500 { 05501 result = mbImpl->get_entities_by_handle( 0, ents );MB_CHK_SET_ERR( result, "Failed to get all entities" ); 05502 } 05503 else 05504 { 05505 result = mbImpl->get_entities_by_dimension( 0, dim, ents );MB_CHK_SET_ERR( result, "Failed to get entities of dimension " << dim ); 05506 } 05507 05508 std::vector< unsigned char > pstatus( ents.size() ); 05509 result = mbImpl->tag_get_data( pstatus_tag(), ents, &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" ); 05510 Range::iterator rit = ents.begin(); 05511 int i = 0; 05512 if( pstatus_val ) 05513 { 05514 for( ; rit != ents.end(); i++, ++rit ) 05515 { 05516 if( pstatus[i] & pstatus_val && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) ) 05517 pstatus_ents.insert( *rit ); 05518 } 05519 } 05520 else 05521 { 05522 for( ; rit != ents.end(); i++, ++rit ) 05523 { 05524 if( !pstatus[i] && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) ) 05525 pstatus_ents.insert( *rit ); 05526 } 05527 } 05528 05529 return MB_SUCCESS; 05530 } 05531 05532 ErrorCode ParallelComm::check_global_ids( EntityHandle this_set, 05533 const int dimension, 05534 const int start_id, 05535 const bool largest_dim_only, 05536 const bool parallel, 05537 const bool owned_only ) 05538 { 05539 // Global id tag 05540 Tag gid_tag = mbImpl->globalId_tag(); 05541 int def_val = -1; 05542 Range dum_range; 05543 05544 void* tag_ptr = &def_val; 05545 ErrorCode result = mbImpl->get_entities_by_type_and_tag( this_set, MBVERTEX, &gid_tag, &tag_ptr, 1, dum_range );MB_CHK_SET_ERR( result, "Failed to get entities by MBVERTEX type and gid tag" ); 05546 05547 if( !dum_range.empty() ) 05548 { 05549 // Just created it, so we need global ids 05550 result = assign_global_ids( this_set, dimension, start_id, largest_dim_only, parallel, owned_only );MB_CHK_SET_ERR( result, "Failed assigning global ids" ); 05551 } 05552 05553 return MB_SUCCESS; 05554 } 05555 05556 bool ParallelComm::is_iface_proc( EntityHandle this_set, int to_proc ) 05557 { 05558 int sharing_procs[MAX_SHARING_PROCS]; 05559 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 ); 05560 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &this_set, 1, sharing_procs ); 05561 if( MB_SUCCESS == result && to_proc == sharing_procs[0] ) return true; 05562 05563 result = mbImpl->tag_get_data( sharedps_tag(), &this_set, 1, sharing_procs ); 05564 if( MB_SUCCESS != result ) return false; 05565 05566 for( int i = 0; i < MAX_SHARING_PROCS; i++ ) 05567 { 05568 if( to_proc == sharing_procs[i] ) 05569 return true; 05570 else if( -1 == sharing_procs[i] ) 05571 return false; 05572 } 05573 05574 return false; 05575 } 05576 05577 ErrorCode ParallelComm::filter_pstatus( Range& ents, 05578 unsigned char pstat, 05579 unsigned char op, 05580 int to_proc, 05581 Range* returned_ents ) 05582 { 05583 Range tmp_ents; 05584 05585 // assert(!ents.empty()); 05586 if( ents.empty() ) 05587 { 05588 if( returned_ents ) returned_ents->clear(); 05589 return MB_SUCCESS; 05590 } 05591 05592 // Put into tmp_ents any entities which are not owned locally or 05593 // who are already shared with to_proc 05594 std::vector< unsigned char > shared_flags( ents.size() ), shared_flags2; 05595 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), ents, &shared_flags[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus flag" ); 05596 Range::const_iterator rit, hint = tmp_ents.begin(); 05597 ; 05598 int i; 05599 if( op == PSTATUS_OR ) 05600 { 05601 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05602 { 05603 if( ( ( shared_flags[i] & ~pstat ) ^ shared_flags[i] ) & pstat ) 05604 { 05605 hint = tmp_ents.insert( hint, *rit ); 05606 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05607 } 05608 } 05609 } 05610 else if( op == PSTATUS_AND ) 05611 { 05612 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05613 { 05614 if( ( shared_flags[i] & pstat ) == pstat ) 05615 { 05616 hint = tmp_ents.insert( hint, *rit ); 05617 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05618 } 05619 } 05620 } 05621 else if( op == PSTATUS_NOT ) 05622 { 05623 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ ) 05624 { 05625 if( !( shared_flags[i] & pstat ) ) 05626 { 05627 hint = tmp_ents.insert( hint, *rit ); 05628 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] ); 05629 } 05630 } 05631 } 05632 else 05633 { 05634 assert( false ); 05635 return MB_FAILURE; 05636 } 05637 05638 if( -1 != to_proc ) 05639 { 05640 int sharing_procs[MAX_SHARING_PROCS]; 05641 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 ); 05642 Range tmp_ents2; 05643 hint = tmp_ents2.begin(); 05644 05645 for( rit = tmp_ents.begin(), i = 0; rit != tmp_ents.end(); ++rit, i++ ) 05646 { 05647 // We need to check sharing procs 05648 if( shared_flags2[i] & PSTATUS_MULTISHARED ) 05649 { 05650 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag" ); 05651 assert( -1 != sharing_procs[0] ); 05652 for( unsigned int j = 0; j < MAX_SHARING_PROCS; j++ ) 05653 { 05654 // If to_proc shares this entity, add it to list 05655 if( sharing_procs[j] == to_proc ) 05656 { 05657 hint = tmp_ents2.insert( hint, *rit ); 05658 } 05659 else if( -1 == sharing_procs[j] ) 05660 break; 05661 05662 sharing_procs[j] = -1; 05663 } 05664 } 05665 else if( shared_flags2[i] & PSTATUS_SHARED ) 05666 { 05667 result = mbImpl->tag_get_data( sharedp_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag" ); 05668 assert( -1 != sharing_procs[0] ); 05669 if( sharing_procs[0] == to_proc ) hint = tmp_ents2.insert( hint, *rit ); 05670 sharing_procs[0] = -1; 05671 } 05672 else 05673 assert( "should never get here" && false ); 05674 } 05675 05676 tmp_ents.swap( tmp_ents2 ); 05677 } 05678 05679 if( returned_ents ) 05680 returned_ents->swap( tmp_ents ); 05681 else 05682 ents.swap( tmp_ents ); 05683 05684 return MB_SUCCESS; 05685 } 05686 05687 ErrorCode ParallelComm::exchange_ghost_cells( int ghost_dim, 05688 int bridge_dim, 05689 int num_layers, 05690 int addl_ents, 05691 bool store_remote_handles, 05692 bool wait_all, 05693 EntityHandle* file_set ) 05694 { 05695 #ifdef MOAB_HAVE_MPE 05696 if( myDebug->get_verbosity() == 2 ) 05697 { 05698 if( !num_layers ) 05699 MPE_Log_event( IFACE_START, procConfig.proc_rank(), "Starting interface exchange." ); 05700 else 05701 MPE_Log_event( GHOST_START, procConfig.proc_rank(), "Starting ghost exchange." ); 05702 } 05703 #endif 05704 05705 myDebug->tprintf( 1, "Entering exchange_ghost_cells with num_layers = %d\n", num_layers ); 05706 if( myDebug->get_verbosity() == 4 ) 05707 { 05708 msgs.clear(); 05709 msgs.reserve( MAX_SHARING_PROCS ); 05710 } 05711 05712 // If we're only finding out about existing ents, we have to be storing 05713 // remote handles too 05714 assert( num_layers > 0 || store_remote_handles ); 05715 05716 const bool is_iface = !num_layers; 05717 05718 // Get the b-dimensional interface(s) with with_proc, where b = bridge_dim 05719 05720 int success; 05721 ErrorCode result = MB_SUCCESS; 05722 int incoming1 = 0, incoming2 = 0; 05723 05724 reset_all_buffers(); 05725 05726 // When this function is called, buffProcs should already have any 05727 // communicating procs 05728 05729 //=========================================== 05730 // Post ghost irecv's for ghost entities from all communicating procs 05731 //=========================================== 05732 #ifdef MOAB_HAVE_MPE 05733 if( myDebug->get_verbosity() == 2 ) 05734 { 05735 MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." ); 05736 } 05737 #endif 05738 05739 // Index reqs the same as buffer/sharing procs indices 05740 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ), 05741 recv_remoteh_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05742 std::vector< unsigned int >::iterator proc_it; 05743 int ind, p; 05744 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05745 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 05746 { 05747 incoming1++; 05748 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 05749 MB_MESG_ENTS_SIZE, incoming1 ); 05750 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 05751 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] ); 05752 if( success != MPI_SUCCESS ) 05753 { 05754 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 05755 } 05756 } 05757 05758 //=========================================== 05759 // Get entities to be sent to neighbors 05760 //=========================================== 05761 Range sent_ents[MAX_SHARING_PROCS], allsent, tmp_range; 05762 TupleList entprocs; 05763 int dum_ack_buff; 05764 result = get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents, allsent, entprocs );MB_CHK_SET_ERR( result, "get_sent_ents failed" ); 05765 05766 // augment file set with the entities to be sent 05767 // we might have created new entities if addl_ents>0, edges and/or faces 05768 if( addl_ents > 0 && file_set && !allsent.empty() ) 05769 { 05770 result = mbImpl->add_entities( *file_set, allsent );MB_CHK_SET_ERR( result, "Failed to add new sub-entities to set" ); 05771 } 05772 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(), 05773 (unsigned long)allsent.size() ); 05774 05775 //=========================================== 05776 // Pack and send ents from this proc to others 05777 //=========================================== 05778 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ ) 05779 { 05780 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", sent_ents[p].compactness(), 05781 (unsigned long)sent_ents[p].size() ); 05782 05783 // Reserve space on front for size and for initial buff size 05784 localOwnedBuffs[p]->reset_buffer( sizeof( int ) ); 05785 05786 // Entities 05787 result = pack_entities( sent_ents[p], localOwnedBuffs[p], store_remote_handles, buffProcs[p], is_iface, 05788 &entprocs, &allsent );MB_CHK_SET_ERR( result, "Packing entities failed" ); 05789 05790 if( myDebug->get_verbosity() == 4 ) 05791 { 05792 msgs.resize( msgs.size() + 1 ); 05793 msgs.back() = new Buffer( *localOwnedBuffs[p] ); 05794 } 05795 05796 // Send the buffer (size stored in front in send_buffer) 05797 result = send_buffer( *proc_it, localOwnedBuffs[p], MB_MESG_ENTS_SIZE, sendReqs[3 * p], 05798 recv_ent_reqs[3 * p + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE, 05799 ( !is_iface && store_remote_handles ? // this used for ghosting only 05800 localOwnedBuffs[p] 05801 : NULL ), 05802 &recv_remoteh_reqs[3 * p], &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" ); 05803 } 05804 05805 entprocs.reset(); 05806 05807 //=========================================== 05808 // Receive/unpack new entities 05809 //=========================================== 05810 // Number of incoming messages for ghosts is the number of procs we 05811 // communicate with; for iface, it's the number of those with lower rank 05812 MPI_Status status; 05813 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() ); 05814 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() ); 05815 std::vector< std::vector< int > > L1p( buffProcs.size() ); 05816 std::vector< EntityHandle > L2hloc, L2hrem; 05817 std::vector< unsigned int > L2p; 05818 std::vector< EntityHandle > new_ents; 05819 05820 while( incoming1 ) 05821 { 05822 // Wait for all recvs of ghost ents before proceeding to sending remote handles, 05823 // b/c some procs may have sent to a 3rd proc ents owned by me; 05824 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 05825 05826 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status ); 05827 if( MPI_SUCCESS != success ) 05828 { 05829 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 05830 } 05831 05832 PRINT_DEBUG_RECD( status ); 05833 05834 // OK, received something; decrement incoming counter 05835 incoming1--; 05836 bool done = false; 05837 05838 // In case ind is for ack, we need index of one before it 05839 unsigned int base_ind = 3 * ( ind / 3 ); 05840 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1], 05841 recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1], 05842 sendReqs[base_ind + 2], done, 05843 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ), 05844 MB_MESG_REMOTEH_SIZE, // maybe base_ind+1? 05845 &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 05846 05847 if( done ) 05848 { 05849 if( myDebug->get_verbosity() == 4 ) 05850 { 05851 msgs.resize( msgs.size() + 1 ); 05852 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] ); 05853 } 05854 05855 // Message completely received - process buffer that was sent 05856 remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 05857 result = unpack_entities( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, ind / 3, is_iface, 05858 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents ); 05859 if( MB_SUCCESS != result ) 05860 { 05861 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl; 05862 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false ); 05863 return result; 05864 } 05865 05866 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 05867 { 05868 // Post irecv's for remote handles from new proc; shouldn't be iface, 05869 // since we know about all procs we share with 05870 assert( !is_iface ); 05871 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05872 for( unsigned int i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 ) 05873 { 05874 localOwnedBuffs[i / 3]->reset_buffer(); 05875 incoming2++; 05876 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr, 05877 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 ); 05878 success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, 05879 buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(), 05880 &recv_remoteh_reqs[i] ); 05881 if( success != MPI_SUCCESS ) 05882 { 05883 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" ); 05884 } 05885 } 05886 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05887 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 05888 } 05889 } 05890 } 05891 05892 // Add requests for any new addl procs 05893 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 05894 { 05895 // Shouldn't get here... 05896 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in ghost exchange" ); 05897 } 05898 05899 #ifdef MOAB_HAVE_MPE 05900 if( myDebug->get_verbosity() == 2 ) 05901 { 05902 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." ); 05903 } 05904 #endif 05905 05906 if( is_iface ) 05907 { 05908 // Need to check over entities I sent and make sure I received 05909 // handles for them from all expected procs; if not, need to clean 05910 // them up 05911 result = check_clean_iface( allsent ); 05912 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05913 05914 // Now set the shared/interface tag on non-vertex entities on interface 05915 result = tag_iface_entities();MB_CHK_SET_ERR( result, "Failed to tag iface entities" ); 05916 05917 #ifndef NDEBUG 05918 result = check_sent_ents( allsent ); 05919 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05920 result = check_all_shared_handles( true ); 05921 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl; 05922 #endif 05923 05924 #ifdef MOAB_HAVE_MPE 05925 if( myDebug->get_verbosity() == 2 ) 05926 { 05927 MPE_Log_event( IFACE_END, procConfig.proc_rank(), "Ending interface exchange." ); 05928 } 05929 #endif 05930 05931 //=========================================== 05932 // Wait if requested 05933 //=========================================== 05934 if( wait_all ) 05935 { 05936 if( myDebug->get_verbosity() == 5 ) 05937 { 05938 success = MPI_Barrier( procConfig.proc_comm() ); 05939 } 05940 else 05941 { 05942 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 05943 success = MPI_Waitall( 3 * buffProcs.size(), &recv_ent_reqs[0], mult_status ); 05944 if( MPI_SUCCESS != success ) 05945 { 05946 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05947 } 05948 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 05949 if( MPI_SUCCESS != success ) 05950 { 05951 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05952 } 05953 /*success = MPI_Waitall(3*buffProcs.size(), &recv_remoteh_reqs[0], mult_status); 05954 if (MPI_SUCCESS != success) { 05955 MB_SET_ERR(MB_FAILURE, "Failed in waitall in ghost exchange"); 05956 }*/ 05957 } 05958 } 05959 05960 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 05961 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==true \n" ); 05962 05963 return MB_SUCCESS; 05964 } 05965 05966 // we still need to wait on sendReqs, if they are not fulfilled yet 05967 if( wait_all ) 05968 { 05969 if( myDebug->get_verbosity() == 5 ) 05970 { 05971 success = MPI_Barrier( procConfig.proc_comm() ); 05972 } 05973 else 05974 { 05975 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 05976 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 05977 if( MPI_SUCCESS != success ) 05978 { 05979 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 05980 } 05981 } 05982 } 05983 //=========================================== 05984 // Send local handles for new ghosts to owner, then add 05985 // those to ghost list for that owner 05986 //=========================================== 05987 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ ) 05988 { 05989 05990 // Reserve space on front for size and for initial buff size 05991 remoteOwnedBuffs[p]->reset_buffer( sizeof( int ) ); 05992 05993 result = pack_remote_handles( L1hloc[p], L1hrem[p], L1p[p], *proc_it, remoteOwnedBuffs[p] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 05994 remoteOwnedBuffs[p]->set_stored_size(); 05995 05996 if( myDebug->get_verbosity() == 4 ) 05997 { 05998 msgs.resize( msgs.size() + 1 ); 05999 msgs.back() = new Buffer( *remoteOwnedBuffs[p] ); 06000 } 06001 result = send_buffer( buffProcs[p], remoteOwnedBuffs[p], MB_MESG_REMOTEH_SIZE, sendReqs[3 * p], 06002 recv_remoteh_reqs[3 * p + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 06003 } 06004 06005 //=========================================== 06006 // Process remote handles of my ghosteds 06007 //=========================================== 06008 while( incoming2 ) 06009 { 06010 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 06011 success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status ); 06012 if( MPI_SUCCESS != success ) 06013 { 06014 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 06015 } 06016 06017 // OK, received something; decrement incoming counter 06018 incoming2--; 06019 06020 PRINT_DEBUG_RECD( status ); 06021 06022 bool done = false; 06023 unsigned int base_ind = 3 * ( ind / 3 ); 06024 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1], 06025 recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3], 06026 sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 06027 if( done ) 06028 { 06029 // Incoming remote handles 06030 if( myDebug->get_verbosity() == 4 ) 06031 { 06032 msgs.resize( msgs.size() + 1 ); 06033 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] ); 06034 } 06035 localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 06036 result = 06037 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 06038 } 06039 } 06040 06041 #ifdef MOAB_HAVE_MPE 06042 if( myDebug->get_verbosity() == 2 ) 06043 { 06044 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." ); 06045 MPE_Log_event( GHOST_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." ); 06046 } 06047 #endif 06048 06049 //=========================================== 06050 // Wait if requested 06051 //=========================================== 06052 if( wait_all ) 06053 { 06054 if( myDebug->get_verbosity() == 5 ) 06055 { 06056 success = MPI_Barrier( procConfig.proc_comm() ); 06057 } 06058 else 06059 { 06060 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 06061 success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status ); 06062 if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 06063 } 06064 if( MPI_SUCCESS != success ) 06065 { 06066 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" ); 06067 } 06068 } 06069 06070 #ifndef NDEBUG 06071 result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" ); 06072 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Failed check on all shared handles" ); 06073 #endif 06074 06075 if( file_set && !new_ents.empty() ) 06076 { 06077 result = mbImpl->add_entities( *file_set, &new_ents[0], new_ents.size() );MB_CHK_SET_ERR( result, "Failed to add new entities to set" ); 06078 } 06079 06080 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() ); 06081 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==false \n" ); 06082 06083 return MB_SUCCESS; 06084 } 06085 06086 ErrorCode ParallelComm::send_buffer( const unsigned int to_proc, 06087 Buffer* send_buff, 06088 int mesg_tag, 06089 MPI_Request& send_req, 06090 MPI_Request& ack_req, 06091 int* ack_buff, 06092 int& this_incoming, 06093 int next_mesg_tag, 06094 Buffer* next_recv_buff, 06095 MPI_Request* next_recv_req, 06096 int* next_incoming ) 06097 { 06098 ErrorCode result = MB_SUCCESS; 06099 int success; 06100 06101 // If small message, post recv for remote handle message 06102 if( send_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE && next_recv_buff ) 06103 { 06104 ( *next_incoming )++; 06105 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, next_mesg_tag, 06106 *next_incoming ); 06107 success = MPI_Irecv( next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, to_proc, next_mesg_tag, 06108 procConfig.proc_comm(), next_recv_req ); 06109 if( success != MPI_SUCCESS ) 06110 { 06111 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for next message in ghost exchange" ); 06112 } 06113 } 06114 // If large, we'll need an ack before sending the rest 06115 else if( send_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE ) 06116 { 06117 this_incoming++; 06118 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, (unsigned char*)ack_buff, sizeof( int ), mesg_tag - 1, 06119 this_incoming ); 06120 success = MPI_Irecv( (void*)ack_buff, sizeof( int ), MPI_UNSIGNED_CHAR, to_proc, mesg_tag - 1, 06121 procConfig.proc_comm(), &ack_req ); 06122 if( success != MPI_SUCCESS ) 06123 { 06124 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for entity ack in ghost exchange" ); 06125 } 06126 } 06127 06128 // Send the buffer 06129 PRINT_DEBUG_ISEND( procConfig.proc_rank(), to_proc, send_buff->mem_ptr, mesg_tag, 06130 std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ) ); 06131 assert( 0 <= send_buff->get_stored_size() && send_buff->get_stored_size() <= (int)send_buff->alloc_size ); 06132 success = MPI_Isend( send_buff->mem_ptr, std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ), 06133 MPI_UNSIGNED_CHAR, to_proc, mesg_tag, procConfig.proc_comm(), &send_req ); 06134 if( success != MPI_SUCCESS ) return MB_FAILURE; 06135 06136 return result; 06137 } 06138 06139 ErrorCode ParallelComm::recv_buffer( int mesg_tag_expected, 06140 const MPI_Status& mpi_status, 06141 Buffer* recv_buff, 06142 MPI_Request& recv_req, 06143 MPI_Request& /*ack_recvd_req*/, 06144 int& this_incoming, 06145 Buffer* send_buff, 06146 MPI_Request& send_req, 06147 MPI_Request& sent_ack_req, 06148 bool& done, 06149 Buffer* next_buff, 06150 int next_tag, 06151 MPI_Request* next_req, 06152 int* next_incoming ) 06153 { 06154 // Process a received message; if there will be more coming, 06155 // post a receive for 2nd part then send an ack message 06156 int from_proc = mpi_status.MPI_SOURCE; 06157 int success; 06158 06159 // Set the buff_ptr on the recv_buffer; needs to point beyond any 06160 // valid data already in the buffer 06161 recv_buff->reset_ptr( std::min( recv_buff->get_stored_size(), (int)recv_buff->alloc_size ) ); 06162 06163 if( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE ) 06164 { 06165 // 1st message & large - allocate buffer, post irecv for 2nd message, 06166 // then send ack 06167 recv_buff->reserve( recv_buff->get_stored_size() ); 06168 assert( recv_buff->alloc_size > INITIAL_BUFF_SIZE ); 06169 06170 // Will expect a 2nd message 06171 this_incoming++; 06172 06173 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr + INITIAL_BUFF_SIZE, 06174 recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, mesg_tag_expected + 1, this_incoming ); 06175 success = MPI_Irecv( recv_buff->mem_ptr + INITIAL_BUFF_SIZE, recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, 06176 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &recv_req ); 06177 if( success != MPI_SUCCESS ) 06178 { 06179 MB_SET_ERR( MB_FAILURE, "Failed to post 2nd iRecv in ghost exchange" ); 06180 } 06181 06182 // Send ack, doesn't matter what data actually is 06183 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr, mesg_tag_expected - 1, 06184 sizeof( int ) ); 06185 success = MPI_Isend( recv_buff->mem_ptr, sizeof( int ), MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected - 1, 06186 procConfig.proc_comm(), &sent_ack_req ); 06187 if( success != MPI_SUCCESS ) 06188 { 06189 MB_SET_ERR( MB_FAILURE, "Failed to send ack in ghost exchange" ); 06190 } 06191 } 06192 else if( mpi_status.MPI_TAG == mesg_tag_expected - 1 ) 06193 { 06194 // Got an ack back, send the 2nd half of message 06195 06196 // Should be a large message if we got this 06197 assert( *( (size_t*)send_buff->mem_ptr ) > INITIAL_BUFF_SIZE ); 06198 06199 // Post irecv for next message, then send 2nd message 06200 if( next_buff ) 06201 { 06202 // We'll expect a return message 06203 ( *next_incoming )++; 06204 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, next_buff->mem_ptr, INITIAL_BUFF_SIZE, next_tag, 06205 *next_incoming ); 06206 06207 success = MPI_Irecv( next_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, next_tag, 06208 procConfig.proc_comm(), next_req ); 06209 if( success != MPI_SUCCESS ) 06210 { 06211 MB_SET_ERR( MB_FAILURE, "Failed to post next irecv in ghost exchange" ); 06212 } 06213 } 06214 06215 // Send 2nd message 06216 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, send_buff->mem_ptr + INITIAL_BUFF_SIZE, 06217 mesg_tag_expected + 1, send_buff->get_stored_size() - INITIAL_BUFF_SIZE ); 06218 06219 assert( send_buff->get_stored_size() - INITIAL_BUFF_SIZE < send_buff->alloc_size && 06220 0 <= send_buff->get_stored_size() ); 06221 success = MPI_Isend( send_buff->mem_ptr + INITIAL_BUFF_SIZE, send_buff->get_stored_size() - INITIAL_BUFF_SIZE, 06222 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &send_req ); 06223 if( success != MPI_SUCCESS ) 06224 { 06225 MB_SET_ERR( MB_FAILURE, "Failed to send 2nd message in ghost exchange" ); 06226 } 06227 } 06228 else if( ( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE ) || 06229 mpi_status.MPI_TAG == mesg_tag_expected + 1 ) 06230 { 06231 // Message completely received - signal that we're done 06232 done = true; 06233 } 06234 06235 return MB_SUCCESS; 06236 } 06237 06238 struct ProcList 06239 { 06240 int procs[MAX_SHARING_PROCS]; 06241 }; 06242 static bool operator<( const ProcList& a, const ProcList& b ) 06243 { 06244 for( int i = 0; i < MAX_SHARING_PROCS; i++ ) 06245 { 06246 if( a.procs[i] < b.procs[i] ) 06247 return true; 06248 else if( b.procs[i] < a.procs[i] ) 06249 return false; 06250 else if( a.procs[i] < 0 ) 06251 return false; 06252 } 06253 return false; 06254 } 06255 06256 ErrorCode ParallelComm::check_clean_iface( Range& allsent ) 06257 { 06258 // allsent is all entities I think are on interface; go over them, looking 06259 // for zero-valued handles, and fix any I find 06260 06261 // Keep lists of entities for which teh sharing data changed, grouped 06262 // by set of sharing procs. 06263 typedef std::map< ProcList, Range > procmap_t; 06264 procmap_t old_procs, new_procs; 06265 06266 ErrorCode result = MB_SUCCESS; 06267 Range::iterator rit; 06268 Range::reverse_iterator rvit; 06269 unsigned char pstatus; 06270 int nump; 06271 ProcList sharedp; 06272 EntityHandle sharedh[MAX_SHARING_PROCS]; 06273 for( rvit = allsent.rbegin(); rvit != allsent.rend(); ++rvit ) 06274 { 06275 result = get_sharing_data( *rvit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data" ); 06276 assert( "Should be shared with at least one other proc" && 06277 ( nump > 1 || sharedp.procs[0] != (int)procConfig.proc_rank() ) ); 06278 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 ); 06279 06280 // Look for first null handle in list 06281 int idx = std::find( sharedh, sharedh + nump, (EntityHandle)0 ) - sharedh; 06282 if( idx == nump ) continue; // All handles are valid 06283 06284 ProcList old_list( sharedp ); 06285 std::sort( old_list.procs, old_list.procs + nump ); 06286 old_procs[old_list].insert( *rvit ); 06287 06288 // Remove null handles and corresponding proc ranks from lists 06289 int new_nump = idx; 06290 bool removed_owner = !idx; 06291 for( ++idx; idx < nump; ++idx ) 06292 { 06293 if( sharedh[idx] ) 06294 { 06295 sharedh[new_nump] = sharedh[idx]; 06296 sharedp.procs[new_nump] = sharedp.procs[idx]; 06297 ++new_nump; 06298 } 06299 } 06300 sharedp.procs[new_nump] = -1; 06301 06302 if( removed_owner && new_nump > 1 ) 06303 { 06304 // The proc that we choose as the entity owner isn't sharing the 06305 // entity (doesn't have a copy of it). We need to pick a different 06306 // owner. Choose the proc with lowest rank. 06307 idx = std::min_element( sharedp.procs, sharedp.procs + new_nump ) - sharedp.procs; 06308 std::swap( sharedp.procs[0], sharedp.procs[idx] ); 06309 std::swap( sharedh[0], sharedh[idx] ); 06310 if( sharedp.procs[0] == (int)proc_config().proc_rank() ) pstatus &= ~PSTATUS_NOT_OWNED; 06311 } 06312 06313 result = set_sharing_data( *rvit, pstatus, nump, new_nump, sharedp.procs, sharedh );MB_CHK_SET_ERR( result, "Failed to set sharing data in check_clean_iface" ); 06314 06315 if( new_nump > 1 ) 06316 { 06317 if( new_nump == 2 ) 06318 { 06319 if( sharedp.procs[1] != (int)proc_config().proc_rank() ) 06320 { 06321 assert( sharedp.procs[0] == (int)proc_config().proc_rank() ); 06322 sharedp.procs[0] = sharedp.procs[1]; 06323 } 06324 sharedp.procs[1] = -1; 06325 } 06326 else 06327 { 06328 std::sort( sharedp.procs, sharedp.procs + new_nump ); 06329 } 06330 new_procs[sharedp].insert( *rvit ); 06331 } 06332 } 06333 06334 if( old_procs.empty() ) 06335 { 06336 assert( new_procs.empty() ); 06337 return MB_SUCCESS; 06338 } 06339 06340 // Update interface sets 06341 procmap_t::iterator pmit; 06342 // std::vector<unsigned char> pstatus_list; 06343 rit = interface_sets().begin(); 06344 while( rit != interface_sets().end() ) 06345 { 06346 result = get_sharing_data( *rit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data for interface set" ); 06347 assert( nump != 2 ); 06348 std::sort( sharedp.procs, sharedp.procs + nump ); 06349 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 ); 06350 06351 pmit = old_procs.find( sharedp ); 06352 if( pmit != old_procs.end() ) 06353 { 06354 result = mbImpl->remove_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" ); 06355 } 06356 06357 pmit = new_procs.find( sharedp ); 06358 if( pmit == new_procs.end() ) 06359 { 06360 int count; 06361 result = mbImpl->get_number_entities_by_handle( *rit, count );MB_CHK_SET_ERR( result, "Failed to get number of entities in interface set" ); 06362 if( !count ) 06363 { 06364 result = mbImpl->delete_entities( &*rit, 1 );MB_CHK_SET_ERR( result, "Failed to delete entities from interface set" ); 06365 rit = interface_sets().erase( rit ); 06366 } 06367 else 06368 { 06369 ++rit; 06370 } 06371 } 06372 else 06373 { 06374 result = mbImpl->add_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 06375 06376 // Remove those that we've processed so that we know which ones 06377 // are new. 06378 new_procs.erase( pmit ); 06379 ++rit; 06380 } 06381 } 06382 06383 // Create interface sets for new proc id combinations 06384 std::fill( sharedh, sharedh + MAX_SHARING_PROCS, 0 ); 06385 for( pmit = new_procs.begin(); pmit != new_procs.end(); ++pmit ) 06386 { 06387 EntityHandle new_set; 06388 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" ); 06389 interfaceSets.insert( new_set ); 06390 06391 // Add entities 06392 result = mbImpl->add_entities( new_set, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" ); 06393 // Tag set with the proc rank(s) 06394 assert( pmit->first.procs[0] >= 0 ); 06395 pstatus = PSTATUS_SHARED | PSTATUS_INTERFACE; 06396 if( pmit->first.procs[1] == -1 ) 06397 { 06398 int other = pmit->first.procs[0]; 06399 assert( other != (int)procConfig.proc_rank() ); 06400 result = mbImpl->tag_set_data( sharedp_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06401 sharedh[0] = 0; 06402 result = mbImpl->tag_set_data( sharedh_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06403 if( other < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED; 06404 } 06405 else 06406 { 06407 result = mbImpl->tag_set_data( sharedps_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06408 result = mbImpl->tag_set_data( sharedhs_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" ); 06409 pstatus |= PSTATUS_MULTISHARED; 06410 if( pmit->first.procs[0] < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED; 06411 } 06412 06413 result = mbImpl->tag_set_data( pstatus_tag(), &new_set, 1, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" ); 06414 06415 // Set pstatus on all interface entities in set 06416 result = mbImpl->tag_clear_data( pstatus_tag(), pmit->second, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface entities with pstatus" ); 06417 } 06418 06419 return MB_SUCCESS; 06420 } 06421 06422 ErrorCode ParallelComm::set_sharing_data( EntityHandle ent, 06423 unsigned char pstatus, 06424 int old_nump, 06425 int new_nump, 06426 int* ps, 06427 EntityHandle* hs ) 06428 { 06429 // If new nump is less than 3, the entity is no longer mutishared 06430 if( old_nump > 2 && ( pstatus & PSTATUS_MULTISHARED ) && new_nump < 3 ) 06431 { 06432 // Unset multishared flag 06433 pstatus ^= PSTATUS_MULTISHARED; 06434 } 06435 06436 // Check for consistency in input data 06437 // DBG 06438 /* bool con1 = ((new_nump == 2 && pstatus&PSTATUS_SHARED && !(pstatus&PSTATUS_MULTISHARED)) || 06439 (new_nump > 2 && pstatus&PSTATUS_SHARED && pstatus&PSTATUS_MULTISHARED)); bool con2 = 06440 (!(pstatus&PSTATUS_GHOST) || pstatus&PSTATUS_SHARED); bool con3 = (new_nump < 3 || 06441 (pstatus&PSTATUS_NOT_OWNED && ps[0] != (int)rank()) || (!(pstatus&PSTATUS_NOT_OWNED) && ps[0] 06442 == (int)rank())); std::cout<<"current rank = "<<rank()<<std::endl; std::cout<<"condition 06443 1::"<<con1<<std::endl; std::cout<<"condition 2::"<<con2<<std::endl; std::cout<<"condition 06444 3::"<<con3<<std::endl;*/ 06445 06446 // DBG 06447 06448 assert( new_nump > 1 && 06449 ( ( new_nump == 2 && pstatus & PSTATUS_SHARED && 06450 !( pstatus & PSTATUS_MULTISHARED ) ) || // If <= 2 must not be multishared 06451 ( new_nump > 2 && pstatus & PSTATUS_SHARED && 06452 pstatus & PSTATUS_MULTISHARED ) ) && // If > 2 procs, must be multishared 06453 ( !( pstatus & PSTATUS_GHOST ) || pstatus & PSTATUS_SHARED ) && // If ghost, it must also be shared 06454 ( new_nump < 3 || 06455 ( pstatus & PSTATUS_NOT_OWNED && ps[0] != (int)rank() ) || // I'm not owner and first proc not me 06456 ( !( pstatus & PSTATUS_NOT_OWNED ) && ps[0] == (int)rank() ) ) // I'm owner and first proc is me 06457 ); 06458 06459 #ifndef NDEBUG 06460 { 06461 // Check for duplicates in proc list 06462 std::set< unsigned int > dumprocs; 06463 int dp = 0; 06464 for( ; dp < old_nump && -1 != ps[dp]; dp++ ) 06465 dumprocs.insert( ps[dp] ); 06466 assert( dp == (int)dumprocs.size() ); 06467 } 06468 #endif 06469 06470 ErrorCode result; 06471 // Reset any old data that needs to be 06472 if( old_nump > 2 && new_nump < 3 ) 06473 { 06474 // Need to remove multishared tags 06475 result = mbImpl->tag_delete_data( sharedps_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:1" ); 06476 result = mbImpl->tag_delete_data( sharedhs_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:2" ); 06477 // if (new_nump < 2) 06478 // pstatus = 0x0; 06479 // else if (ps[0] != (int)proc_config().proc_rank()) 06480 // pstatus |= PSTATUS_NOT_OWNED; 06481 } 06482 else if( ( old_nump < 3 && new_nump > 2 ) || ( old_nump > 1 && new_nump == 1 ) ) 06483 { 06484 // Reset sharedp and sharedh tags 06485 int tmp_p = -1; 06486 EntityHandle tmp_h = 0; 06487 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, &tmp_p );MB_CHK_SET_ERR( result, "set_sharing_data:3" ); 06488 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, &tmp_h );MB_CHK_SET_ERR( result, "set_sharing_data:4" ); 06489 } 06490 06491 assert( "check for multishared/owner I'm first proc" && 06492 ( !( pstatus & PSTATUS_MULTISHARED ) || ( pstatus & ( PSTATUS_NOT_OWNED | PSTATUS_GHOST ) ) || 06493 ( ps[0] == (int)rank() ) ) && 06494 "interface entities should have > 1 proc" && ( !( pstatus & PSTATUS_INTERFACE ) || new_nump > 1 ) && 06495 "ghost entities should have > 1 proc" && ( !( pstatus & PSTATUS_GHOST ) || new_nump > 1 ) ); 06496 06497 // Now set new data 06498 if( new_nump > 2 ) 06499 { 06500 result = mbImpl->tag_set_data( sharedps_tag(), &ent, 1, ps );MB_CHK_SET_ERR( result, "set_sharing_data:5" ); 06501 result = mbImpl->tag_set_data( sharedhs_tag(), &ent, 1, hs );MB_CHK_SET_ERR( result, "set_sharing_data:6" ); 06502 } 06503 else 06504 { 06505 unsigned int j = ( ps[0] == (int)procConfig.proc_rank() ? 1 : 0 ); 06506 assert( -1 != ps[j] ); 06507 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, ps + j );MB_CHK_SET_ERR( result, "set_sharing_data:7" ); 06508 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, hs + j );MB_CHK_SET_ERR( result, "set_sharing_data:8" ); 06509 } 06510 06511 result = mbImpl->tag_set_data( pstatus_tag(), &ent, 1, &pstatus );MB_CHK_SET_ERR( result, "set_sharing_data:9" ); 06512 06513 if( old_nump > 1 && new_nump < 2 ) sharedEnts.erase( ent ); 06514 06515 return result; 06516 } 06517 06518 ErrorCode ParallelComm::get_sent_ents( const bool is_iface, 06519 const int bridge_dim, 06520 const int ghost_dim, 06521 const int num_layers, 06522 const int addl_ents, 06523 Range* sent_ents, 06524 Range& allsent, 06525 TupleList& entprocs ) 06526 { 06527 ErrorCode result; 06528 unsigned int ind; 06529 std::vector< unsigned int >::iterator proc_it; 06530 Range tmp_range; 06531 06532 // Done in a separate loop over procs because sometimes later procs 06533 // need to add info to earlier procs' messages 06534 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 06535 { 06536 if( !is_iface ) 06537 { 06538 result = 06539 get_ghosted_entities( bridge_dim, ghost_dim, buffProcs[ind], num_layers, addl_ents, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get ghost layers" ); 06540 } 06541 else 06542 { 06543 result = get_iface_entities( buffProcs[ind], -1, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get interface layers" ); 06544 } 06545 06546 // Filter out entities already shared with destination 06547 tmp_range.clear(); 06548 result = filter_pstatus( sent_ents[ind], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 06549 if( !tmp_range.empty() ) sent_ents[ind] = subtract( sent_ents[ind], tmp_range ); 06550 06551 allsent.merge( sent_ents[ind] ); 06552 } 06553 06554 //=========================================== 06555 // Need to get procs each entity is sent to 06556 //=========================================== 06557 06558 // Get the total # of proc/handle pairs 06559 int npairs = 0; 06560 for( ind = 0; ind < buffProcs.size(); ind++ ) 06561 npairs += sent_ents[ind].size(); 06562 06563 // Allocate a TupleList of that size 06564 entprocs.initialize( 1, 0, 1, 0, npairs ); 06565 entprocs.enableWriteAccess(); 06566 06567 // Put the proc/handle pairs in the list 06568 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ ) 06569 { 06570 for( Range::iterator rit = sent_ents[ind].begin(); rit != sent_ents[ind].end(); ++rit ) 06571 { 06572 entprocs.vi_wr[entprocs.get_n()] = *proc_it; 06573 entprocs.vul_wr[entprocs.get_n()] = *rit; 06574 entprocs.inc_n(); 06575 } 06576 } 06577 // Sort by handle 06578 moab::TupleList::buffer sort_buffer; 06579 sort_buffer.buffer_init( npairs ); 06580 entprocs.sort( 1, &sort_buffer ); 06581 06582 entprocs.disableWriteAccess(); 06583 sort_buffer.reset(); 06584 06585 return MB_SUCCESS; 06586 } 06587 06588 ErrorCode ParallelComm::exchange_ghost_cells( ParallelComm** pcs, 06589 unsigned int num_procs, 06590 int ghost_dim, 06591 int bridge_dim, 06592 int num_layers, 06593 int addl_ents, 06594 bool store_remote_handles, 06595 EntityHandle* file_sets ) 06596 { 06597 // Static version of function, exchanging info through buffers rather 06598 // than through messages 06599 06600 // If we're only finding out about existing ents, we have to be storing 06601 // remote handles too 06602 assert( num_layers > 0 || store_remote_handles ); 06603 06604 const bool is_iface = !num_layers; 06605 06606 unsigned int ind; 06607 ParallelComm* pc; 06608 ErrorCode result = MB_SUCCESS; 06609 06610 std::vector< Error* > ehs( num_procs ); 06611 for( unsigned int i = 0; i < num_procs; i++ ) 06612 { 06613 result = pcs[i]->get_moab()->query_interface( ehs[i] ); 06614 assert( MB_SUCCESS == result ); 06615 } 06616 06617 // When this function is called, buffProcs should already have any 06618 // communicating procs 06619 06620 //=========================================== 06621 // Get entities to be sent to neighbors 06622 //=========================================== 06623 06624 // Done in a separate loop over procs because sometimes later procs 06625 // need to add info to earlier procs' messages 06626 Range sent_ents[MAX_SHARING_PROCS][MAX_SHARING_PROCS], allsent[MAX_SHARING_PROCS]; 06627 06628 //=========================================== 06629 // Get entities to be sent to neighbors 06630 //=========================================== 06631 TupleList entprocs[MAX_SHARING_PROCS]; 06632 for( unsigned int p = 0; p < num_procs; p++ ) 06633 { 06634 pc = pcs[p]; 06635 result = pc->get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents[p], allsent[p], 06636 entprocs[p] );MB_CHK_SET_ERR( result, "p = " << p << ", get_sent_ents failed" ); 06637 06638 //=========================================== 06639 // Pack entities into buffers 06640 //=========================================== 06641 for( ind = 0; ind < pc->buffProcs.size(); ind++ ) 06642 { 06643 // Entities 06644 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06645 result = pc->pack_entities( sent_ents[p][ind], pc->localOwnedBuffs[ind], store_remote_handles, 06646 pc->buffProcs[ind], is_iface, &entprocs[p], &allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", packing entities failed" ); 06647 } 06648 06649 entprocs[p].reset(); 06650 } 06651 06652 //=========================================== 06653 // Receive/unpack new entities 06654 //=========================================== 06655 // Number of incoming messages for ghosts is the number of procs we 06656 // communicate with; for iface, it's the number of those with lower rank 06657 std::vector< std::vector< EntityHandle > > L1hloc[MAX_SHARING_PROCS], L1hrem[MAX_SHARING_PROCS]; 06658 std::vector< std::vector< int > > L1p[MAX_SHARING_PROCS]; 06659 std::vector< EntityHandle > L2hloc[MAX_SHARING_PROCS], L2hrem[MAX_SHARING_PROCS]; 06660 std::vector< unsigned int > L2p[MAX_SHARING_PROCS]; 06661 std::vector< EntityHandle > new_ents[MAX_SHARING_PROCS]; 06662 06663 for( unsigned int p = 0; p < num_procs; p++ ) 06664 { 06665 L1hloc[p].resize( pcs[p]->buffProcs.size() ); 06666 L1hrem[p].resize( pcs[p]->buffProcs.size() ); 06667 L1p[p].resize( pcs[p]->buffProcs.size() ); 06668 } 06669 06670 for( unsigned int p = 0; p < num_procs; p++ ) 06671 { 06672 pc = pcs[p]; 06673 06674 for( ind = 0; ind < pc->buffProcs.size(); ind++ ) 06675 { 06676 // Incoming ghost entities; unpack; returns entities received 06677 // both from sending proc and from owning proc (which may be different) 06678 06679 // Buffer could be empty, which means there isn't any message to 06680 // unpack (due to this comm proc getting added as a result of indirect 06681 // communication); just skip this unpack 06682 if( pc->localOwnedBuffs[ind]->get_stored_size() == 0 ) continue; 06683 06684 unsigned int to_p = pc->buffProcs[ind]; 06685 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06686 result = pcs[to_p]->unpack_entities( pc->localOwnedBuffs[ind]->buff_ptr, store_remote_handles, ind, 06687 is_iface, L1hloc[to_p], L1hrem[to_p], L1p[to_p], L2hloc[to_p], 06688 L2hrem[to_p], L2p[to_p], new_ents[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack entities" ); 06689 } 06690 } 06691 06692 if( is_iface ) 06693 { 06694 // Need to check over entities I sent and make sure I received 06695 // handles for them from all expected procs; if not, need to clean 06696 // them up 06697 for( unsigned int p = 0; p < num_procs; p++ ) 06698 { 06699 result = pcs[p]->check_clean_iface( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06700 } 06701 06702 #ifndef NDEBUG 06703 for( unsigned int p = 0; p < num_procs; p++ ) 06704 { 06705 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06706 } 06707 result = check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" ); 06708 #endif 06709 return MB_SUCCESS; 06710 } 06711 06712 //=========================================== 06713 // Send local handles for new ghosts to owner, then add 06714 // those to ghost list for that owner 06715 //=========================================== 06716 std::vector< unsigned int >::iterator proc_it; 06717 for( unsigned int p = 0; p < num_procs; p++ ) 06718 { 06719 pc = pcs[p]; 06720 06721 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ ) 06722 { 06723 // Skip if iface layer and higher-rank proc 06724 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06725 result = pc->pack_remote_handles( L1hloc[p][ind], L1hrem[p][ind], L1p[p][ind], *proc_it, 06726 pc->localOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to pack remote handles" ); 06727 } 06728 } 06729 06730 //=========================================== 06731 // Process remote handles of my ghosteds 06732 //=========================================== 06733 for( unsigned int p = 0; p < num_procs; p++ ) 06734 { 06735 pc = pcs[p]; 06736 06737 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ ) 06738 { 06739 // Incoming remote handles 06740 unsigned int to_p = pc->buffProcs[ind]; 06741 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 06742 result = pcs[to_p]->unpack_remote_handles( p, pc->localOwnedBuffs[ind]->buff_ptr, L2hloc[to_p], 06743 L2hrem[to_p], L2p[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack remote handles" ); 06744 } 06745 } 06746 06747 #ifndef NDEBUG 06748 for( unsigned int p = 0; p < num_procs; p++ ) 06749 { 06750 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" ); 06751 } 06752 06753 result = ParallelComm::check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" ); 06754 #endif 06755 06756 if( file_sets ) 06757 { 06758 for( unsigned int p = 0; p < num_procs; p++ ) 06759 { 06760 if( new_ents[p].empty() ) continue; 06761 result = pcs[p]->get_moab()->add_entities( file_sets[p], &new_ents[p][0], new_ents[p].size() );MB_CHK_SET_ERR( result, "p = " << p << ", failed to add new entities to set" ); 06762 } 06763 } 06764 06765 return MB_SUCCESS; 06766 } 06767 06768 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& exchange_procs ) 06769 { 06770 // Set buffers 06771 int n_proc = exchange_procs.size(); 06772 for( int i = 0; i < n_proc; i++ ) 06773 get_buffers( exchange_procs[i] ); 06774 reset_all_buffers(); 06775 06776 // Post ghost irecv's for entities from all communicating procs 06777 // Index requests the same as buffer/sharing procs indices 06778 int success; 06779 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06780 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06781 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06782 06783 int incoming = 0; 06784 for( int i = 0; i < n_proc; i++ ) 06785 { 06786 int ind = get_buffers( exchange_procs[i] ); 06787 incoming++; 06788 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 06789 MB_MESG_ENTS_SIZE, incoming ); 06790 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 06791 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] ); 06792 if( success != MPI_SUCCESS ) 06793 { 06794 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" ); 06795 } 06796 } 06797 06798 return MB_SUCCESS; 06799 } 06800 06801 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs ) 06802 { 06803 // Set buffers 06804 int num = shared_procs.size(); 06805 for( int i = 0; i < num; i++ ) 06806 get_buffers( shared_procs[i] ); 06807 reset_all_buffers(); 06808 num = remoteOwnedBuffs.size(); 06809 for( int i = 0; i < num; i++ ) 06810 remoteOwnedBuffs[i]->set_stored_size(); 06811 num = localOwnedBuffs.size(); 06812 for( int i = 0; i < num; i++ ) 06813 localOwnedBuffs[i]->set_stored_size(); 06814 06815 // Post ghost irecv's for entities from all communicating procs 06816 // Index requests the same as buffer/sharing procs indices 06817 int success; 06818 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06819 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06820 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL ); 06821 06822 int incoming = 0; 06823 std::set< unsigned int >::iterator it = recv_procs.begin(); 06824 std::set< unsigned int >::iterator eit = recv_procs.end(); 06825 for( ; it != eit; ++it ) 06826 { 06827 int ind = get_buffers( *it ); 06828 incoming++; 06829 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 06830 MB_MESG_ENTS_SIZE, incoming ); 06831 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 06832 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] ); 06833 if( success != MPI_SUCCESS ) 06834 { 06835 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" ); 06836 } 06837 } 06838 06839 return MB_SUCCESS; 06840 } 06841 06842 ErrorCode ParallelComm::exchange_owned_meshs( std::vector< unsigned int >& exchange_procs, 06843 std::vector< Range* >& exchange_ents, 06844 std::vector< MPI_Request >& recv_ent_reqs, 06845 std::vector< MPI_Request >& recv_remoteh_reqs, 06846 bool store_remote_handles, 06847 bool wait_all, 06848 bool migrate, 06849 int dim ) 06850 { 06851 // Filter out entities already shared with destination 06852 // Exchange twice for entities and sets 06853 ErrorCode result; 06854 std::vector< unsigned int > exchange_procs_sets; 06855 std::vector< Range* > exchange_sets; 06856 int n_proc = exchange_procs.size(); 06857 for( int i = 0; i < n_proc; i++ ) 06858 { 06859 Range set_range = exchange_ents[i]->subset_by_type( MBENTITYSET ); 06860 *exchange_ents[i] = subtract( *exchange_ents[i], set_range ); 06861 Range* tmp_range = new Range( set_range ); 06862 exchange_sets.push_back( tmp_range ); 06863 exchange_procs_sets.push_back( exchange_procs[i] ); 06864 } 06865 06866 if( dim == 2 ) 06867 { 06868 // Exchange entities first 06869 result = exchange_owned_mesh( exchange_procs, exchange_ents, recvReqs, recvRemotehReqs, true, 06870 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" ); 06871 06872 // Exchange sets 06873 result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recvReqs, recvRemotehReqs, false, 06874 store_remote_handles, wait_all, migrate ); 06875 } 06876 else 06877 { 06878 // Exchange entities first 06879 result = exchange_owned_mesh( exchange_procs, exchange_ents, recv_ent_reqs, recv_remoteh_reqs, false, 06880 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" ); 06881 06882 // Exchange sets 06883 result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recv_ent_reqs, recv_remoteh_reqs, false, 06884 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh sets" ); 06885 } 06886 06887 for( int i = 0; i < n_proc; i++ ) 06888 delete exchange_sets[i]; 06889 06890 // Build up the list of shared entities 06891 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs; 06892 int procs[MAX_SHARING_PROCS]; 06893 EntityHandle handles[MAX_SHARING_PROCS]; 06894 int nprocs; 06895 unsigned char pstat; 06896 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 06897 { 06898 if( mbImpl->dimension_from_handle( *vit ) > 2 ) continue; 06899 result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data in exchange_owned_meshs" ); 06900 std::sort( procs, procs + nprocs ); 06901 std::vector< int > tmp_procs( procs, procs + nprocs ); 06902 assert( tmp_procs.size() != 2 ); 06903 proc_nvecs[tmp_procs].push_back( *vit ); 06904 } 06905 06906 // Create interface sets from shared entities 06907 result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" ); 06908 06909 return MB_SUCCESS; 06910 } 06911 06912 ErrorCode ParallelComm::exchange_owned_mesh( std::vector< unsigned int >& exchange_procs, 06913 std::vector< Range* >& exchange_ents, 06914 std::vector< MPI_Request >& recv_ent_reqs, 06915 std::vector< MPI_Request >& recv_remoteh_reqs, 06916 const bool recv_posted, 06917 bool store_remote_handles, 06918 bool wait_all, 06919 bool migrate ) 06920 { 06921 #ifdef MOAB_HAVE_MPE 06922 if( myDebug->get_verbosity() == 2 ) 06923 { 06924 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting owned ents exchange." ); 06925 } 06926 #endif 06927 06928 myDebug->tprintf( 1, "Entering exchange_owned_mesh\n" ); 06929 if( myDebug->get_verbosity() == 4 ) 06930 { 06931 msgs.clear(); 06932 msgs.reserve( MAX_SHARING_PROCS ); 06933 } 06934 unsigned int i; 06935 int ind, success; 06936 ErrorCode result = MB_SUCCESS; 06937 int incoming1 = 0, incoming2 = 0; 06938 06939 // Set buffProcs with communicating procs 06940 unsigned int n_proc = exchange_procs.size(); 06941 for( i = 0; i < n_proc; i++ ) 06942 { 06943 ind = get_buffers( exchange_procs[i] ); 06944 result = add_verts( *exchange_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" ); 06945 06946 // Filter out entities already shared with destination 06947 Range tmp_range; 06948 result = filter_pstatus( *exchange_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" ); 06949 if( !tmp_range.empty() ) 06950 { 06951 *exchange_ents[i] = subtract( *exchange_ents[i], tmp_range ); 06952 } 06953 } 06954 06955 //=========================================== 06956 // Post ghost irecv's for entities from all communicating procs 06957 //=========================================== 06958 #ifdef MOAB_HAVE_MPE 06959 if( myDebug->get_verbosity() == 2 ) 06960 { 06961 MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." ); 06962 } 06963 #endif 06964 06965 // Index reqs the same as buffer/sharing procs indices 06966 if( !recv_posted ) 06967 { 06968 reset_all_buffers(); 06969 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 06970 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 06971 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 06972 06973 for( i = 0; i < n_proc; i++ ) 06974 { 06975 ind = get_buffers( exchange_procs[i] ); 06976 incoming1++; 06977 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, 06978 INITIAL_BUFF_SIZE, MB_MESG_ENTS_SIZE, incoming1 ); 06979 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind], 06980 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] ); 06981 if( success != MPI_SUCCESS ) 06982 { 06983 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" ); 06984 } 06985 } 06986 } 06987 else 06988 incoming1 += n_proc; 06989 06990 //=========================================== 06991 // Get entities to be sent to neighbors 06992 // Need to get procs each entity is sent to 06993 //=========================================== 06994 Range allsent, tmp_range; 06995 int dum_ack_buff; 06996 int npairs = 0; 06997 TupleList entprocs; 06998 for( i = 0; i < n_proc; i++ ) 06999 { 07000 int n_ents = exchange_ents[i]->size(); 07001 if( n_ents > 0 ) 07002 { 07003 npairs += n_ents; // Get the total # of proc/handle pairs 07004 allsent.merge( *exchange_ents[i] ); 07005 } 07006 } 07007 07008 // Allocate a TupleList of that size 07009 entprocs.initialize( 1, 0, 1, 0, npairs ); 07010 entprocs.enableWriteAccess(); 07011 07012 // Put the proc/handle pairs in the list 07013 for( i = 0; i < n_proc; i++ ) 07014 { 07015 for( Range::iterator rit = exchange_ents[i]->begin(); rit != exchange_ents[i]->end(); ++rit ) 07016 { 07017 entprocs.vi_wr[entprocs.get_n()] = exchange_procs[i]; 07018 entprocs.vul_wr[entprocs.get_n()] = *rit; 07019 entprocs.inc_n(); 07020 } 07021 } 07022 07023 // Sort by handle 07024 moab::TupleList::buffer sort_buffer; 07025 sort_buffer.buffer_init( npairs ); 07026 entprocs.sort( 1, &sort_buffer ); 07027 sort_buffer.reset(); 07028 07029 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(), 07030 (unsigned long)allsent.size() ); 07031 07032 //=========================================== 07033 // Pack and send ents from this proc to others 07034 //=========================================== 07035 for( i = 0; i < n_proc; i++ ) 07036 { 07037 ind = get_buffers( exchange_procs[i] ); 07038 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", exchange_ents[i]->compactness(), 07039 (unsigned long)exchange_ents[i]->size() ); 07040 // Reserve space on front for size and for initial buff size 07041 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 07042 result = pack_buffer( *exchange_ents[i], false, true, store_remote_handles, buffProcs[ind], 07043 localOwnedBuffs[ind], &entprocs, &allsent ); 07044 07045 if( myDebug->get_verbosity() == 4 ) 07046 { 07047 msgs.resize( msgs.size() + 1 ); 07048 msgs.back() = new Buffer( *localOwnedBuffs[ind] ); 07049 } 07050 07051 // Send the buffer (size stored in front in send_buffer) 07052 result = send_buffer( exchange_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind], 07053 recv_ent_reqs[3 * ind + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE, 07054 ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recv_remoteh_reqs[3 * ind], 07055 &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" ); 07056 } 07057 07058 entprocs.reset(); 07059 07060 //=========================================== 07061 // Receive/unpack new entities 07062 //=========================================== 07063 // Number of incoming messages is the number of procs we communicate with 07064 MPI_Status status; 07065 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() ); 07066 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() ); 07067 std::vector< std::vector< int > > L1p( buffProcs.size() ); 07068 std::vector< EntityHandle > L2hloc, L2hrem; 07069 std::vector< unsigned int > L2p; 07070 std::vector< EntityHandle > new_ents; 07071 07072 while( incoming1 ) 07073 { 07074 // Wait for all recvs of ents before proceeding to sending remote handles, 07075 // b/c some procs may have sent to a 3rd proc ents owned by me; 07076 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() ); 07077 07078 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status ); 07079 if( MPI_SUCCESS != success ) 07080 { 07081 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 07082 } 07083 07084 PRINT_DEBUG_RECD( status ); 07085 07086 // OK, received something; decrement incoming counter 07087 incoming1--; 07088 bool done = false; 07089 07090 // In case ind is for ack, we need index of one before it 07091 unsigned int base_ind = 3 * ( ind / 3 ); 07092 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1], 07093 recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1], 07094 sendReqs[base_ind + 2], done, ( store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ), 07095 MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" ); 07096 07097 if( done ) 07098 { 07099 if( myDebug->get_verbosity() == 4 ) 07100 { 07101 msgs.resize( msgs.size() + 1 ); 07102 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] ); 07103 } 07104 07105 // Message completely received - process buffer that was sent 07106 remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 07107 result = unpack_buffer( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, buffProcs[ind / 3], 07108 ind / 3, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true ); 07109 if( MB_SUCCESS != result ) 07110 { 07111 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl; 07112 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false ); 07113 return result; 07114 } 07115 07116 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 07117 { 07118 // Post irecv's for remote handles from new proc 07119 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07120 for( i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 ) 07121 { 07122 localOwnedBuffs[i / 3]->reset_buffer(); 07123 incoming2++; 07124 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr, 07125 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 ); 07126 success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, 07127 buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(), 07128 &recv_remoteh_reqs[i] ); 07129 if( success != MPI_SUCCESS ) 07130 { 07131 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" ); 07132 } 07133 } 07134 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07135 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07136 } 07137 } 07138 } 07139 07140 // Assign and remove newly created elements from/to receive processor 07141 result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" ); 07142 if( migrate ) 07143 { 07144 result = remove_entities_part( allsent, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to remove entities to part" ); 07145 } 07146 07147 // Add requests for any new addl procs 07148 if( recv_ent_reqs.size() != 3 * buffProcs.size() ) 07149 { 07150 // Shouldn't get here... 07151 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" ); 07152 } 07153 07154 #ifdef MOAB_HAVE_MPE 07155 if( myDebug->get_verbosity() == 2 ) 07156 { 07157 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." ); 07158 } 07159 #endif 07160 07161 // we still need to wait on sendReqs, if they are not fulfilled yet 07162 if( wait_all ) 07163 { 07164 if( myDebug->get_verbosity() == 5 ) 07165 { 07166 success = MPI_Barrier( procConfig.proc_comm() ); 07167 } 07168 else 07169 { 07170 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 07171 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 07172 if( MPI_SUCCESS != success ) 07173 { 07174 MB_SET_ERR( MB_FAILURE, "Failed in waitall in exchange owned mesh" ); 07175 } 07176 } 07177 } 07178 07179 //=========================================== 07180 // Send local handles for new entity to owner 07181 //=========================================== 07182 for( i = 0; i < n_proc; i++ ) 07183 { 07184 ind = get_buffers( exchange_procs[i] ); 07185 // Reserve space on front for size and for initial buff size 07186 remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) ); 07187 07188 result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" ); 07189 remoteOwnedBuffs[ind]->set_stored_size(); 07190 07191 if( myDebug->get_verbosity() == 4 ) 07192 { 07193 msgs.resize( msgs.size() + 1 ); 07194 msgs.back() = new Buffer( *remoteOwnedBuffs[ind] ); 07195 } 07196 result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[3 * ind], 07197 recv_remoteh_reqs[3 * ind + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" ); 07198 } 07199 07200 //=========================================== 07201 // Process remote handles of my ghosteds 07202 //=========================================== 07203 while( incoming2 ) 07204 { 07205 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() ); 07206 success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status ); 07207 if( MPI_SUCCESS != success ) 07208 { 07209 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" ); 07210 } 07211 07212 // OK, received something; decrement incoming counter 07213 incoming2--; 07214 07215 PRINT_DEBUG_RECD( status ); 07216 07217 bool done = false; 07218 unsigned int base_ind = 3 * ( ind / 3 ); 07219 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1], 07220 recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3], 07221 sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" ); 07222 07223 if( done ) 07224 { 07225 // Incoming remote handles 07226 if( myDebug->get_verbosity() == 4 ) 07227 { 07228 msgs.resize( msgs.size() + 1 ); 07229 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] ); 07230 } 07231 07232 localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) ); 07233 result = 07234 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" ); 07235 } 07236 } 07237 07238 #ifdef MOAB_HAVE_MPE 07239 if( myDebug->get_verbosity() == 2 ) 07240 { 07241 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." ); 07242 MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." ); 07243 } 07244 #endif 07245 07246 //=========================================== 07247 // Wait if requested 07248 //=========================================== 07249 if( wait_all ) 07250 { 07251 if( myDebug->get_verbosity() == 5 ) 07252 { 07253 success = MPI_Barrier( procConfig.proc_comm() ); 07254 } 07255 else 07256 { 07257 MPI_Status mult_status[3 * MAX_SHARING_PROCS]; 07258 success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status ); 07259 if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status ); 07260 } 07261 if( MPI_SUCCESS != success ) 07262 { 07263 MB_SET_ERR( MB_FAILURE, "Failed in waitall in owned entity exchange" ); 07264 } 07265 } 07266 07267 #ifndef NDEBUG 07268 result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" ); 07269 #endif 07270 myDebug->tprintf( 1, "Exiting exchange_owned_mesh\n" ); 07271 07272 return MB_SUCCESS; 07273 } 07274 07275 ErrorCode ParallelComm::get_iface_entities( int other_proc, int dim, Range& iface_ents ) 07276 { 07277 Range iface_sets; 07278 ErrorCode result = MB_SUCCESS; 07279 07280 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 07281 { 07282 if( -1 != other_proc && !is_iface_proc( *rit, other_proc ) ) continue; 07283 07284 if( -1 == dim ) 07285 { 07286 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" ); 07287 } 07288 else 07289 { 07290 result = mbImpl->get_entities_by_dimension( *rit, dim, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" ); 07291 } 07292 } 07293 07294 return MB_SUCCESS; 07295 } 07296 07297 ErrorCode ParallelComm::assign_entities_part( std::vector< EntityHandle >& entities, const int proc ) 07298 { 07299 EntityHandle part_set; 07300 ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" ); 07301 07302 if( part_set > 0 ) 07303 { 07304 result = mbImpl->add_entities( part_set, &entities[0], entities.size() );MB_CHK_SET_ERR( result, "Failed to add entities to part set" ); 07305 } 07306 07307 return MB_SUCCESS; 07308 } 07309 07310 ErrorCode ParallelComm::remove_entities_part( Range& entities, const int proc ) 07311 { 07312 EntityHandle part_set; 07313 ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" ); 07314 07315 if( part_set > 0 ) 07316 { 07317 result = mbImpl->remove_entities( part_set, entities );MB_CHK_SET_ERR( result, "Failed to remove entities from part set" ); 07318 } 07319 07320 return MB_SUCCESS; 07321 } 07322 07323 ErrorCode ParallelComm::check_sent_ents( Range& allsent ) 07324 { 07325 // Check entities to make sure there are no zero-valued remote handles 07326 // where they shouldn't be 07327 std::vector< unsigned char > pstat( allsent.size() ); 07328 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), allsent, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 07329 std::vector< EntityHandle > handles( allsent.size() ); 07330 result = mbImpl->tag_get_data( sharedh_tag(), allsent, &handles[0] );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" ); 07331 std::vector< int > procs( allsent.size() ); 07332 result = mbImpl->tag_get_data( sharedp_tag(), allsent, &procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 07333 07334 Range bad_entities; 07335 07336 Range::iterator rit; 07337 unsigned int i; 07338 EntityHandle dum_hs[MAX_SHARING_PROCS]; 07339 int dum_ps[MAX_SHARING_PROCS]; 07340 07341 for( rit = allsent.begin(), i = 0; rit != allsent.end(); ++rit, i++ ) 07342 { 07343 if( -1 != procs[i] && 0 == handles[i] ) 07344 bad_entities.insert( *rit ); 07345 else 07346 { 07347 // Might be multi-shared... 07348 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, dum_ps ); 07349 if( MB_TAG_NOT_FOUND == result ) 07350 continue; 07351 else if( MB_SUCCESS != result ) 07352 MB_SET_ERR( result, "Failed to get sharedps tag data" ); 07353 result = mbImpl->tag_get_data( sharedhs_tag(), &( *rit ), 1, dum_hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 07354 07355 // Find first non-set proc 07356 int* ns_proc = std::find( dum_ps, dum_ps + MAX_SHARING_PROCS, -1 ); 07357 int num_procs = ns_proc - dum_ps; 07358 assert( num_procs <= MAX_SHARING_PROCS ); 07359 // Now look for zero handles in active part of dum_hs 07360 EntityHandle* ns_handle = std::find( dum_hs, dum_hs + num_procs, 0 ); 07361 int num_handles = ns_handle - dum_hs; 07362 assert( num_handles <= num_procs ); 07363 if( num_handles != num_procs ) bad_entities.insert( *rit ); 07364 } 07365 } 07366 07367 return MB_SUCCESS; 07368 } 07369 07370 ErrorCode ParallelComm::pack_remote_handles( std::vector< EntityHandle >& L1hloc, 07371 std::vector< EntityHandle >& L1hrem, 07372 std::vector< int >& L1p, 07373 unsigned int /*to_proc*/, 07374 Buffer* buff ) 07375 { 07376 assert( std::find( L1hloc.begin(), L1hloc.end(), (EntityHandle)0 ) == L1hloc.end() ); 07377 07378 // 2 vectors of handles plus ints 07379 buff->check_space( ( ( L1p.size() + 1 ) * sizeof( int ) + ( L1hloc.size() + 1 ) * sizeof( EntityHandle ) + 07380 ( L1hrem.size() + 1 ) * sizeof( EntityHandle ) ) ); 07381 07382 // Should be in pairs of handles 07383 PACK_INT( buff->buff_ptr, L1hloc.size() ); 07384 PACK_INTS( buff->buff_ptr, &L1p[0], L1p.size() ); 07385 // Pack handles in reverse order, (remote, local), so on destination they 07386 // are ordered (local, remote) 07387 PACK_EH( buff->buff_ptr, &L1hrem[0], L1hrem.size() ); 07388 PACK_EH( buff->buff_ptr, &L1hloc[0], L1hloc.size() ); 07389 07390 buff->set_stored_size(); 07391 07392 return MB_SUCCESS; 07393 } 07394 07395 ErrorCode ParallelComm::unpack_remote_handles( unsigned int from_proc, 07396 unsigned char*& buff_ptr, 07397 std::vector< EntityHandle >& L2hloc, 07398 std::vector< EntityHandle >& L2hrem, 07399 std::vector< unsigned int >& L2p ) 07400 { 07401 // Incoming remote handles; use to set remote handles 07402 int num_eh; 07403 UNPACK_INT( buff_ptr, num_eh ); 07404 07405 unsigned char* buff_proc = buff_ptr; 07406 buff_ptr += num_eh * sizeof( int ); 07407 unsigned char* buff_rem = buff_ptr + num_eh * sizeof( EntityHandle ); 07408 ErrorCode result; 07409 EntityHandle hpair[2], new_h; 07410 int proc; 07411 for( int i = 0; i < num_eh; i++ ) 07412 { 07413 UNPACK_INT( buff_proc, proc ); 07414 // Handles packed (local, remote), though here local is either on this 07415 // proc or owner proc, depending on value of proc (-1 = here, otherwise owner); 07416 // this is decoded in find_existing_entity 07417 UNPACK_EH( buff_ptr, hpair, 1 ); 07418 UNPACK_EH( buff_rem, hpair + 1, 1 ); 07419 07420 if( -1 != proc ) 07421 { 07422 result = find_existing_entity( false, proc, hpair[0], 3, NULL, 0, mbImpl->type_from_handle( hpair[1] ), 07423 L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Didn't get existing entity" ); 07424 if( new_h ) 07425 hpair[0] = new_h; 07426 else 07427 hpair[0] = 0; 07428 } 07429 if( !( hpair[0] && hpair[1] ) ) return MB_FAILURE; 07430 int this_proc = from_proc; 07431 result = update_remote_data( hpair[0], &this_proc, hpair + 1, 1, 0 );MB_CHK_SET_ERR( result, "Failed to set remote data range on sent entities in ghost exchange" ); 07432 } 07433 07434 return MB_SUCCESS; 07435 } 07436 07437 ErrorCode ParallelComm::get_ghosted_entities( int bridge_dim, 07438 int ghost_dim, 07439 int to_proc, 07440 int num_layers, 07441 int addl_ents, 07442 Range& ghosted_ents ) 07443 { 07444 // Get bridge ents on interface(s) 07445 Range from_ents; 07446 ErrorCode result = MB_SUCCESS; 07447 assert( 0 < num_layers ); 07448 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit ) 07449 { 07450 if( !is_iface_proc( *rit, to_proc ) ) continue; 07451 07452 // Get starting "from" entities 07453 if( bridge_dim == -1 ) 07454 { 07455 result = mbImpl->get_entities_by_handle( *rit, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" ); 07456 } 07457 else 07458 { 07459 result = mbImpl->get_entities_by_dimension( *rit, bridge_dim, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" ); 07460 } 07461 07462 // Need to get layers of bridge-adj entities 07463 if( from_ents.empty() ) continue; 07464 result = 07465 MeshTopoUtil( mbImpl ).get_bridge_adjacencies( from_ents, bridge_dim, ghost_dim, ghosted_ents, num_layers );MB_CHK_SET_ERR( result, "Failed to get bridge adjacencies" ); 07466 } 07467 07468 result = add_verts( ghosted_ents );MB_CHK_SET_ERR( result, "Failed to add verts" ); 07469 07470 if( addl_ents ) 07471 { 07472 // First get the ents of ghost_dim 07473 Range tmp_ents, tmp_owned, tmp_notowned; 07474 tmp_owned = ghosted_ents.subset_by_dimension( ghost_dim ); 07475 if( tmp_owned.empty() ) return result; 07476 07477 tmp_notowned = tmp_owned; 07478 07479 // Next, filter by pstatus; can only create adj entities for entities I own 07480 result = filter_pstatus( tmp_owned, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &tmp_owned );MB_CHK_SET_ERR( result, "Failed to filter owned entities" ); 07481 07482 tmp_notowned -= tmp_owned; 07483 07484 // Get edges first 07485 if( 1 == addl_ents || 3 == addl_ents ) 07486 { 07487 result = mbImpl->get_adjacencies( tmp_owned, 1, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for owned ghost entities" ); 07488 result = mbImpl->get_adjacencies( tmp_notowned, 1, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for notowned ghost entities" ); 07489 } 07490 if( 2 == addl_ents || 3 == addl_ents ) 07491 { 07492 result = mbImpl->get_adjacencies( tmp_owned, 2, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for owned ghost entities" ); 07493 result = mbImpl->get_adjacencies( tmp_notowned, 2, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for notowned ghost entities" ); 07494 } 07495 07496 ghosted_ents.merge( tmp_ents ); 07497 } 07498 07499 return result; 07500 } 07501 07502 ErrorCode ParallelComm::add_verts( Range& sent_ents ) 07503 { 07504 // Get the verts adj to these entities, since we'll have to send those too 07505 07506 // First check sets 07507 std::pair< Range::const_iterator, Range::const_iterator > set_range = sent_ents.equal_range( MBENTITYSET ); 07508 ErrorCode result = MB_SUCCESS, tmp_result; 07509 for( Range::const_iterator rit = set_range.first; rit != set_range.second; ++rit ) 07510 { 07511 tmp_result = mbImpl->get_entities_by_type( *rit, MBVERTEX, sent_ents );MB_CHK_SET_ERR( tmp_result, "Failed to get contained verts" ); 07512 } 07513 07514 // Now non-sets 07515 Range tmp_ents; 07516 std::copy( sent_ents.begin(), set_range.first, range_inserter( tmp_ents ) ); 07517 result = mbImpl->get_adjacencies( tmp_ents, 0, false, sent_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get vertices adj to ghosted ents" ); 07518 07519 // if polyhedra, need to add all faces from there 07520 Range polyhedra = sent_ents.subset_by_type( MBPOLYHEDRON ); 07521 // get all faces adjacent to every polyhedra 07522 result = mbImpl->get_connectivity( polyhedra, sent_ents );MB_CHK_SET_ERR( result, "Failed to get polyhedra faces" ); 07523 return result; 07524 } 07525 07526 ErrorCode ParallelComm::exchange_tags( const std::vector< Tag >& src_tags, 07527 const std::vector< Tag >& dst_tags, 07528 const Range& entities_in ) 07529 { 07530 ErrorCode result; 07531 int success; 07532 07533 myDebug->tprintf( 1, "Entering exchange_tags\n" ); 07534 07535 // Get all procs interfacing to this proc 07536 std::set< unsigned int > exch_procs; 07537 result = get_comm_procs( exch_procs ); 07538 07539 // Post ghost irecv's for all interface procs 07540 // Index requests the same as buffer/sharing procs indices 07541 std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07542 // sent_ack_reqs(buffProcs.size(), MPI_REQUEST_NULL); 07543 std::vector< unsigned int >::iterator sit; 07544 int ind; 07545 07546 reset_all_buffers(); 07547 int incoming = 0; 07548 07549 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 07550 { 07551 incoming++; 07552 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 07553 MB_MESG_TAGS_SIZE, incoming ); 07554 07555 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit, 07556 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] ); 07557 if( success != MPI_SUCCESS ) 07558 { 07559 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 07560 } 07561 } 07562 07563 // Pack and send tags from this proc to others 07564 // Make sendReqs vector to simplify initialization 07565 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07566 07567 // Take all shared entities if incoming list is empty 07568 Range entities; 07569 if( entities_in.empty() ) 07570 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) ); 07571 else 07572 entities = entities_in; 07573 07574 int dum_ack_buff; 07575 07576 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 07577 { 07578 Range tag_ents = entities; 07579 07580 // Get ents shared by proc *sit 07581 result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" ); 07582 07583 // Remote nonowned entities 07584 if( !tag_ents.empty() ) 07585 { 07586 result = filter_pstatus( tag_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed pstatus NOT check" ); 07587 } 07588 07589 // Pack-send; this also posts receives if store_remote_handles is true 07590 std::vector< Range > tag_ranges; 07591 for( std::vector< Tag >::const_iterator vit = src_tags.begin(); vit != src_tags.end(); ++vit ) 07592 { 07593 const void* ptr; 07594 int sz; 07595 if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS ) 07596 { 07597 Range tagged_ents; 07598 mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents ); 07599 tag_ranges.push_back( intersect( tag_ents, tagged_ents ) ); 07600 } 07601 else 07602 { 07603 tag_ranges.push_back( tag_ents ); 07604 } 07605 } 07606 07607 // Pack the data 07608 // Reserve space on front for size and for initial buff size 07609 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 07610 07611 result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" ); 07612 07613 // Now send it 07614 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind], 07615 recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" ); 07616 } 07617 07618 // Receive/unpack tags 07619 while( incoming ) 07620 { 07621 MPI_Status status; 07622 int index_in_recv_requests; 07623 PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() ); 07624 success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status ); 07625 if( MPI_SUCCESS != success ) 07626 { 07627 MB_SET_ERR( MB_FAILURE, "Failed in waitany in tag exchange" ); 07628 } 07629 // Processor index in the list is divided by 3 07630 ind = index_in_recv_requests / 3; 07631 07632 PRINT_DEBUG_RECD( status ); 07633 07634 // OK, received something; decrement incoming counter 07635 incoming--; 07636 07637 bool done = false; 07638 std::vector< EntityHandle > dum_vec; 07639 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind], 07640 recv_tag_reqs[3 * ind + 1], // This is for receiving the second message 07641 recv_tag_reqs[3 * ind + 2], // This would be for ack, but it is not 07642 // used; consider removing it 07643 incoming, localOwnedBuffs[ind], 07644 sendReqs[3 * ind + 1], // Send request for sending the second message 07645 sendReqs[3 * ind + 2], // This is for sending the ack 07646 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" ); 07647 if( done ) 07648 { 07649 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 07650 result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind] );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" ); 07651 } 07652 } 07653 07654 // OK, now wait 07655 if( myDebug->get_verbosity() == 5 ) 07656 { 07657 success = MPI_Barrier( procConfig.proc_comm() ); 07658 } 07659 else 07660 { 07661 MPI_Status status[3 * MAX_SHARING_PROCS]; 07662 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status ); 07663 } 07664 if( MPI_SUCCESS != success ) 07665 { 07666 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" ); 07667 } 07668 07669 // If source tag is not equal to destination tag, then 07670 // do local copy for owned entities (communicate w/ self) 07671 assert( src_tags.size() == dst_tags.size() ); 07672 if( src_tags != dst_tags ) 07673 { 07674 std::vector< unsigned char > data; 07675 Range owned_ents; 07676 if( entities_in.empty() ) 07677 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) ); 07678 else 07679 owned_ents = entities_in; 07680 result = filter_pstatus( owned_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failure to get subset of owned entities" ); 07681 07682 if( !owned_ents.empty() ) 07683 { // Check this here, otherwise we get 07684 // Unexpected results from get_entities_by_type_and_tag w/ Interface::INTERSECT 07685 for( size_t i = 0; i < src_tags.size(); i++ ) 07686 { 07687 if( src_tags[i] == dst_tags[i] ) continue; 07688 07689 Range tagged_ents( owned_ents ); 07690 result = mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &src_tags[0], 0, 1, tagged_ents, 07691 Interface::INTERSECT );MB_CHK_SET_ERR( result, "get_entities_by_type_and_tag(type == MBMAXTYPE) failed" ); 07692 07693 int sz, size2; 07694 result = mbImpl->tag_get_bytes( src_tags[i], sz );MB_CHK_SET_ERR( result, "tag_get_size failed" ); 07695 result = mbImpl->tag_get_bytes( dst_tags[i], size2 );MB_CHK_SET_ERR( result, "tag_get_size failed" ); 07696 if( sz != size2 ) 07697 { 07698 MB_SET_ERR( MB_FAILURE, "tag sizes don't match" ); 07699 } 07700 07701 data.resize( sz * tagged_ents.size() ); 07702 result = mbImpl->tag_get_data( src_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_get_data failed" ); 07703 result = mbImpl->tag_set_data( dst_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_set_data failed" ); 07704 } 07705 } 07706 } 07707 07708 myDebug->tprintf( 1, "Exiting exchange_tags" ); 07709 07710 return MB_SUCCESS; 07711 } 07712 07713 ErrorCode ParallelComm::reduce_tags( const std::vector< Tag >& src_tags, 07714 const std::vector< Tag >& dst_tags, 07715 const MPI_Op mpi_op, 07716 const Range& entities_in ) 07717 { 07718 ErrorCode result; 07719 int success; 07720 07721 myDebug->tprintf( 1, "Entering reduce_tags\n" ); 07722 07723 // Check that restrictions are met: number of source/dst tags... 07724 if( src_tags.size() != dst_tags.size() ) 07725 { 07726 MB_SET_ERR( MB_FAILURE, "Source and destination tag handles must be specified for reduce_tags" ); 07727 } 07728 07729 // ... tag data types 07730 std::vector< Tag >::const_iterator vits, vitd; 07731 int tags_size, tagd_size; 07732 DataType tags_type, tagd_type; 07733 std::vector< unsigned char > vals; 07734 std::vector< int > tags_sizes; 07735 for( vits = src_tags.begin(), vitd = dst_tags.begin(); vits != src_tags.end(); ++vits, ++vitd ) 07736 { 07737 // Checks on tag characteristics 07738 result = mbImpl->tag_get_data_type( *vits, tags_type );MB_CHK_SET_ERR( result, "Failed to get src tag data type" ); 07739 if( tags_type != MB_TYPE_INTEGER && tags_type != MB_TYPE_DOUBLE && tags_type != MB_TYPE_BIT ) 07740 { 07741 MB_SET_ERR( MB_FAILURE, "Src/dst tags must have integer, double, or bit data type" ); 07742 } 07743 07744 result = mbImpl->tag_get_bytes( *vits, tags_size );MB_CHK_SET_ERR( result, "Failed to get src tag bytes" ); 07745 vals.resize( tags_size ); 07746 result = mbImpl->tag_get_default_value( *vits, &vals[0] );MB_CHK_SET_ERR( result, "Src tag must have default value" ); 07747 07748 tags_sizes.push_back( tags_size ); 07749 07750 // OK, those passed; now check whether dest tags, if specified, agree with src tags 07751 if( *vits == *vitd ) continue; 07752 07753 result = mbImpl->tag_get_bytes( *vitd, tagd_size );MB_CHK_SET_ERR( result, "Coudln't get dst tag bytes" ); 07754 if( tags_size != tagd_size ) 07755 { 07756 MB_SET_ERR( MB_FAILURE, "Sizes between src and dst tags don't match" ); 07757 } 07758 result = mbImpl->tag_get_data_type( *vitd, tagd_type );MB_CHK_SET_ERR( result, "Coudln't get dst tag data type" ); 07759 if( tags_type != tagd_type ) 07760 { 07761 MB_SET_ERR( MB_FAILURE, "Src and dst tags must be of same data type" ); 07762 } 07763 } 07764 07765 // Get all procs interfacing to this proc 07766 std::set< unsigned int > exch_procs; 07767 result = get_comm_procs( exch_procs ); 07768 07769 // Post ghost irecv's for all interface procs 07770 // Index requests the same as buffer/sharing procs indices 07771 std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07772 07773 std::vector< unsigned int >::iterator sit; 07774 int ind; 07775 07776 reset_all_buffers(); 07777 int incoming = 0; 07778 07779 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 07780 { 07781 incoming++; 07782 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 07783 MB_MESG_TAGS_SIZE, incoming ); 07784 07785 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit, 07786 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] ); 07787 if( success != MPI_SUCCESS ) 07788 { 07789 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" ); 07790 } 07791 } 07792 07793 // Pack and send tags from this proc to others 07794 // Make sendReqs vector to simplify initialization 07795 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 07796 07797 // Take all shared entities if incoming list is empty 07798 Range entities; 07799 if( entities_in.empty() ) 07800 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) ); 07801 else 07802 entities = entities_in; 07803 07804 // If the tags are different, copy the source to the dest tag locally 07805 std::vector< Tag >::const_iterator vit = src_tags.begin(), vit2 = dst_tags.begin(); 07806 std::vector< int >::const_iterator vsizes = tags_sizes.begin(); 07807 for( ; vit != src_tags.end(); ++vit, ++vit2, ++vsizes ) 07808 { 07809 if( *vit == *vit2 ) continue; 07810 vals.resize( entities.size() * ( *vsizes ) ); 07811 result = mbImpl->tag_get_data( *vit, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't get data properly" ); 07812 result = mbImpl->tag_set_data( *vit2, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't set data properly" ); 07813 } 07814 07815 int dum_ack_buff; 07816 07817 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 07818 { 07819 Range tag_ents = entities; 07820 07821 // Get ents shared by proc *sit 07822 result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" ); 07823 07824 // Pack-send 07825 std::vector< Range > tag_ranges; 07826 for( vit = src_tags.begin(); vit != src_tags.end(); ++vit ) 07827 { 07828 const void* ptr; 07829 int sz; 07830 if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS ) 07831 { 07832 Range tagged_ents; 07833 mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents ); 07834 tag_ranges.push_back( intersect( tag_ents, tagged_ents ) ); 07835 } 07836 else 07837 tag_ranges.push_back( tag_ents ); 07838 } 07839 07840 // Pack the data 07841 // Reserve space on front for size and for initial buff size 07842 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 07843 07844 result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" ); 07845 07846 // Now send it 07847 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind], 07848 recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" ); 07849 } 07850 07851 // Receive/unpack tags 07852 while( incoming ) 07853 { 07854 MPI_Status status; 07855 int index_in_recv_requests; 07856 PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() ); 07857 success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status ); 07858 if( MPI_SUCCESS != success ) 07859 { 07860 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 07861 } 07862 ind = index_in_recv_requests / 3; 07863 07864 PRINT_DEBUG_RECD( status ); 07865 07866 // OK, received something; decrement incoming counter 07867 incoming--; 07868 07869 bool done = false; 07870 std::vector< EntityHandle > dum_vec; 07871 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind], 07872 recv_tag_reqs[3 * ind + 1], // This is for receiving the second message 07873 recv_tag_reqs[3 * ind + 2], // This would be for ack, but it is not 07874 // used; consider removing it 07875 incoming, localOwnedBuffs[ind], 07876 sendReqs[3 * ind + 1], // Send request for sending the second message 07877 sendReqs[3 * ind + 2], // This is for sending the ack 07878 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" ); 07879 if( done ) 07880 { 07881 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) ); 07882 result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind], &mpi_op );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" ); 07883 } 07884 } 07885 07886 // OK, now wait 07887 if( myDebug->get_verbosity() == 5 ) 07888 { 07889 success = MPI_Barrier( procConfig.proc_comm() ); 07890 } 07891 else 07892 { 07893 MPI_Status status[3 * MAX_SHARING_PROCS]; 07894 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status ); 07895 } 07896 if( MPI_SUCCESS != success ) 07897 { 07898 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" ); 07899 } 07900 07901 myDebug->tprintf( 1, "Exiting reduce_tags" ); 07902 07903 return MB_SUCCESS; 07904 } 07905 07906 //! return sharedp tag 07907 Tag ParallelComm::sharedp_tag() 07908 { 07909 if( !sharedpTag ) 07910 { 07911 int def_val = -1; 07912 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROC_TAG_NAME, 1, MB_TYPE_INTEGER, sharedpTag, 07913 MB_TAG_DENSE | MB_TAG_CREAT, &def_val ); 07914 if( MB_SUCCESS != result ) return 0; 07915 } 07916 07917 return sharedpTag; 07918 } 07919 07920 //! return sharedps tag 07921 Tag ParallelComm::sharedps_tag() 07922 { 07923 if( !sharedpsTag ) 07924 { 07925 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROCS_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_INTEGER, 07926 sharedpsTag, MB_TAG_SPARSE | MB_TAG_CREAT ); 07927 if( MB_SUCCESS != result ) return 0; 07928 } 07929 07930 return sharedpsTag; 07931 } 07932 07933 //! return sharedh tag 07934 Tag ParallelComm::sharedh_tag() 07935 { 07936 if( !sharedhTag ) 07937 { 07938 EntityHandle def_val = 0; 07939 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLE_TAG_NAME, 1, MB_TYPE_HANDLE, sharedhTag, 07940 MB_TAG_DENSE | MB_TAG_CREAT, &def_val ); 07941 if( MB_SUCCESS != result ) return 0; 07942 } 07943 07944 return sharedhTag; 07945 } 07946 07947 //! return sharedhs tag 07948 Tag ParallelComm::sharedhs_tag() 07949 { 07950 if( !sharedhsTag ) 07951 { 07952 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLES_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_HANDLE, 07953 sharedhsTag, MB_TAG_SPARSE | MB_TAG_CREAT ); 07954 if( MB_SUCCESS != result ) return 0; 07955 } 07956 07957 return sharedhsTag; 07958 } 07959 07960 //! return pstatus tag 07961 Tag ParallelComm::pstatus_tag() 07962 { 07963 if( !pstatusTag ) 07964 { 07965 unsigned char tmp_pstatus = 0; 07966 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_STATUS_TAG_NAME, 1, MB_TYPE_OPAQUE, pstatusTag, 07967 MB_TAG_DENSE | MB_TAG_CREAT, &tmp_pstatus ); 07968 if( MB_SUCCESS != result ) return 0; 07969 } 07970 07971 return pstatusTag; 07972 } 07973 07974 //! return partition set tag 07975 Tag ParallelComm::partition_tag() 07976 { 07977 if( !partitionTag ) 07978 { 07979 int dum_id = -1; 07980 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_PARTITION_TAG_NAME, 1, MB_TYPE_INTEGER, partitionTag, 07981 MB_TAG_SPARSE | MB_TAG_CREAT, &dum_id ); 07982 if( MB_SUCCESS != result ) return 0; 07983 } 07984 07985 return partitionTag; 07986 } 07987 07988 //! return pcomm tag; passes in impl 'cuz this is a static function 07989 Tag ParallelComm::pcomm_tag( Interface* impl, bool create_if_missing ) 07990 { 07991 Tag this_tag = 0; 07992 ErrorCode result; 07993 if( create_if_missing ) 07994 { 07995 result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ), 07996 MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE | MB_TAG_CREAT ); 07997 } 07998 else 07999 { 08000 result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ), 08001 MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE ); 08002 } 08003 08004 if( MB_SUCCESS != result ) return 0; 08005 08006 return this_tag; 08007 } 08008 08009 //! get the indexed pcomm object from the interface 08010 ParallelComm* ParallelComm::get_pcomm( Interface* impl, const int index ) 08011 { 08012 Tag pc_tag = pcomm_tag( impl, false ); 08013 if( 0 == pc_tag ) return NULL; 08014 08015 const EntityHandle root = 0; 08016 ParallelComm* pc_array[MAX_SHARING_PROCS]; 08017 ErrorCode result = impl->tag_get_data( pc_tag, &root, 1, (void*)pc_array ); 08018 if( MB_SUCCESS != result ) return NULL; 08019 08020 return pc_array[index]; 08021 } 08022 08023 ErrorCode ParallelComm::get_all_pcomm( Interface* impl, std::vector< ParallelComm* >& list ) 08024 { 08025 Tag pc_tag = pcomm_tag( impl, false ); 08026 if( 0 == pc_tag ) return MB_TAG_NOT_FOUND; 08027 08028 const EntityHandle root = 0; 08029 ParallelComm* pc_array[MAX_SHARING_PROCS]; 08030 ErrorCode rval = impl->tag_get_data( pc_tag, &root, 1, pc_array ); 08031 if( MB_SUCCESS != rval ) return rval; 08032 08033 for( int i = 0; i < MAX_SHARING_PROCS; i++ ) 08034 { 08035 if( pc_array[i] ) list.push_back( pc_array[i] ); 08036 } 08037 08038 return MB_SUCCESS; 08039 } 08040 08041 //! get the indexed pcomm object from the interface 08042 ParallelComm* ParallelComm::get_pcomm( Interface* impl, EntityHandle prtn, const MPI_Comm* comm ) 08043 { 08044 ErrorCode rval; 08045 ParallelComm* result = 0; 08046 08047 Tag prtn_tag; 08048 rval = 08049 impl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag, MB_TAG_SPARSE | MB_TAG_CREAT ); 08050 if( MB_SUCCESS != rval ) return 0; 08051 08052 int pcomm_id; 08053 rval = impl->tag_get_data( prtn_tag, &prtn, 1, &pcomm_id ); 08054 if( MB_SUCCESS == rval ) 08055 { 08056 result = get_pcomm( impl, pcomm_id ); 08057 } 08058 else if( MB_TAG_NOT_FOUND == rval && comm ) 08059 { 08060 result = new ParallelComm( impl, *comm, &pcomm_id ); 08061 if( !result ) return 0; 08062 result->set_partitioning( prtn ); 08063 08064 rval = impl->tag_set_data( prtn_tag, &prtn, 1, &pcomm_id ); 08065 if( MB_SUCCESS != rval ) 08066 { 08067 delete result; 08068 result = 0; 08069 } 08070 } 08071 08072 return result; 08073 } 08074 08075 ErrorCode ParallelComm::set_partitioning( EntityHandle set ) 08076 { 08077 ErrorCode rval; 08078 Tag prtn_tag; 08079 rval = mbImpl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag, 08080 MB_TAG_SPARSE | MB_TAG_CREAT ); 08081 if( MB_SUCCESS != rval ) return rval; 08082 08083 // Get my id 08084 ParallelComm* pcomm_arr[MAX_SHARING_PROCS]; 08085 Tag pc_tag = pcomm_tag( mbImpl, false ); 08086 if( 0 == pc_tag ) return MB_FAILURE; 08087 const EntityHandle root = 0; 08088 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, pcomm_arr ); 08089 if( MB_SUCCESS != result ) return MB_FAILURE; 08090 int id = std::find( pcomm_arr, pcomm_arr + MAX_SHARING_PROCS, this ) - pcomm_arr; 08091 if( id == MAX_SHARING_PROCS ) return MB_FAILURE; 08092 08093 EntityHandle old = partitioningSet; 08094 if( old ) 08095 { 08096 rval = mbImpl->tag_delete_data( prtn_tag, &old, 1 ); 08097 if( MB_SUCCESS != rval ) return rval; 08098 partitioningSet = 0; 08099 } 08100 08101 if( !set ) return MB_SUCCESS; 08102 08103 Range contents; 08104 if( old ) 08105 { 08106 rval = mbImpl->get_entities_by_handle( old, contents ); 08107 if( MB_SUCCESS != rval ) return rval; 08108 } 08109 else 08110 { 08111 contents = partition_sets(); 08112 } 08113 08114 rval = mbImpl->add_entities( set, contents ); 08115 if( MB_SUCCESS != rval ) return rval; 08116 08117 // Store pcomm id on new partition set 08118 rval = mbImpl->tag_set_data( prtn_tag, &set, 1, &id ); 08119 if( MB_SUCCESS != rval ) return rval; 08120 08121 partitioningSet = set; 08122 return MB_SUCCESS; 08123 } 08124 08125 //! return all the entities in parts owned locally 08126 ErrorCode ParallelComm::get_part_entities( Range& ents, int dim ) 08127 { 08128 ErrorCode result; 08129 08130 for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit ) 08131 { 08132 Range tmp_ents; 08133 if( -1 == dim ) 08134 result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true ); 08135 else 08136 result = mbImpl->get_entities_by_dimension( *rit, dim, tmp_ents, true ); 08137 08138 if( MB_SUCCESS != result ) return result; 08139 ents.merge( tmp_ents ); 08140 } 08141 08142 return MB_SUCCESS; 08143 } 08144 08145 /** \brief Return the rank of the entity owner 08146 */ 08147 ErrorCode ParallelComm::get_owner_handle( EntityHandle entity, int& owner, EntityHandle& handle ) 08148 { 08149 unsigned char pstat; 08150 int sharing_procs[MAX_SHARING_PROCS]; 08151 EntityHandle sharing_handles[MAX_SHARING_PROCS]; 08152 08153 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 08154 if( !( pstat & PSTATUS_NOT_OWNED ) ) 08155 { 08156 owner = proc_config().proc_rank(); 08157 handle = entity; 08158 } 08159 else if( pstat & PSTATUS_MULTISHARED ) 08160 { 08161 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" ); 08162 owner = sharing_procs[0]; 08163 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" ); 08164 handle = sharing_handles[0]; 08165 } 08166 else if( pstat & PSTATUS_SHARED ) 08167 { 08168 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 08169 owner = sharing_procs[0]; 08170 result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" ); 08171 handle = sharing_handles[0]; 08172 } 08173 else 08174 { 08175 owner = -1; 08176 handle = 0; 08177 } 08178 08179 return MB_SUCCESS; 08180 } 08181 08182 ErrorCode ParallelComm::get_global_part_count( int& count_out ) const 08183 { 08184 count_out = globalPartCount; 08185 return count_out < 0 ? MB_FAILURE : MB_SUCCESS; 08186 } 08187 08188 ErrorCode ParallelComm::get_part_owner( int part_id, int& owner ) const 08189 { 08190 // FIXME: assumes only 1 local part 08191 owner = part_id; 08192 return MB_SUCCESS; 08193 } 08194 08195 ErrorCode ParallelComm::get_part_id( EntityHandle /*part*/, int& id_out ) const 08196 { 08197 // FIXME: assumes only 1 local part 08198 id_out = proc_config().proc_rank(); 08199 return MB_SUCCESS; 08200 } 08201 08202 ErrorCode ParallelComm::get_part_handle( int id, EntityHandle& handle_out ) const 08203 { 08204 // FIXME: assumes only 1 local part 08205 if( (unsigned)id != proc_config().proc_rank() ) return MB_ENTITY_NOT_FOUND; 08206 handle_out = partition_sets().front(); 08207 return MB_SUCCESS; 08208 } 08209 08210 ErrorCode ParallelComm::create_part( EntityHandle& set_out ) 08211 { 08212 // Mark as invalid so we know that it needs to be updated 08213 globalPartCount = -1; 08214 08215 // Create set representing part 08216 ErrorCode rval = mbImpl->create_meshset( MESHSET_SET, set_out ); 08217 if( MB_SUCCESS != rval ) return rval; 08218 08219 // Set tag on set 08220 int val = proc_config().proc_rank(); 08221 rval = mbImpl->tag_set_data( part_tag(), &set_out, 1, &val ); 08222 08223 if( MB_SUCCESS != rval ) 08224 { 08225 mbImpl->delete_entities( &set_out, 1 ); 08226 return rval; 08227 } 08228 08229 if( get_partitioning() ) 08230 { 08231 rval = mbImpl->add_entities( get_partitioning(), &set_out, 1 ); 08232 if( MB_SUCCESS != rval ) 08233 { 08234 mbImpl->delete_entities( &set_out, 1 ); 08235 return rval; 08236 } 08237 } 08238 08239 moab::Range& pSets = this->partition_sets(); 08240 if( pSets.index( set_out ) < 0 ) 08241 { 08242 pSets.insert( set_out ); 08243 } 08244 08245 return MB_SUCCESS; 08246 } 08247 08248 ErrorCode ParallelComm::destroy_part( EntityHandle part_id ) 08249 { 08250 // Mark as invalid so we know that it needs to be updated 08251 globalPartCount = -1; 08252 08253 ErrorCode rval; 08254 if( get_partitioning() ) 08255 { 08256 rval = mbImpl->remove_entities( get_partitioning(), &part_id, 1 ); 08257 if( MB_SUCCESS != rval ) return rval; 08258 } 08259 08260 moab::Range& pSets = this->partition_sets(); 08261 if( pSets.index( part_id ) >= 0 ) 08262 { 08263 pSets.erase( part_id ); 08264 } 08265 return mbImpl->delete_entities( &part_id, 1 ); 08266 } 08267 08268 ErrorCode ParallelComm::collective_sync_partition() 08269 { 08270 int count = partition_sets().size(); 08271 globalPartCount = 0; 08272 int err = MPI_Allreduce( &count, &globalPartCount, 1, MPI_INT, MPI_SUM, proc_config().proc_comm() ); 08273 return err ? MB_FAILURE : MB_SUCCESS; 08274 } 08275 08276 ErrorCode ParallelComm::get_part_neighbor_ids( EntityHandle part, 08277 int neighbors_out[MAX_SHARING_PROCS], 08278 int& num_neighbors_out ) 08279 { 08280 ErrorCode rval; 08281 Range iface; 08282 rval = get_interface_sets( part, iface ); 08283 if( MB_SUCCESS != rval ) return rval; 08284 08285 num_neighbors_out = 0; 08286 int n, j = 0; 08287 int tmp[MAX_SHARING_PROCS] = { 0 }, curr[MAX_SHARING_PROCS] = { 0 }; 08288 int* parts[2] = { neighbors_out, tmp }; 08289 for( Range::iterator i = iface.begin(); i != iface.end(); ++i ) 08290 { 08291 unsigned char pstat; 08292 rval = get_sharing_data( *i, curr, NULL, pstat, n ); 08293 if( MB_SUCCESS != rval ) return rval; 08294 std::sort( curr, curr + n ); 08295 assert( num_neighbors_out < MAX_SHARING_PROCS ); 08296 int* k = std::set_union( parts[j], parts[j] + num_neighbors_out, curr, curr + n, parts[1 - j] ); 08297 j = 1 - j; 08298 num_neighbors_out = k - parts[j]; 08299 } 08300 if( parts[j] != neighbors_out ) std::copy( parts[j], parts[j] + num_neighbors_out, neighbors_out ); 08301 08302 // Remove input part from list 08303 int id; 08304 rval = get_part_id( part, id ); 08305 if( MB_SUCCESS == rval ) 08306 num_neighbors_out = std::remove( neighbors_out, neighbors_out + num_neighbors_out, id ) - neighbors_out; 08307 return rval; 08308 } 08309 08310 ErrorCode ParallelComm::get_interface_sets( EntityHandle, Range& iface_sets_out, int* adj_part_id ) 08311 { 08312 // FIXME : assumes one part per processor. 08313 // Need to store part iface sets as children to implement 08314 // this correctly. 08315 iface_sets_out = interface_sets(); 08316 08317 if( adj_part_id ) 08318 { 08319 int part_ids[MAX_SHARING_PROCS], num_parts; 08320 Range::iterator i = iface_sets_out.begin(); 08321 while( i != iface_sets_out.end() ) 08322 { 08323 unsigned char pstat; 08324 ErrorCode rval = get_sharing_data( *i, part_ids, NULL, pstat, num_parts ); 08325 if( MB_SUCCESS != rval ) return rval; 08326 08327 if( std::find( part_ids, part_ids + num_parts, *adj_part_id ) - part_ids != num_parts ) 08328 ++i; 08329 else 08330 i = iface_sets_out.erase( i ); 08331 } 08332 } 08333 08334 return MB_SUCCESS; 08335 } 08336 08337 ErrorCode ParallelComm::get_owning_part( EntityHandle handle, int& owning_part_id, EntityHandle* remote_handle ) 08338 { 08339 // FIXME : assumes one part per proc, and therefore part_id == rank 08340 08341 // If entity is not shared, then we're the owner. 08342 unsigned char pstat; 08343 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &handle, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 08344 if( !( pstat & PSTATUS_NOT_OWNED ) ) 08345 { 08346 owning_part_id = proc_config().proc_rank(); 08347 if( remote_handle ) *remote_handle = handle; 08348 return MB_SUCCESS; 08349 } 08350 08351 // If entity is shared with one other proc, then 08352 // sharedp_tag will contain a positive value. 08353 result = mbImpl->tag_get_data( sharedp_tag(), &handle, 1, &owning_part_id );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 08354 if( owning_part_id != -1 ) 08355 { 08356 // Done? 08357 if( !remote_handle ) return MB_SUCCESS; 08358 08359 // Get handles on remote processors (and this one) 08360 return mbImpl->tag_get_data( sharedh_tag(), &handle, 1, remote_handle ); 08361 } 08362 08363 // If here, then the entity is shared with at least two other processors. 08364 // Get the list from the sharedps_tag 08365 const void* part_id_list = 0; 08366 result = mbImpl->tag_get_by_ptr( sharedps_tag(), &handle, 1, &part_id_list ); 08367 if( MB_SUCCESS != result ) return result; 08368 owning_part_id = ( (const int*)part_id_list )[0]; 08369 08370 // Done? 08371 if( !remote_handle ) return MB_SUCCESS; 08372 08373 // Get remote handles 08374 const void* handle_list = 0; 08375 result = mbImpl->tag_get_by_ptr( sharedhs_tag(), &handle, 1, &handle_list ); 08376 if( MB_SUCCESS != result ) return result; 08377 08378 *remote_handle = ( (const EntityHandle*)handle_list )[0]; 08379 return MB_SUCCESS; 08380 } 08381 08382 ErrorCode ParallelComm::get_sharing_parts( EntityHandle entity, 08383 int part_ids_out[MAX_SHARING_PROCS], 08384 int& num_part_ids_out, 08385 EntityHandle remote_handles[MAX_SHARING_PROCS] ) 08386 { 08387 // FIXME : assumes one part per proc, and therefore part_id == rank 08388 08389 // If entity is not shared, then we're the owner. 08390 unsigned char pstat; 08391 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" ); 08392 if( !( pstat & PSTATUS_SHARED ) ) 08393 { 08394 part_ids_out[0] = proc_config().proc_rank(); 08395 if( remote_handles ) remote_handles[0] = entity; 08396 num_part_ids_out = 1; 08397 return MB_SUCCESS; 08398 } 08399 08400 // If entity is shared with one other proc, then 08401 // sharedp_tag will contain a positive value. 08402 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, part_ids_out );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" ); 08403 if( part_ids_out[0] != -1 ) 08404 { 08405 num_part_ids_out = 2; 08406 part_ids_out[1] = proc_config().proc_rank(); 08407 08408 // Done? 08409 if( !remote_handles ) return MB_SUCCESS; 08410 08411 // Get handles on remote processors (and this one) 08412 remote_handles[1] = entity; 08413 return mbImpl->tag_get_data( sharedh_tag(), &entity, 1, remote_handles ); 08414 } 08415 08416 // If here, then the entity is shared with at least two other processors. 08417 // Get the list from the sharedps_tag 08418 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, part_ids_out ); 08419 if( MB_SUCCESS != result ) return result; 08420 // Count number of valid (positive) entries in sharedps_tag 08421 for( num_part_ids_out = 0; num_part_ids_out < MAX_SHARING_PROCS && part_ids_out[num_part_ids_out] >= 0; 08422 num_part_ids_out++ ) 08423 ; 08424 // part_ids_out[num_part_ids_out++] = proc_config().proc_rank(); 08425 #ifndef NDEBUG 08426 int my_idx = std::find( part_ids_out, part_ids_out + num_part_ids_out, proc_config().proc_rank() ) - part_ids_out; 08427 assert( my_idx < num_part_ids_out ); 08428 #endif 08429 08430 // Done? 08431 if( !remote_handles ) return MB_SUCCESS; 08432 08433 // Get remote handles 08434 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, remote_handles ); 08435 // remote_handles[num_part_ids_out - 1] = entity; 08436 assert( remote_handles[my_idx] == entity ); 08437 08438 return result; 08439 } 08440 08441 ErrorCode ParallelComm::pack_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data ) 08442 { 08443 // Build up send buffers 08444 ErrorCode rval = MB_SUCCESS; 08445 int ent_procs[MAX_SHARING_PROCS]; 08446 EntityHandle handles[MAX_SHARING_PROCS]; 08447 int num_sharing, tmp_int; 08448 SharedEntityData tmp; 08449 send_data.resize( buffProcs.size() ); 08450 for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i ) 08451 { 08452 tmp.remote = *i; // Swap local/remote so they're correct on the remote proc. 08453 rval = get_owner( *i, tmp_int ); 08454 tmp.owner = tmp_int; 08455 if( MB_SUCCESS != rval ) return rval; 08456 08457 unsigned char pstat; 08458 rval = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing ); 08459 if( MB_SUCCESS != rval ) return rval; 08460 for( int j = 0; j < num_sharing; j++ ) 08461 { 08462 if( ent_procs[j] == (int)proc_config().proc_rank() ) continue; 08463 tmp.local = handles[j]; 08464 int ind = get_buffers( ent_procs[j] ); 08465 assert( -1 != ind ); 08466 if( (int)send_data.size() < ind + 1 ) send_data.resize( ind + 1 ); 08467 send_data[ind].push_back( tmp ); 08468 } 08469 } 08470 08471 return MB_SUCCESS; 08472 } 08473 08474 ErrorCode ParallelComm::exchange_all_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data, 08475 std::vector< std::vector< SharedEntityData > >& result ) 08476 { 08477 int ierr; 08478 const int tag = 0; 08479 const MPI_Comm cm = procConfig.proc_comm(); 08480 const int num_proc = buffProcs.size(); 08481 const std::vector< int > procs( buffProcs.begin(), buffProcs.end() ); 08482 std::vector< MPI_Request > recv_req( buffProcs.size(), MPI_REQUEST_NULL ); 08483 std::vector< MPI_Request > send_req( buffProcs.size(), MPI_REQUEST_NULL ); 08484 08485 // Set up to receive sizes 08486 std::vector< int > sizes_send( num_proc ), sizes_recv( num_proc ); 08487 for( int i = 0; i < num_proc; i++ ) 08488 { 08489 ierr = MPI_Irecv( &sizes_recv[i], 1, MPI_INT, procs[i], tag, cm, &recv_req[i] ); 08490 if( ierr ) return MB_FILE_WRITE_ERROR; 08491 } 08492 08493 // Send sizes 08494 assert( num_proc == (int)send_data.size() ); 08495 08496 result.resize( num_proc ); 08497 for( int i = 0; i < num_proc; i++ ) 08498 { 08499 sizes_send[i] = send_data[i].size(); 08500 ierr = MPI_Isend( &sizes_send[i], 1, MPI_INT, buffProcs[i], tag, cm, &send_req[i] ); 08501 if( ierr ) return MB_FILE_WRITE_ERROR; 08502 } 08503 08504 // Receive sizes 08505 std::vector< MPI_Status > stat( num_proc ); 08506 ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] ); 08507 if( ierr ) return MB_FILE_WRITE_ERROR; 08508 08509 // Wait until all sizes are sent (clean up pending req's) 08510 ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] ); 08511 if( ierr ) return MB_FILE_WRITE_ERROR; 08512 08513 // Set up to receive data 08514 for( int i = 0; i < num_proc; i++ ) 08515 { 08516 result[i].resize( sizes_recv[i] ); 08517 ierr = MPI_Irecv( (void*)( &( result[i][0] ) ), sizeof( SharedEntityData ) * sizes_recv[i], MPI_UNSIGNED_CHAR, 08518 buffProcs[i], tag, cm, &recv_req[i] ); 08519 if( ierr ) return MB_FILE_WRITE_ERROR; 08520 } 08521 08522 // Send data 08523 for( int i = 0; i < num_proc; i++ ) 08524 { 08525 ierr = MPI_Isend( (void*)( &( send_data[i][0] ) ), sizeof( SharedEntityData ) * sizes_send[i], 08526 MPI_UNSIGNED_CHAR, buffProcs[i], tag, cm, &send_req[i] ); 08527 if( ierr ) return MB_FILE_WRITE_ERROR; 08528 } 08529 08530 // Receive data 08531 ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] ); 08532 if( ierr ) return MB_FILE_WRITE_ERROR; 08533 08534 // Wait until everything is sent to release send buffers 08535 ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] ); 08536 if( ierr ) return MB_FILE_WRITE_ERROR; 08537 08538 return MB_SUCCESS; 08539 } 08540 08541 ErrorCode ParallelComm::check_all_shared_handles( bool print_em ) 08542 { 08543 // Get all shared ent data from other procs 08544 std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() ); 08545 08546 ErrorCode result; 08547 bool done = false; 08548 08549 while( !done ) 08550 { 08551 result = check_local_shared(); 08552 if( MB_SUCCESS != result ) 08553 { 08554 done = true; 08555 continue; 08556 } 08557 08558 result = pack_shared_handles( send_data ); 08559 if( MB_SUCCESS != result ) 08560 { 08561 done = true; 08562 continue; 08563 } 08564 08565 result = exchange_all_shared_handles( send_data, shents ); 08566 if( MB_SUCCESS != result ) 08567 { 08568 done = true; 08569 continue; 08570 } 08571 08572 if( !shents.empty() ) result = check_my_shared_handles( shents ); 08573 done = true; 08574 } 08575 08576 if( MB_SUCCESS != result && print_em ) 08577 { 08578 #ifdef MOAB_HAVE_HDF5 08579 std::ostringstream ent_str; 08580 ent_str << "mesh." << procConfig.proc_rank() << ".h5m"; 08581 mbImpl->write_mesh( ent_str.str().c_str() ); 08582 #endif 08583 } 08584 08585 return result; 08586 } 08587 08588 ErrorCode ParallelComm::check_local_shared() 08589 { 08590 // Do some checks on shared entities to make sure things look 08591 // consistent 08592 08593 // Check that non-vertex shared entities are shared by same procs as all 08594 // their vertices 08595 // std::pair<Range::const_iterator,Range::const_iterator> vert_it = 08596 // sharedEnts.equal_range(MBVERTEX); 08597 std::vector< EntityHandle > dum_connect; 08598 const EntityHandle* connect; 08599 int num_connect; 08600 int tmp_procs[MAX_SHARING_PROCS]; 08601 EntityHandle tmp_hs[MAX_SHARING_PROCS]; 08602 std::set< int > tmp_set, vset; 08603 int num_ps; 08604 ErrorCode result; 08605 unsigned char pstat; 08606 std::vector< EntityHandle > bad_ents; 08607 std::vector< std::string > errors; 08608 08609 std::set< EntityHandle >::iterator vit; 08610 for( vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit ) 08611 { 08612 // Get sharing procs for this ent 08613 result = get_sharing_data( *vit, tmp_procs, tmp_hs, pstat, num_ps ); 08614 if( MB_SUCCESS != result ) 08615 { 08616 bad_ents.push_back( *vit ); 08617 errors.push_back( std::string( "Failure getting sharing data." ) ); 08618 continue; 08619 } 08620 08621 bool bad = false; 08622 // Entity must be shared 08623 if( !( pstat & PSTATUS_SHARED ) ) 08624 errors.push_back( std::string( "Entity should be shared but isn't." ) ), bad = true; 08625 08626 // If entity is not owned this must not be first proc 08627 if( pstat & PSTATUS_NOT_OWNED && tmp_procs[0] == (int)procConfig.proc_rank() ) 08628 errors.push_back( std::string( "Entity not owned but is first proc." ) ), bad = true; 08629 08630 // If entity is owned and multishared, this must be first proc 08631 if( !( pstat & PSTATUS_NOT_OWNED ) && pstat & PSTATUS_MULTISHARED && 08632 ( tmp_procs[0] != (int)procConfig.proc_rank() || tmp_hs[0] != *vit ) ) 08633 errors.push_back( std::string( "Entity owned and multishared but not first proc or not first handle." ) ), 08634 bad = true; 08635 08636 if( bad ) 08637 { 08638 bad_ents.push_back( *vit ); 08639 continue; 08640 } 08641 08642 EntityType type = mbImpl->type_from_handle( *vit ); 08643 if( type == MBVERTEX || type == MBENTITYSET ) continue; 08644 08645 // Copy element's procs to vset and save size 08646 int orig_ps = num_ps; 08647 vset.clear(); 08648 std::copy( tmp_procs, tmp_procs + num_ps, std::inserter( vset, vset.begin() ) ); 08649 08650 // Get vertices for this ent and intersection of sharing procs 08651 result = mbImpl->get_connectivity( *vit, connect, num_connect, false, &dum_connect ); 08652 if( MB_SUCCESS != result ) 08653 { 08654 bad_ents.push_back( *vit ); 08655 errors.push_back( std::string( "Failed to get connectivity." ) ); 08656 continue; 08657 } 08658 08659 for( int i = 0; i < num_connect; i++ ) 08660 { 08661 result = get_sharing_data( connect[i], tmp_procs, NULL, pstat, num_ps ); 08662 if( MB_SUCCESS != result ) 08663 { 08664 bad_ents.push_back( *vit ); 08665 continue; 08666 } 08667 if( !num_ps ) 08668 { 08669 vset.clear(); 08670 break; 08671 } 08672 std::sort( tmp_procs, tmp_procs + num_ps ); 08673 tmp_set.clear(); 08674 std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(), 08675 std::inserter( tmp_set, tmp_set.end() ) ); 08676 vset.swap( tmp_set ); 08677 if( vset.empty() ) break; 08678 } 08679 08680 // Intersect them; should be the same size as orig_ps 08681 tmp_set.clear(); 08682 std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(), 08683 std::inserter( tmp_set, tmp_set.end() ) ); 08684 if( orig_ps != (int)tmp_set.size() ) 08685 { 08686 errors.push_back( std::string( "Vertex proc set not same size as entity proc set." ) ); 08687 bad_ents.push_back( *vit ); 08688 for( int i = 0; i < num_connect; i++ ) 08689 { 08690 bad_ents.push_back( connect[i] ); 08691 errors.push_back( std::string( "vertex in connect" ) ); 08692 } 08693 } 08694 } 08695 08696 if( !bad_ents.empty() ) 08697 { 08698 std::cout << "Found bad entities in check_local_shared, proc rank " << procConfig.proc_rank() << "," 08699 << std::endl; 08700 std::vector< std::string >::iterator sit; 08701 std::vector< EntityHandle >::iterator rit; 08702 for( rit = bad_ents.begin(), sit = errors.begin(); rit != bad_ents.end(); ++rit, ++sit ) 08703 { 08704 list_entities( &( *rit ), 1 ); 08705 std::cout << "Reason: " << *sit << std::endl; 08706 } 08707 return MB_FAILURE; 08708 } 08709 08710 // To do: check interface sets 08711 08712 return MB_SUCCESS; 08713 } 08714 08715 ErrorCode ParallelComm::check_all_shared_handles( ParallelComm** pcs, int num_pcs ) 08716 { 08717 std::vector< std::vector< std::vector< SharedEntityData > > > shents, send_data; 08718 ErrorCode result = MB_SUCCESS, tmp_result; 08719 08720 // Get all shared ent data from each proc to all other procs 08721 send_data.resize( num_pcs ); 08722 for( int p = 0; p < num_pcs; p++ ) 08723 { 08724 tmp_result = pcs[p]->pack_shared_handles( send_data[p] ); 08725 if( MB_SUCCESS != tmp_result ) result = tmp_result; 08726 } 08727 if( MB_SUCCESS != result ) return result; 08728 08729 // Move the data sorted by sending proc to data sorted by receiving proc 08730 shents.resize( num_pcs ); 08731 for( int p = 0; p < num_pcs; p++ ) 08732 shents[p].resize( pcs[p]->buffProcs.size() ); 08733 08734 for( int p = 0; p < num_pcs; p++ ) 08735 { 08736 for( unsigned int idx_p = 0; idx_p < pcs[p]->buffProcs.size(); idx_p++ ) 08737 { 08738 // Move send_data[p][to_p] to shents[to_p][idx_p] 08739 int to_p = pcs[p]->buffProcs[idx_p]; 08740 int top_idx_p = pcs[to_p]->get_buffers( p ); 08741 assert( -1 != top_idx_p ); 08742 shents[to_p][top_idx_p] = send_data[p][idx_p]; 08743 } 08744 } 08745 08746 for( int p = 0; p < num_pcs; p++ ) 08747 { 08748 std::ostringstream ostr; 08749 ostr << "Processor " << p << " bad entities:"; 08750 tmp_result = pcs[p]->check_my_shared_handles( shents[p], ostr.str().c_str() ); 08751 if( MB_SUCCESS != tmp_result ) result = tmp_result; 08752 } 08753 08754 return result; 08755 } 08756 08757 ErrorCode ParallelComm::check_my_shared_handles( std::vector< std::vector< SharedEntityData > >& shents, 08758 const char* prefix ) 08759 { 08760 // Now check against what I think data should be 08761 // Get all shared entities 08762 ErrorCode result; 08763 Range all_shared; 08764 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( all_shared ) ); 08765 std::vector< EntityHandle > dum_vec; 08766 all_shared.erase( all_shared.upper_bound( MBPOLYHEDRON ), all_shared.end() ); 08767 08768 Range bad_ents, local_shared; 08769 std::vector< SharedEntityData >::iterator vit; 08770 unsigned char tmp_pstat; 08771 for( unsigned int i = 0; i < shents.size(); i++ ) 08772 { 08773 int other_proc = buffProcs[i]; 08774 result = get_shared_entities( other_proc, local_shared ); 08775 if( MB_SUCCESS != result ) return result; 08776 for( vit = shents[i].begin(); vit != shents[i].end(); ++vit ) 08777 { 08778 EntityHandle localh = vit->local, remoteh = vit->remote, dumh; 08779 local_shared.erase( localh ); 08780 result = get_remote_handles( true, &localh, &dumh, 1, other_proc, dum_vec ); 08781 if( MB_SUCCESS != result || dumh != remoteh ) bad_ents.insert( localh ); 08782 result = get_pstatus( localh, tmp_pstat ); 08783 if( MB_SUCCESS != result || ( !( tmp_pstat & PSTATUS_NOT_OWNED ) && (unsigned)vit->owner != rank() ) || 08784 ( tmp_pstat & PSTATUS_NOT_OWNED && (unsigned)vit->owner == rank() ) ) 08785 bad_ents.insert( localh ); 08786 } 08787 08788 if( !local_shared.empty() ) bad_ents.merge( local_shared ); 08789 } 08790 08791 if( !bad_ents.empty() ) 08792 { 08793 if( prefix ) std::cout << prefix << std::endl; 08794 list_entities( bad_ents ); 08795 return MB_FAILURE; 08796 } 08797 else 08798 return MB_SUCCESS; 08799 } 08800 08801 ErrorCode ParallelComm::get_shared_entities( int other_proc, 08802 Range& shared_ents, 08803 int dim, 08804 const bool iface, 08805 const bool owned_filter ) 08806 { 08807 shared_ents.clear(); 08808 ErrorCode result = MB_SUCCESS; 08809 08810 // Dimension 08811 if( -1 != dim ) 08812 { 08813 DimensionPair dp = CN::TypeDimensionMap[dim]; 08814 Range dum_range; 08815 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( dum_range ) ); 08816 shared_ents.merge( dum_range.lower_bound( dp.first ), dum_range.upper_bound( dp.second ) ); 08817 } 08818 else 08819 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) ); 08820 08821 // Filter by iface 08822 if( iface ) 08823 { 08824 result = filter_pstatus( shared_ents, PSTATUS_INTERFACE, PSTATUS_AND );MB_CHK_SET_ERR( result, "Failed to filter by iface" ); 08825 } 08826 08827 // Filter by owned 08828 if( owned_filter ) 08829 { 08830 result = filter_pstatus( shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed to filter by owned" ); 08831 } 08832 08833 // Filter by proc 08834 if( -1 != other_proc ) 08835 { 08836 result = filter_pstatus( shared_ents, PSTATUS_SHARED, PSTATUS_AND, other_proc );MB_CHK_SET_ERR( result, "Failed to filter by proc" ); 08837 } 08838 08839 return result; 08840 } 08841 08842 ErrorCode ParallelComm::clean_shared_tags( std::vector< Range* >& exchange_ents ) 08843 { 08844 for( unsigned int i = 0; i < exchange_ents.size(); i++ ) 08845 { 08846 Range* ents = exchange_ents[i]; 08847 int num_ents = ents->size(); 08848 Range::iterator it = ents->begin(); 08849 08850 for( int n = 0; n < num_ents; n++ ) 08851 { 08852 int sharing_proc; 08853 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &( *ents->begin() ), 1, &sharing_proc ); 08854 if( result != MB_TAG_NOT_FOUND && sharing_proc == -1 ) 08855 { 08856 result = mbImpl->tag_delete_data( sharedp_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedp tag data" ); 08857 result = mbImpl->tag_delete_data( sharedh_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedh tag data" ); 08858 result = mbImpl->tag_delete_data( pstatus_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete pstatus tag data" ); 08859 } 08860 ++it; 08861 } 08862 } 08863 08864 return MB_SUCCESS; 08865 } 08866 08867 void ParallelComm::set_debug_verbosity( int verb ) 08868 { 08869 myDebug->set_verbosity( verb ); 08870 } 08871 08872 int ParallelComm::get_debug_verbosity() 08873 { 08874 return myDebug->get_verbosity(); 08875 } 08876 08877 ErrorCode ParallelComm::get_entityset_procs( EntityHandle set, std::vector< unsigned >& ranks ) const 08878 { 08879 return sharedSetData->get_sharing_procs( set, ranks ); 08880 } 08881 08882 ErrorCode ParallelComm::get_entityset_owner( EntityHandle entity_set, 08883 unsigned& owner_rank, 08884 EntityHandle* remote_handle ) const 08885 { 08886 if( remote_handle ) 08887 return sharedSetData->get_owner( entity_set, owner_rank, *remote_handle ); 08888 else 08889 return sharedSetData->get_owner( entity_set, owner_rank ); 08890 } 08891 08892 ErrorCode ParallelComm::get_entityset_local_handle( unsigned owning_rank, 08893 EntityHandle remote_handle, 08894 EntityHandle& local_handle ) const 08895 { 08896 return sharedSetData->get_local_handle( owning_rank, remote_handle, local_handle ); 08897 } 08898 08899 ErrorCode ParallelComm::get_shared_sets( Range& result ) const 08900 { 08901 return sharedSetData->get_shared_sets( result ); 08902 } 08903 08904 ErrorCode ParallelComm::get_entityset_owners( std::vector< unsigned >& ranks ) const 08905 { 08906 return sharedSetData->get_owning_procs( ranks ); 08907 } 08908 08909 ErrorCode ParallelComm::get_owned_sets( unsigned owning_rank, Range& sets_out ) const 08910 { 08911 return sharedSetData->get_shared_sets( owning_rank, sets_out ); 08912 } 08913 08914 ErrorCode ParallelComm::gather_data( Range& gather_ents, 08915 Tag& tag_handle, 08916 Tag id_tag, 08917 EntityHandle gather_set, 08918 int root_proc_rank ) 08919 { 08920 int dim = mbImpl->dimension_from_handle( *gather_ents.begin() ); 08921 int bytes_per_tag = 0; 08922 ErrorCode rval = mbImpl->tag_get_bytes( tag_handle, bytes_per_tag ); 08923 if( rval != MB_SUCCESS ) return rval; 08924 08925 int sz_buffer = sizeof( int ) + gather_ents.size() * ( sizeof( int ) + bytes_per_tag ); 08926 void* senddata = malloc( sz_buffer ); 08927 ( (int*)senddata )[0] = (int)gather_ents.size(); 08928 int* ptr_int = (int*)senddata + 1; 08929 rval = mbImpl->tag_get_data( id_tag, gather_ents, (void*)ptr_int ); 08930 if( rval != MB_SUCCESS ) return rval; 08931 ptr_int = (int*)( senddata ) + 1 + gather_ents.size(); 08932 rval = mbImpl->tag_get_data( tag_handle, gather_ents, (void*)ptr_int ); 08933 if( rval != MB_SUCCESS ) return rval; 08934 std::vector< int > displs( proc_config().proc_size(), 0 ); 08935 MPI_Gather( &sz_buffer, 1, MPI_INT, &displs[0], 1, MPI_INT, root_proc_rank, comm() ); 08936 std::vector< int > recvcnts( proc_config().proc_size(), 0 ); 08937 std::copy( displs.begin(), displs.end(), recvcnts.begin() ); 08938 std::partial_sum( displs.begin(), displs.end(), displs.begin() ); 08939 std::vector< int >::iterator lastM1 = displs.end() - 1; 08940 std::copy_backward( displs.begin(), lastM1, displs.end() ); 08941 // std::copy_backward(displs.begin(), --displs.end(), displs.end()); 08942 displs[0] = 0; 08943 08944 if( (int)rank() != root_proc_rank ) 08945 MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, NULL, NULL, NULL, MPI_BYTE, root_proc_rank, comm() ); 08946 else 08947 { 08948 Range gents; 08949 mbImpl->get_entities_by_dimension( gather_set, dim, gents ); 08950 int recvbuffsz = gents.size() * ( bytes_per_tag + sizeof( int ) ) + proc_config().proc_size() * sizeof( int ); 08951 void* recvbuf = malloc( recvbuffsz ); 08952 MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, recvbuf, &recvcnts[0], &displs[0], MPI_BYTE, root_proc_rank, 08953 comm() ); 08954 08955 void* gvals = NULL; 08956 08957 // Test whether gents has multiple sequences 08958 bool multiple_sequences = false; 08959 if( gents.psize() > 1 ) 08960 multiple_sequences = true; 08961 else 08962 { 08963 int count; 08964 rval = mbImpl->tag_iterate( tag_handle, gents.begin(), gents.end(), count, gvals ); 08965 assert( NULL != gvals ); 08966 assert( count > 0 ); 08967 if( (size_t)count != gents.size() ) 08968 { 08969 multiple_sequences = true; 08970 gvals = NULL; 08971 } 08972 } 08973 08974 // If gents has multiple sequences, create a temp buffer for gathered values 08975 if( multiple_sequences ) 08976 { 08977 gvals = malloc( gents.size() * bytes_per_tag ); 08978 assert( NULL != gvals ); 08979 } 08980 08981 for( int i = 0; i != (int)size(); i++ ) 08982 { 08983 int numents = *(int*)( ( (char*)recvbuf ) + displs[i] ); 08984 int* id_ptr = (int*)( ( (char*)recvbuf ) + displs[i] + sizeof( int ) ); 08985 char* val_ptr = (char*)( id_ptr + numents ); 08986 for( int j = 0; j != numents; j++ ) 08987 { 08988 int idx = id_ptr[j]; 08989 memcpy( (char*)gvals + ( idx - 1 ) * bytes_per_tag, val_ptr + j * bytes_per_tag, bytes_per_tag ); 08990 } 08991 } 08992 08993 // Free the receive buffer 08994 free( recvbuf ); 08995 08996 // If gents has multiple sequences, copy tag data (stored in the temp buffer) to each 08997 // sequence separately 08998 if( multiple_sequences ) 08999 { 09000 Range::iterator iter = gents.begin(); 09001 size_t start_idx = 0; 09002 while( iter != gents.end() ) 09003 { 09004 int count; 09005 void* ptr; 09006 rval = mbImpl->tag_iterate( tag_handle, iter, gents.end(), count, ptr ); 09007 assert( NULL != ptr ); 09008 assert( count > 0 ); 09009 memcpy( (char*)ptr, (char*)gvals + start_idx * bytes_per_tag, bytes_per_tag * count ); 09010 09011 iter += count; 09012 start_idx += count; 09013 } 09014 assert( start_idx == gents.size() ); 09015 09016 // Free the temp buffer 09017 free( gvals ); 09018 } 09019 } 09020 09021 // Free the send data 09022 free( senddata ); 09023 09024 return MB_SUCCESS; 09025 } 09026 09027 /* 09028 * This call is collective, so we will use the message ids for tag communications; 09029 * they are similar, but simpler 09030 * Pack the number of edges, the remote edge handles, then for each edge, the number 09031 * of intersection points, and then 3 doubles for each intersection point 09032 * On average, there is one intx point per edge, in some cases 2, in some cases 0 09033 * so on average, the message size is num_edges * (sizeof(eh) + sizeof(int) + 1*3*sizeof(double)) 09034 * = num_edges * (8 + 4 + 24) 09035 */ 09036 ErrorCode ParallelComm::settle_intersection_points( Range& edges, 09037 Range& shared_edges_owned, 09038 std::vector< std::vector< EntityHandle >* >& extraNodesVec, 09039 double tolerance ) 09040 { 09041 // The index of an edge in the edges Range will give the index for extraNodesVec 09042 // the strategy of this follows exchange tags strategy: 09043 ErrorCode result; 09044 int success; 09045 09046 myDebug->tprintf( 1, "Entering settle_intersection_points\n" ); 09047 09048 // Get all procs interfacing to this proc 09049 std::set< unsigned int > exch_procs; 09050 result = get_comm_procs( exch_procs ); 09051 09052 // Post ghost irecv's for all interface procs 09053 // Index requests the same as buffer/sharing procs indices 09054 std::vector< MPI_Request > recv_intx_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 09055 std::vector< unsigned int >::iterator sit; 09056 int ind; 09057 09058 reset_all_buffers(); 09059 int incoming = 0; 09060 09061 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 09062 { 09063 incoming++; 09064 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, 09065 MB_MESG_TAGS_SIZE, incoming ); 09066 09067 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit, 09068 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_intx_reqs[3 * ind] ); 09069 if( success != MPI_SUCCESS ) 09070 { 09071 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in settle intersection point" ); 09072 } 09073 } 09074 09075 // Pack and send intersection points from this proc to others 09076 // Make sendReqs vector to simplify initialization 09077 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL ); 09078 09079 // Take all shared entities if incoming list is empty 09080 Range& entities = shared_edges_owned; 09081 09082 int dum_ack_buff; 09083 09084 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ ) 09085 { 09086 Range edges_to_send = entities; 09087 09088 // Get ents shared by proc *sit 09089 result = filter_pstatus( edges_to_send, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" ); 09090 09091 // Remote nonowned entities; not needed, edges are already owned by this proc 09092 09093 // Pack the data 09094 // Reserve space on front for size and for initial buff size 09095 Buffer* buff = localOwnedBuffs[ind]; 09096 buff->reset_ptr( sizeof( int ) ); 09097 09098 /*result = pack_intx_points(edges_to_send, edges, extraNodesVec, 09099 localOwnedBuffs[ind], *sit);*/ 09100 09101 // Count first data, and see if it is enough room? 09102 // Send the remote handles 09103 std::vector< EntityHandle > dum_remote_edges( edges_to_send.size() ); 09104 /* 09105 * get_remote_handles(const bool store_remote_handles, 09106 EntityHandle *from_vec, 09107 EntityHandle *to_vec_tmp, 09108 int num_ents, int to_proc, 09109 const std::vector<EntityHandle> &new_ents); 09110 */ 09111 // We are sending count, num edges, remote edges handles, and then, for each edge: 09112 // -- nb intx points, 3*nbintPointsforEdge "doubles" 09113 std::vector< EntityHandle > dum_vec; 09114 result = get_remote_handles( true, edges_to_send, &dum_remote_edges[0], *sit, dum_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles" ); 09115 int count = 4; // Size of data 09116 count += sizeof( int ) * (int)edges_to_send.size(); 09117 count += sizeof( EntityHandle ) * (int)edges_to_send.size(); // We will send the remote handles 09118 for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit ) 09119 { 09120 EntityHandle edge = *eit; 09121 unsigned int indx = edges.find( edge ) - edges.begin(); 09122 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] ); 09123 count += (int)intx_nodes.size() * 3 * sizeof( double ); // 3 integer for each entity handle 09124 } 09125 // 09126 buff->check_space( count ); 09127 PACK_INT( buff->buff_ptr, edges_to_send.size() ); 09128 PACK_EH( buff->buff_ptr, &dum_remote_edges[0], dum_remote_edges.size() ); 09129 for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit ) 09130 { 09131 EntityHandle edge = *eit; 09132 // Pack the remote edge 09133 unsigned int indx = edges.find( edge ) - edges.begin(); 09134 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] ); 09135 PACK_INT( buff->buff_ptr, intx_nodes.size() ); 09136 09137 result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), (double*)buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get coords" ); 09138 buff->buff_ptr += 3 * sizeof( double ) * intx_nodes.size(); 09139 } 09140 09141 // Done packing the intx points and remote edges 09142 buff->set_stored_size(); 09143 09144 // Now send it 09145 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind], 09146 recv_intx_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" ); 09147 } 09148 09149 // Receive/unpack intx points 09150 while( incoming ) 09151 { 09152 MPI_Status status; 09153 int index_in_recv_requests; 09154 PRINT_DEBUG_WAITANY( recv_intx_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() ); 09155 success = MPI_Waitany( 3 * buffProcs.size(), &recv_intx_reqs[0], &index_in_recv_requests, &status ); 09156 if( MPI_SUCCESS != success ) 09157 { 09158 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" ); 09159 } 09160 // Processor index in the list is divided by 3 09161 ind = index_in_recv_requests / 3; 09162 09163 PRINT_DEBUG_RECD( status ); 09164 09165 // OK, received something; decrement incoming counter 09166 incoming--; 09167 09168 bool done = false; 09169 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind], 09170 recv_intx_reqs[3 * ind + 1], // This is for receiving the second message 09171 recv_intx_reqs[3 * ind + 2], // This would be for ack, but it is not 09172 // used; consider removing it 09173 incoming, localOwnedBuffs[ind], 09174 sendReqs[3 * ind + 1], // Send request for sending the second message 09175 sendReqs[3 * ind + 2], // This is for sending the ack 09176 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" ); 09177 if( done ) 09178 { 09179 Buffer* buff = remoteOwnedBuffs[ind]; 09180 buff->reset_ptr( sizeof( int ) ); 09181 /*result = unpack_tags(remoteOwnedBuffs[ind/2]->buff_ptr, dum_vec, true, 09182 buffProcs[ind/2]);*/ 09183 // Unpack now the edges and vertex info; compare with the existing vertex positions 09184 09185 int num_edges; 09186 09187 UNPACK_INT( buff->buff_ptr, num_edges ); 09188 std::vector< EntityHandle > rec_edges; 09189 rec_edges.resize( num_edges ); 09190 UNPACK_EH( buff->buff_ptr, &rec_edges[0], num_edges ); 09191 for( int i = 0; i < num_edges; i++ ) 09192 { 09193 EntityHandle edge = rec_edges[i]; 09194 unsigned int indx = edges.find( edge ) - edges.begin(); 09195 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] ); 09196 // Now get the number of nodes on this (now local) edge 09197 int nverts; 09198 UNPACK_INT( buff->buff_ptr, nverts ); 09199 std::vector< double > pos_from_owner; 09200 pos_from_owner.resize( 3 * nverts ); 09201 UNPACK_DBLS( buff->buff_ptr, &pos_from_owner[0], 3 * nverts ); 09202 std::vector< double > current_positions( 3 * intx_nodes.size() ); 09203 result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), ¤t_positions[0] );MB_CHK_SET_ERR( result, "Failed to get current positions" ); 09204 // Now, look at what we have in current pos, compare to pos from owner, and reset 09205 for( int k = 0; k < (int)intx_nodes.size(); k++ ) 09206 { 09207 double* pk = ¤t_positions[3 * k]; 09208 // Take the current pos k, and settle among the ones from owner: 09209 bool found = false; 09210 for( int j = 0; j < nverts && !found; j++ ) 09211 { 09212 double* pj = &pos_from_owner[3 * j]; 09213 double dist2 = ( pk[0] - pj[0] ) * ( pk[0] - pj[0] ) + ( pk[1] - pj[1] ) * ( pk[1] - pj[1] ) + 09214 ( pk[2] - pj[2] ) * ( pk[2] - pj[2] ); 09215 if( dist2 < tolerance ) 09216 { 09217 pk[0] = pj[0]; 09218 pk[1] = pj[1]; 09219 pk[2] = pj[2]; // Correct it! 09220 found = true; 09221 break; 09222 } 09223 } 09224 if( !found ) 09225 { 09226 #ifndef NDEBUG 09227 std::cout << " pk:" << pk[0] << " " << pk[1] << " " << pk[2] << " not found \n"; 09228 #endif 09229 result = MB_FAILURE; 09230 } 09231 } 09232 // After we are done resetting, we can set the new positions of nodes: 09233 result = mbImpl->set_coords( &intx_nodes[0], (int)intx_nodes.size(), ¤t_positions[0] );MB_CHK_SET_ERR( result, "Failed to set new current positions" ); 09234 } 09235 } 09236 } 09237 09238 // OK, now wait 09239 if( myDebug->get_verbosity() == 5 ) 09240 { 09241 success = MPI_Barrier( procConfig.proc_comm() ); 09242 } 09243 else 09244 { 09245 MPI_Status status[3 * MAX_SHARING_PROCS]; 09246 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status ); 09247 } 09248 if( MPI_SUCCESS != success ) 09249 { 09250 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" ); 09251 } 09252 09253 myDebug->tprintf( 1, "Exiting settle_intersection_points" ); 09254 09255 return MB_SUCCESS; 09256 } 09257 09258 ErrorCode ParallelComm::delete_entities( Range& to_delete ) 09259 { 09260 // Will not look at shared sets yet, but maybe we should 09261 // First, see if any of the entities to delete is shared; then inform the other processors 09262 // about their fate (to be deleted), using a crystal router transfer 09263 ErrorCode rval = MB_SUCCESS; 09264 unsigned char pstat; 09265 EntityHandle tmp_handles[MAX_SHARING_PROCS]; 09266 int tmp_procs[MAX_SHARING_PROCS]; 09267 unsigned int num_ps; 09268 TupleList ents_to_delete; 09269 ents_to_delete.initialize( 1, 0, 1, 0, to_delete.size() * ( MAX_SHARING_PROCS + 1 ) ); // A little bit of overkill 09270 ents_to_delete.enableWriteAccess(); 09271 unsigned int i = 0; 09272 for( Range::iterator it = to_delete.begin(); it != to_delete.end(); ++it ) 09273 { 09274 EntityHandle eh = *it; // Entity to be deleted 09275 09276 rval = get_sharing_data( eh, tmp_procs, tmp_handles, pstat, num_ps ); 09277 if( rval != MB_SUCCESS || num_ps == 0 ) continue; 09278 // Add to the tuple list the information to be sent (to the remote procs) 09279 for( unsigned int p = 0; p < num_ps; p++ ) 09280 { 09281 ents_to_delete.vi_wr[i] = tmp_procs[p]; 09282 ents_to_delete.vul_wr[i] = (unsigned long)tmp_handles[p]; 09283 i++; 09284 ents_to_delete.inc_n(); 09285 } 09286 } 09287 09288 gs_data::crystal_data* cd = this->procConfig.crystal_router(); 09289 // All communication happens here; no other mpi calls 09290 // Also, this is a collective call 09291 rval = cd->gs_transfer( 1, ents_to_delete, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" ); 09292 09293 // Add to the range of ents to delete the new ones that were sent from other procs 09294 unsigned int received = ents_to_delete.get_n(); 09295 for( i = 0; i < received; i++ ) 09296 { 09297 // int from = ents_to_delete.vi_rd[i]; 09298 unsigned long valrec = ents_to_delete.vul_rd[i]; 09299 to_delete.insert( (EntityHandle)valrec ); 09300 } 09301 rval = mbImpl->delete_entities( to_delete );MB_CHK_SET_ERR( rval, "Error in deleting actual entities" ); 09302 09303 std::set< EntityHandle > good_ents; 09304 for( std::set< EntityHandle >::iterator sst = sharedEnts.begin(); sst != sharedEnts.end(); sst++ ) 09305 { 09306 EntityHandle eh = *sst; 09307 int index = to_delete.index( eh ); 09308 if( -1 == index ) good_ents.insert( eh ); 09309 } 09310 sharedEnts = good_ents; 09311 09312 // What about shared sets? Who is updating them? 09313 return MB_SUCCESS; 09314 } 09315 09316 void ParallelComm::print_pstatus( unsigned char pstat, std::string& ostr ) 09317 { 09318 std::ostringstream str; 09319 int num = 0; 09320 #define ppstat( a, b ) \ 09321 { \ 09322 if( pstat & ( a ) ) \ 09323 { \ 09324 if( num ) str << ", "; \ 09325 str << ( b ); \ 09326 num++; \ 09327 } \ 09328 } 09329 09330 ppstat( PSTATUS_NOT_OWNED, "NOT_OWNED" ); 09331 ppstat( PSTATUS_SHARED, "SHARED" ); 09332 ppstat( PSTATUS_MULTISHARED, "MULTISHARED" ); 09333 ppstat( PSTATUS_INTERFACE, "INTERFACE" ); 09334 ppstat( PSTATUS_GHOST, "GHOST" ); 09335 09336 ostr = str.str(); 09337 } 09338 09339 void ParallelComm::print_pstatus( unsigned char pstat ) 09340 { 09341 std::string str; 09342 print_pstatus( pstat, str ); 09343 std::cout << str.c_str() << std::endl; 09344 } 09345 09346 ErrorCode ParallelComm::correct_thin_ghost_layers() 09347 { 09348 09349 // Get all shared ent data from other procs 09350 std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() ); 09351 09352 // will work only on multi-shared tags sharedps_tag(), sharedhs_tag(); 09353 09354 /* 09355 * domain0 | domain1 | domain2 | domain3 09356 * vertices from domain 1 and 2 are visible from both 0 and 3, but 09357 * domain 0 might not have info about multi-sharing from domain 3 09358 * so we will force that domain 0 vertices owned by 1 and 2 have information 09359 * about the domain 3 sharing 09360 * 09361 * SharedEntityData will have : 09362 * struct SharedEntityData { 09363 EntityHandle local; // this is same meaning, for the proc we sent to, it is local 09364 EntityHandle remote; // this will be the far away handle that will need to be added 09365 EntityID owner; // this will be the remote proc 09366 }; 09367 // so we need to add data like this: 09368 a multishared entity owned by proc x will have data like 09369 multishared procs: proc x, a, b, c 09370 multishared handles: h1, h2, h3, h4 09371 we will need to send data from proc x like this: 09372 to proc a we will send 09373 (h2, h3, b), (h2, h4, c) 09374 to proc b we will send 09375 (h3, h2, a), (h3, h4, c) 09376 to proc c we will send 09377 (h4, h2, a), (h4, h3, b) 09378 * 09379 */ 09380 09381 ErrorCode result = MB_SUCCESS; 09382 int ent_procs[MAX_SHARING_PROCS + 1]; 09383 EntityHandle handles[MAX_SHARING_PROCS + 1]; 09384 int num_sharing; 09385 SharedEntityData tmp; 09386 09387 for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i ) 09388 { 09389 09390 unsigned char pstat; 09391 result = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" ); 09392 if( !( pstat & PSTATUS_MULTISHARED ) || 09393 num_sharing <= 2 ) // if not multishared, skip, it should have no problems 09394 continue; 09395 // we should skip the ones that are not owned locally 09396 // the owned ones will have the most multi-shared info, because the info comes from other 09397 // remote processors 09398 if( pstat & PSTATUS_NOT_OWNED ) continue; 09399 for( int j = 1; j < num_sharing; j++ ) 09400 { 09401 // we will send to proc 09402 int send_to_proc = ent_procs[j]; // 09403 tmp.local = handles[j]; 09404 int ind = get_buffers( send_to_proc ); 09405 assert( -1 != ind ); // THIS SHOULD NEVER HAPPEN 09406 for( int k = 1; k < num_sharing; k++ ) 09407 { 09408 // do not send to self proc 09409 if( j == k ) continue; 09410 tmp.remote = handles[k]; // this will be the handle of entity on proc 09411 tmp.owner = ent_procs[k]; 09412 send_data[ind].push_back( tmp ); 09413 } 09414 } 09415 } 09416 09417 result = exchange_all_shared_handles( send_data, shents );MB_CHK_ERR( result ); 09418 09419 // loop over all shents and add if vertex type, add if missing 09420 for( size_t i = 0; i < shents.size(); i++ ) 09421 { 09422 std::vector< SharedEntityData >& shEnts = shents[i]; 09423 for( size_t j = 0; j < shEnts.size(); j++ ) 09424 { 09425 tmp = shEnts[j]; 09426 // basically, check the shared data for tmp.local entity 09427 // it should have inside the tmp.owner and tmp.remote 09428 EntityHandle eh = tmp.local; 09429 unsigned char pstat; 09430 result = get_sharing_data( eh, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" ); 09431 // see if the proc tmp.owner is in the list of ent_procs; if not, we have to increase 09432 // handles, and ent_procs; and set 09433 09434 int proc_remote = tmp.owner; // 09435 if( std::find( ent_procs, ent_procs + num_sharing, proc_remote ) == ent_procs + num_sharing ) 09436 { 09437 // so we did not find on proc 09438 #ifndef NDEBUG 09439 std::cout << "THIN GHOST: we did not find on proc " << rank() << " for shared ent " << eh 09440 << " the proc " << proc_remote << "\n"; 09441 #endif 09442 // increase num_sharing, and set the multi-shared tags 09443 if( num_sharing >= MAX_SHARING_PROCS ) return MB_FAILURE; 09444 handles[num_sharing] = tmp.remote; 09445 handles[num_sharing + 1] = 0; // end of list 09446 ent_procs[num_sharing] = tmp.owner; 09447 ent_procs[num_sharing + 1] = -1; // this should be already set 09448 result = mbImpl->tag_set_data( sharedps_tag(), &eh, 1, ent_procs );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" ); 09449 result = mbImpl->tag_set_data( sharedhs_tag(), &eh, 1, handles );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" ); 09450 if( 2 == num_sharing ) // it means the sharedp and sharedh tags were set with a 09451 // value non default 09452 { 09453 // so entity eh was simple shared before, we need to set those dense tags back 09454 // to default 09455 // values 09456 EntityHandle zero = 0; 09457 int no_proc = -1; 09458 result = mbImpl->tag_set_data( sharedp_tag(), &eh, 1, &no_proc );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" ); 09459 result = mbImpl->tag_set_data( sharedh_tag(), &eh, 1, &zero );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" ); 09460 // also, add multishared pstatus tag 09461 // also add multishared status to pstatus 09462 pstat = pstat | PSTATUS_MULTISHARED; 09463 result = mbImpl->tag_set_data( pstatus_tag(), &eh, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" ); 09464 } 09465 } 09466 } 09467 } 09468 return MB_SUCCESS; 09469 } 09470 } // namespace moab