|
MOAB: Mesh Oriented datABase
(version 5.4.1)
|
|
MOAB: Mesh Oriented datABase
(version 5.4.1)
|
#include "moab/Core.hpp"#include "moab_mpi.h"#include "moab/ParallelComm.hpp"#include "MBParallelConventions.h"#include "moab/iMOAB.h"#include "TestUtil.hpp"#include "moab/CpuTimer.hpp"#include "moab/ProgOptions.hpp"#include <iostream>#include <sstream>
Include dependency graph for commgraph_test.cpp:Go to the source code of this file.
Defines | |
| #define | CHECKIERR(rc, message) |
Functions | |
| std::string | readopts ("PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS") |
| std::string | readoptsPC ("PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION") |
| std::string | fileWriteOptions ("PARALLEL=WRITE_PART") |
| std::string | tagT ("a2oTbot") |
| std::string | tagU ("a2oUbot") |
| std::string | tagV ("a2oVbot") |
| std::string | separ (":") |
| std::string | tagT1 ("a2oTbot_1") |
| std::string | tagU1 ("a2oUbot_1") |
| std::string | tagV1 ("a2oVbot_1") |
| std::string | tagT2 ("a2oTbot_2") |
| int | commgraphtest () |
| void | testspectral_phys () |
| void | testspectral_lnd () |
| void | testphysatm_lnd () |
| int | main (int argc, char *argv[]) |
Variables | |
| int | ierr |
| int | rankInGlobalComm |
| int | numProcesses |
| MPI_Group | jgroup |
| std::string | atmFilename = "unittest/wholeATM_T.h5m" |
| int | rankInAtmComm = -1 |
| int | cmpatm = 605 |
| std::string | atmPhysFilename = "unittest/AtmPhys.h5m" |
| std::string | atmPhysOutFilename = "outPhys.h5m" |
| std::string | atmFilename2 = "wholeATM_new.h5m" |
| int | rankInPhysComm = -1 |
| int | physatm = 5 |
| int | nghlay = 0 |
| std::vector< int > | groupTasks |
| int | startG1 = 0 |
| int | startG2 = 0 |
| int | endG1 = numProcesses - 1 |
| int | endG2 = numProcesses - 1 |
| int | typeA = 1 |
| int | typeB = 2 |
| #define CHECKIERR | ( | rc, | |
| message | |||
| ) |
if( 0 != ( rc ) ) \ { \ printf( "%s. ErrorCode = %d.\n", message, rc ); \ CHECK( 0 ); \ }
Definition at line 50 of file commgraph_test.cpp.
Referenced by commgraphtest().
| int commgraphtest | ( | ) |
Definition at line 136 of file commgraph_test.cpp.
References atmFilename, atmFilename2, atmPhysFilename, atmPhysOutFilename, CHECKIERR, cmpatm, DENSE_DOUBLE, endG1, endG2, fileWriteOptions(), groupTasks, ierr, iMOAB_AppID, iMOAB_DefineTagStorage(), iMOAB_DeregisterApplication(), iMOAB_Finalize(), iMOAB_Initialize(), iMOAB_LoadMesh(), iMOAB_RegisterApplication(), iMOAB_WriteMesh(), jgroup, MPI_COMM_WORLD, nghlay, numProcesses, physatm, rankInAtmComm, rankInGlobalComm, rankInPhysComm, readopts(), readoptsPC(), separ(), startG1, startG2, tagT(), tagT1(), tagT2(), tagU(), tagU1(), tagV(), tagV1(), typeA, and typeB.
Referenced by testphysatm_lnd(), testspectral_lnd(), and testspectral_phys().
{
if( !rankInGlobalComm )
{
std::cout << " first file: " << atmFilename << "\n on tasks : " << startG1 << ":" << endG1
<< "\n second file: " << atmPhysFilename << "\n on tasks : " << startG2 << ":" << endG2 << "\n ";
}
// load files on 2 different communicators, groups
// will create a joint comm for rendezvous
MPI_Group atmPEGroup;
groupTasks.resize( numProcesses, 0 );
for( int i = startG1; i <= endG1; i++ )
groupTasks[i - startG1] = i;
ierr = MPI_Group_incl( jgroup, endG1 - startG1 + 1, &groupTasks[0], &atmPEGroup );
CHECKIERR( ierr, "Cannot create atmPEGroup" )
groupTasks.clear();
groupTasks.resize( numProcesses, 0 );
MPI_Group atmPhysGroup;
for( int i = startG2; i <= endG2; i++ )
groupTasks[i - startG2] = i;
ierr = MPI_Group_incl( jgroup, endG2 - startG2 + 1, &groupTasks[0], &atmPhysGroup );
CHECKIERR( ierr, "Cannot create atmPhysGroup" )
// create 2 communicators, one for each group
int ATM_COMM_TAG = 1;
MPI_Comm atmComm;
// atmComm is for atmosphere app;
ierr = MPI_Comm_create_group( MPI_COMM_WORLD, atmPEGroup, ATM_COMM_TAG, &atmComm );
CHECKIERR( ierr, "Cannot create atmComm" )
int PHYS_COMM_TAG = 2;
MPI_Comm physComm;
// physComm is for phys atm app
ierr = MPI_Comm_create_group( MPI_COMM_WORLD, atmPhysGroup, PHYS_COMM_TAG, &physComm );
CHECKIERR( ierr, "Cannot create physComm" )
// now, create the joint communicator atm physatm
//
MPI_Group joinAtmPhysAtmGroup;
ierr = MPI_Group_union( atmPEGroup, atmPhysGroup, &joinAtmPhysAtmGroup );
CHECKIERR( ierr, "Cannot create joint atm - phys atm group" )
int JOIN_COMM_TAG = 5;
MPI_Comm joinComm;
ierr = MPI_Comm_create_group( MPI_COMM_WORLD, joinAtmPhysAtmGroup, JOIN_COMM_TAG, &joinComm );
CHECKIERR( ierr, "Cannot create joint atm cou communicator" )
ierr = iMOAB_Initialize( 0, NULL ); // not really needed anything from argc, argv, yet; maybe we should
CHECKIERR( ierr, "Cannot initialize iMOAB" )
int cmpAtmAppID = -1;
iMOAB_AppID cmpAtmPID = &cmpAtmAppID; // atm
int physAtmAppID = -1; // -1 means it is not initialized
iMOAB_AppID physAtmPID = &physAtmAppID; // phys atm on phys pes
// load atm mesh
if( atmComm != MPI_COMM_NULL )
{
MPI_Comm_rank( atmComm, &rankInAtmComm );
ierr = iMOAB_RegisterApplication( "ATM1", &atmComm, &cmpatm, cmpAtmPID );
CHECKIERR( ierr, "Cannot register ATM App" )
// load first model
std::string rdopts = readopts;
if( typeA == 2 ) rdopts = readoptsPC; // point cloud
ierr = iMOAB_LoadMesh( cmpAtmPID, atmFilename.c_str(), rdopts.c_str(), &nghlay );
CHECKIERR( ierr, "Cannot load ATM mesh" )
}
// load atm phys mesh
if( physComm != MPI_COMM_NULL )
{
MPI_Comm_rank( physComm, &rankInPhysComm );
ierr = iMOAB_RegisterApplication( "PhysATM", &physComm, &physatm, physAtmPID );
CHECKIERR( ierr, "Cannot register PHYS ATM App" )
// load phys atm mesh all tests this is PC
ierr = iMOAB_LoadMesh( physAtmPID, atmPhysFilename.c_str(), readoptsPC.c_str(), &nghlay );
CHECKIERR( ierr, "Cannot load Phys ATM mesh" )
}
if( MPI_COMM_NULL != joinComm )
{
ierr = iMOAB_ComputeCommGraph( cmpAtmPID, physAtmPID, &joinComm, &atmPEGroup, &atmPhysGroup, &typeA, &typeB,
&cmpatm, &physatm );
// it will generate parcomm graph between atm and atmPhys models
// 2 meshes, that are distributed in parallel
CHECKIERR( ierr, "Cannot compute comm graph between the two apps " )
}
if( atmComm != MPI_COMM_NULL )
{
// call send tag;
std::string tags = tagT + separ + tagU + separ + tagV;
ierr = iMOAB_SendElementTag( cmpAtmPID, tags.c_str(), &joinComm, &physatm );
CHECKIERR( ierr, "cannot send tag values" )
}
if( physComm != MPI_COMM_NULL )
{
// need to define tag storage
std::string tags1 = tagT1 + separ + tagU1 + separ + tagV1 + separ;
int tagType = DENSE_DOUBLE;
int ndof = 1;
if( typeB == 1 ) ndof = 16;
int tagIndex = 0;
ierr = iMOAB_DefineTagStorage( physAtmPID, tagT1.c_str(), &tagType, &ndof, &tagIndex );
CHECKIERR( ierr, "failed to define the field tag a2oTbot" );
ierr = iMOAB_DefineTagStorage( physAtmPID, tagU1.c_str(), &tagType, &ndof, &tagIndex );
CHECKIERR( ierr, "failed to define the field tag a2oUbot" );
ierr = iMOAB_DefineTagStorage( physAtmPID, tagV1.c_str(), &tagType, &ndof, &tagIndex );
CHECKIERR( ierr, "failed to define the field tag a2oVbot" );
ierr = iMOAB_ReceiveElementTag( physAtmPID, tags1.c_str(), &joinComm, &cmpatm );
CHECKIERR( ierr, "cannot receive tag values" )
}
// we can now free the sender buffers
if( atmComm != MPI_COMM_NULL )
{
ierr = iMOAB_FreeSenderBuffers( cmpAtmPID, &physatm );
CHECKIERR( ierr, "cannot free buffers" )
}
if( physComm != MPI_COMM_NULL )
{
ierr = iMOAB_WriteMesh( physAtmPID, atmPhysOutFilename.c_str(), fileWriteOptions.c_str() );
}
if( physComm != MPI_COMM_NULL )
{
// send back first tag only
ierr = iMOAB_SendElementTag( physAtmPID, tagT1.c_str(), &joinComm, &cmpatm );
CHECKIERR( ierr, "cannot send tag values" )
}
// receive it in a different tag
if( atmComm != MPI_COMM_NULL )
{
// need to define tag storage
int tagType = DENSE_DOUBLE;
int ndof = 16;
if( typeA == 2 ) ndof = 1;
int tagIndex = 0;
ierr = iMOAB_DefineTagStorage( cmpAtmPID, tagT2.c_str(), &tagType, &ndof, &tagIndex );
CHECKIERR( ierr, "failed to define the field tag a2oTbot_2" );
ierr = iMOAB_ReceiveElementTag( cmpAtmPID, tagT2.c_str(), &joinComm, &physatm );
CHECKIERR( ierr, "cannot receive tag values a2oTbot_2" )
}
// now send back one tag , into a different tag, and see if we get the same values back
// we can now free the sender buffers
if( physComm != MPI_COMM_NULL )
{
ierr = iMOAB_FreeSenderBuffers( physAtmPID, &cmpatm );
CHECKIERR( ierr, "cannot free buffers " )
}
if( atmComm != MPI_COMM_NULL )
{
ierr = iMOAB_WriteMesh( cmpAtmPID, atmFilename2.c_str(), fileWriteOptions.c_str() );
}
// unregister in reverse order
if( physComm != MPI_COMM_NULL )
{
ierr = iMOAB_DeregisterApplication( physAtmPID );
CHECKIERR( ierr, "cannot deregister second app model" )
}
if( atmComm != MPI_COMM_NULL )
{
ierr = iMOAB_DeregisterApplication( cmpAtmPID );
CHECKIERR( ierr, "cannot deregister first app model" )
}
ierr = iMOAB_Finalize();
CHECKIERR( ierr, "did not finalize iMOAB" )
// free atm group and comm
if( MPI_COMM_NULL != atmComm ) MPI_Comm_free( &atmComm );
MPI_Group_free( &atmPEGroup );
// free atm phys group and comm
if( MPI_COMM_NULL != physComm ) MPI_Comm_free( &physComm );
MPI_Group_free( &atmPhysGroup );
// free atm phys group and comm
if( MPI_COMM_NULL != joinComm ) MPI_Comm_free( &joinComm );
MPI_Group_free( &joinAtmPhysAtmGroup );
return 0;
}
| std::string fileWriteOptions | ( | ) |
Referenced by commgraphtest(), and main().
| int main | ( | int | argc, |
| char * | argv[] | ||
| ) |
Definition at line 333 of file commgraph_test.cpp.
References ProgOptions::addOpt(), atmFilename, atmFilename2, atmPhysFilename, atmPhysOutFilename, endG1, endG2, jgroup, MPI_COMM_WORLD, numProcesses, ProgOptions::parseCommandLine(), rankInGlobalComm, RUN_TEST, startG1, startG2, testphysatm_lnd(), testspectral_lnd(), testspectral_phys(), typeA, and typeB.
{
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rankInGlobalComm );
MPI_Comm_size( MPI_COMM_WORLD, &numProcesses );
MPI_Comm_group( MPI_COMM_WORLD, &jgroup ); // all processes in global group
// default: load atm on 2 proc, phys grid on 2 procs, establish comm graph, then migrate
// data from atm pes to phys pes, and back
startG1 = 0, startG2 = 0, endG1 = numProcesses - 1, endG2 = numProcesses - 1;
ProgOptions opts;
opts.addOpt< std::string >( "modelA,m", "first model file ", &atmFilename );
opts.addOpt< int >( "typeA,t", " type of first model ", &typeA );
opts.addOpt< std::string >( "modelB,n", "second model file", &atmPhysFilename );
opts.addOpt< int >( "typeB,v", " type of the second model ", &typeB );
opts.addOpt< int >( "startAtm,a", "start task for first model layout", &startG1 );
opts.addOpt< int >( "endAtm,b", "end task for first model layout", &endG1 );
opts.addOpt< int >( "startPhys,c", "start task for second model layout", &startG2 );
opts.addOpt< int >( "endPhys,d", "end task for second model layout", &endG2 );
opts.addOpt< std::string >( "output,o", "output filename", &atmPhysOutFilename );
opts.addOpt< std::string >( "output,o", "output filename", &atmFilename2 );
opts.parseCommandLine( argc, argv );
int num_err = 0;
num_err += RUN_TEST( testspectral_phys );
//
if( argc == 1 )
{
num_err += RUN_TEST( testspectral_lnd );
num_err += RUN_TEST( testphysatm_lnd );
}
MPI_Group_free( &jgroup );
MPI_Finalize();
return num_err;
}
| std::string readopts | ( | ) |
Referenced by commgraphtest(), moab::Remapper::LoadNativeMesh(), main(), migrate(), and migrate_smart().
| std::string readoptsPC | ( | ) |
Referenced by commgraphtest().
| std::string separ | ( | ":" | ) |
Referenced by commgraphtest().
| std::string tagT | ( | "a2oTbot" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagT1 | ( | "a2oTbot_1" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagT2 | ( | "a2oTbot_2" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagU | ( | "a2oUbot" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagU1 | ( | "a2oUbot_1" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagV | ( | "a2oVbot" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| std::string tagV1 | ( | "a2oVbot_1" | ) |
Referenced by commgraphtest(), and testphysatm_lnd().
| void testphysatm_lnd | ( | ) |
Definition at line 121 of file commgraph_test.cpp.
References atmFilename, atmFilename2, atmPhysFilename, atmPhysOutFilename, commgraphtest(), tagT(), tagT1(), tagT2(), tagU(), tagU1(), tagV(), tagV1(), and typeA.
Referenced by main().
{
// use for first file the output "outPhys.h5m" from first test
atmFilename = std::string( "outPhys.h5m" );
atmPhysFilename = std::string( "outLnd.h5m" );
atmPhysOutFilename = std::string( "physAtm_lnd.h5m" );
atmFilename2 = std::string( "physBack_lnd.h5m" );
tagT = tagT1;
tagU = tagU1;
tagV = tagV1;
tagT1 = std::string( "newT" );
tagT2 = std::string( "newT2" );
typeA = 2;
commgraphtest();
}
| void testspectral_lnd | ( | ) |
Definition at line 112 of file commgraph_test.cpp.
References atmFilename2, atmPhysFilename, atmPhysOutFilename, and commgraphtest().
Referenced by main().
{
// first model is spectral, second is land
atmPhysFilename = TestDir + "unittest/wholeLnd.h5m";
atmPhysOutFilename = std::string( "outLnd.h5m" );
atmFilename2 = std::string( "wholeATM_lnd.h5m" );
commgraphtest();
}
| void testspectral_phys | ( | ) |
Definition at line 106 of file commgraph_test.cpp.
References commgraphtest().
Referenced by main().
{
// no changes
commgraphtest();
}
| std::string atmFilename = "unittest/wholeATM_T.h5m" |
Definition at line 66 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), and testphysatm_lnd().
| std::string atmFilename2 = "wholeATM_new.h5m" |
Definition at line 78 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), testphysatm_lnd(), and testspectral_lnd().
| std::string atmPhysFilename = "unittest/AtmPhys.h5m" |
Definition at line 76 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), testphysatm_lnd(), and testspectral_lnd().
| std::string atmPhysOutFilename = "outPhys.h5m" |
Definition at line 77 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), testphysatm_lnd(), and testspectral_lnd().
| int cmpatm = 605 |
Definition at line 74 of file commgraph_test.cpp.
Referenced by commgraphtest(), and main().
| int endG1 = numProcesses - 1 |
Definition at line 88 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), migrate_1_1(), migrate_1_2(), migrate_2_1(), migrate_2_2(), migrate_2_4(), migrate_4_2(), migrate_4_3(), migrate_overlap(), and migrate_smart().
| int endG2 = numProcesses - 1 |
Definition at line 88 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), migrate_1_1(), migrate_1_2(), migrate_2_1(), migrate_2_2(), migrate_2_4(), migrate_4_2(), migrate_4_3(), migrate_overlap(), and migrate_smart().
| std::vector< int > groupTasks |
Definition at line 87 of file commgraph_test.cpp.
Referenced by commgraphtest(), create_group_and_comm(), main(), migrate(), and migrate_smart().
| int ierr |
Definition at line 63 of file commgraph_test.cpp.
Referenced by PartMap::build_map(), check_consistent_ids(), check_one_layer(), commgraphtest(), create_group_and_comm(), create_joint_comm_group(), create_mesh(), create_mesh_in_memory(), moab::Coupler::do_normalization(), moab::ParallelComm::exchange_all_shared_handles(), moab::ReadHDF5::find_sets_containing(), get_adj(), get_boundary_vertices(), get_coords(), get_imesh_mesh(), get_local_parts(), moab::Coupler::get_matching_entities(), get_num_adj_all(), get_num_adj_quads(), get_part_boundary_verts(), get_part_quads_and_verts(), iMeshP_getNumOfTopoAll(), iMeshP_getNumOfTypeAll(), interface_verts(), moab::TempestOnlineMap::IsConservative(), moab::TempestOnlineMap::IsConsistent(), moab::TempestOnlineMap::IsMonotone(), main(), migrate(), migrate_smart(), PartMap::part_from_coords(), moab::ReadHDF5::read_all_set_meta(), moab::ParCommGraph::receive_comm_graph(), moab::ParCommGraph::receive_mesh(), moab::ParCommGraph::receive_tag_values(), moab::ParCommGraph::release_send_buffers(), run_global_smoother(), run_local_smoother(), run_local_smoother2(), run_quality_optimizer(), run_solution_mesh_optimizer(), run_test(), moab::ParCommGraph::send_graph(), moab::ParCommGraph::send_graph_partition(), moab::ParCommGraph::send_mesh_parts(), moab::ParCommGraph::send_tag_values(), moab::TempestOnlineMap::SetDOFmapTags(), setup_component_coupler_meshes(), test_create_ghost_ents(), test_entity_copies(), test_entity_copy_parts(), test_entity_owner(), test_entity_status(), test_exchange_ents(), test_get_by_topo(), test_get_by_type(), test_get_by_type_topo_all(), test_get_by_type_topo_local(), test_get_neighbors(), test_get_part_boundary(), test_get_partitions(), test_get_parts(), test_interface_owners_common(), test_load(), test_part_boundary_iter(), test_part_id_handle(), test_part_rank(), test_push_tag_data_common(), test_tempest_cs_create(), test_tempest_ico_create(), test_tempest_mpas_create(), test_tempest_overlap_combinations(), test_tempest_rll_create(), vertex_tag(), and moab::TempestOnlineMap::WriteSCRIPMapFile().
| MPI_Group jgroup |
Definition at line 65 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), and migrate_smart().
| int nghlay = 0 |
Definition at line 85 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), migrate_smart(), and setup_component_coupler_meshes().
| int numProcesses |
Definition at line 64 of file commgraph_test.cpp.
Referenced by commgraphtest(), and main().
| int physatm = 5 |
Definition at line 81 of file commgraph_test.cpp.
Referenced by commgraphtest().
| int rankInAtmComm = -1 |
Definition at line 72 of file commgraph_test.cpp.
Referenced by commgraphtest(), and main().
| int rankInGlobalComm |
Definition at line 64 of file commgraph_test.cpp.
Referenced by commgraphtest(), and main().
| int rankInPhysComm = -1 |
Definition at line 79 of file commgraph_test.cpp.
Referenced by commgraphtest().
| int startG1 = 0 |
Definition at line 88 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), migrate_1_1(), migrate_1_2(), migrate_2_1(), migrate_2_2(), migrate_2_4(), migrate_4_2(), migrate_4_3(), migrate_overlap(), and migrate_smart().
| int startG2 = 0 |
Definition at line 88 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), migrate(), migrate_1_1(), migrate_1_2(), migrate_2_1(), migrate_2_2(), migrate_2_4(), migrate_4_2(), migrate_4_3(), migrate_overlap(), and migrate_smart().
| int typeA = 1 |
Definition at line 89 of file commgraph_test.cpp.
Referenced by commgraphtest(), main(), and testphysatm_lnd().
| int typeB = 2 |
Definition at line 90 of file commgraph_test.cpp.
Referenced by commgraphtest(), and main().
1.7.6.1