ReduceExchangeTags.cpp

Go to the documentation of this file.

/** @example ReduceExchangeTags.cpp
 * \brief Example program that shows the use case for performing tag data exchange
 * between parallel processors in order to sync data on shared entities. The reduction
 * operation on tag data is also shown where the user can perform any of the actions supported
 * by MPI_Op on data residing on shared entities. \n
 *
 * This example :
 *    -# Initialize MPI and instantiate MOAB
 *    -# Get user options: Input mesh file name, tag name (default: USERTAG), tag value
 * (default: 1.0)
 *    -# Create the root and partition sets
 *    -# Instantiate ParallelComm and read the mesh file in parallel using appropriate options
 *    -# Create two tags: USERTAG_EXC (exchange) and USERTAG_RED (reduction)
 *    -# Set tag data and exchange shared entity information between processors
 *      -# Get entities in all dimensions and set local (current rank, dimension) dependent data for
 *     exchange tag (USERTAG_EXC)
 *      -# Perform exchange of tag data so that data on shared entities are synced via
 * ParallelCommunicator.
 *    -#  Set tag data and reduce shared entity information between processors using MPI_SUM
 *      -#  Get higher dimensional entities in the current partition and set local (current rank)
 *     dependent data for reduce tag (USERTAG_EXC)
 *      -#  Perform the reduction operation (MPI_SUM) on shared entities via ParallelCommunicator.
 *    -#  Destroy the MOAB instance and finalize MPI
 *
 * To run: \n mpiexec -n 2 ./ReduceExchangeTags    \n
 * Example: \n mpiexec -n 2 ./ReduceExchangeTags ../MeshFiles/unittest/64bricks_1khex.h5m
 * USERTAG 100 \n
 *
 */

#include "moab/Core.hpp"
#ifdef MOAB_HAVE_MPI
#include "moab/ParallelComm.hpp"
#endif
#include "MBParallelConventions.h"
#include 
#include 
#include 

using namespace moab;
using namespace std;

// Error routines for use with MPI API
#define MPICHKERR( CODE, MSG )       \
    do                               \
    {                                \
        if( 0 != ( CODE ) )          \
        {                            \
            cerr << ( MSG ) << endl; \
            MPI_Finalize();          \
        }                            \
    } while( false )

#define dbgprint( MSG )                      \
    do                                       \
    {                                        \
        if( !rank ) cerr << ( MSG ) << endl; \
    } while( false )

#define dbgprintall( MSG )                               \
    do                                                   \
    {                                                    \
        cerr << "[" << rank << "]: " << ( MSG ) << endl; \
    } while( false )

// Function to parse input parameters
ErrorCode get_file_options( int argc, char** argv, string& filename, string& tagName, double& tagValues )
{
    // Get mesh filename
    if( argc > 1 )
        filename = string( argv[1] );
    else
        filename = string( MESH_DIR ) + string( "/64bricks_1khex.h5m" );

    // Get tag selection options
    if( argc > 2 )
        tagName = string( argv[2] );
    else
        tagName = "USERTAG";

    if( argc > 3 )
        tagValues = atof( argv[3] );
    else
        tagValues = 1.0;

    return MB_SUCCESS;
}

//
// Start of main test program
//
int main( int argc, char** argv )
{
#ifdef MOAB_HAVE_MPI
    ErrorCode err;
    int ierr, rank;
    string filename, tagName;
    double tagValue;
    MPI_Comm comm = MPI_COMM_WORLD;
    /// Parallel Read options:
    ///   PARALLEL = type {READ_PART}
    ///   PARTITION = PARALLEL_PARTITION : Partition as you read
    ///   PARALLEL_RESOLVE_SHARED_ENTS : Communicate to all processors to get the shared adjacencies
    ///   consistently in parallel PARALLEL_GHOSTS : a.b.c
    ///                   : a = 3 - highest dimension of entities
    ///                   : b = 0 -
    ///                   : c = 1 - number of layers
    ///   PARALLEL_COMM = index
    string read_options = "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_"
                          "ENTS;PARTITION_DISTRIBUTE;PARALLEL_GHOSTS=3.0.1;PARALLEL_COMM=0";

    // Print usage if not enough arguments
    if( argc < 1 )
    {
        cerr << "Usage: ";
        cerr << argv[0] << "   " << endl;
        cerr << "file_name    : mesh file name" << endl;
        cerr << "tag_name     : name of tag to add to mesh" << endl;
        cerr << "tag_value    : a double valued string to set for highest-dimensional entities in "
                "the mesh for the named tag"
             << endl;

        ierr = MPI_Finalize();
        MPICHKERR( ierr, "MPI_Finalize failed; Aborting" );

        return 1;
    }

    // Initialize MPI first
    ierr = MPI_Init( &argc, &argv );
    MPICHKERR( ierr, "MPI_Init failed" );

    ierr = MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPICHKERR( ierr, "MPI_Comm_rank failed" );

    dbgprint( "********** reduce_exchange_tags **********\n" );

    // Create the moab instance
    Interface* mbi = new( std::nothrow ) Core;
    if( NULL == mbi ) return 1;

    // Get the input options
    err = get_file_options( argc, argv, filename, tagName, tagValue );MB_CHK_SET_ERR( err, "get_file_options failed" );

    // Print out the input parameters
    dbgprint( " Input Parameters - " );
    dbgprint( "   Filenames: " << filename );
    dbgprint( "   Tag: Name=" << tagName << " Value=" << tagValue << endl );

    // Create root sets for each mesh.  Then pass these
    // to the load_file functions to be populated.
    EntityHandle rootset, partnset;
    err = mbi->create_meshset( MESHSET_SET, rootset );MB_CHK_SET_ERR( err, "Creating root set failed" );
    err = mbi->create_meshset( MESHSET_SET, partnset );MB_CHK_SET_ERR( err, "Creating partition set failed" );

    // Create the parallel communicator object with the partition handle associated with MOAB
    ParallelComm* parallel_communicator = ParallelComm::get_pcomm( mbi, partnset, &comm );

    // Load the file from disk with given options
    err = mbi->load_file( filename.c_str(), &rootset, read_options.c_str() );MB_CHK_SET_ERR( err, "MOAB::load_file failed" );

    // Create two tag handles: Exchange and Reduction operations
    dbgprint( "-Creating tag handle " << tagName << "..." );
    Tag tagReduce, tagExchange;
    {
        stringstream sstr;
        // Create the exchange tag: default name = USERTAG_EXC
        sstr << tagName << "_EXC";
        err = mbi->tag_get_handle( sstr.str().c_str(), 1, MB_TYPE_INTEGER, tagExchange, MB_TAG_CREAT | MB_TAG_DENSE,
                                   &tagValue );MB_CHK_SET_ERR( err, "Retrieving tag handles failed" );

        // Create the exchange tag: default name = USERTAG_RED
        sstr.str( "" );
        sstr << tagName << "_RED";
        err = mbi->tag_get_handle( sstr.str().c_str(), 1, MB_TYPE_DOUBLE, tagReduce, MB_TAG_CREAT | MB_TAG_DENSE,
                                   &tagValue );MB_CHK_SET_ERR( err, "Retrieving tag handles failed" );
    }

    // Perform exchange tag data
    dbgprint( "-Exchanging tags between processors " );
    {
        Range partEnts, dimEnts;
        for( int dim = 0; dim <= 3; dim++ )
        {
            // Get all entities of dimension = dim
            err = mbi->get_entities_by_dimension( rootset, dim, dimEnts, false );MB_CHK_ERR( err );

            vector< int > tagValues( dimEnts.size(), static_cast< int >( tagValue ) * ( rank + 1 ) * ( dim + 1 ) );
            // Set local tag data for exchange
            err = mbi->tag_set_data( tagExchange, dimEnts, &tagValues[0] );MB_CHK_SET_ERR( err, "Setting local tag data failed during exchange phase" );
            // Merge entities into parent set
            partEnts.merge( dimEnts );
        }

        // Exchange tags between processors
        err = parallel_communicator->exchange_tags( tagExchange, partEnts );MB_CHK_SET_ERR( err, "Exchanging tags between processors failed" );
    }

    // Perform reduction of tag data
    dbgprint( "-Reducing tags between processors " );
    {
        Range partEnts;
        // Get all higher dimensional entities belonging to current partition
        err = parallel_communicator->get_part_entities( partEnts );MB_CHK_SET_ERR( err, "ParallelComm::get_part_entities failed" );

        // Output what is in current partition sets
        dbgprintall( "Number of Partitioned entities: " << partEnts.size() );
        MPI_Barrier( comm );

        // Set local tag data for reduction
        vector< double > tagValues( partEnts.size(), tagValue * ( rank + 1 ) );
        err = mbi->tag_set_data( tagReduce, partEnts, &tagValues[0] );MB_CHK_SET_ERR( err, "Setting local tag data failed during reduce phase" );

        Range dummy;
        // Reduce tag data using MPI_SUM on the interface between partitions
        err = parallel_communicator->reduce_tags( tagReduce, MPI_SUM, dummy /*partEnts*/ );MB_CHK_SET_ERR( err, "Reducing tags between processors failed" );
    }
    // Write out to output file to visualize reduction/exchange of tag data
    err = mbi->write_file( "test.h5m", "H5M", "PARALLEL=WRITE_PART" );MB_CHK_ERR( err );

    // Done, cleanup
    delete mbi;

    dbgprint( "\n********** reduce_exchange_tags DONE! **********" );

    MPI_Finalize();
#else
    std::cout << " compile with MPI and HDF5 for this example to work \n";
#endif
    return 0;
}