TestRunner.hpp

Go to the documentation of this file.

#ifndef TEST_RUNNER_HPP
#define TEST_RUNNER_HPP

#include "TestUtil.hpp"

/***************************************************************************************
 * Begin test runner implementation.
 * This is a higher-level API that can be used to register tests,
 * test dependencies, and to run-time select a subset of tests to
 * run.
 ***************************************************************************************/

/* Register a test to be run */
#define REGISTER_TEST( TEST_FUNC ) runner_register_test( __FILE__, __LINE__, #TEST_FUNC, ( TEST_FUNC ), NULL )

/* Mark a dependency between tests.  The second argument must be
 * an already-registered test.  The first argument will be registered
 * as a test if it has not already been registered.  The test specified
 * by the first argument will be run only if the test specified by
 * the second argument is run and succeeds.
 */
#define REGISTER_DEP_TEST( TEST_FUNC, REQUIRED_FUNC ) \
    runner_register_test( __FILE__, __LINE__, #TEST_FUNC, ( TEST_FUNC ), ( REQUIRED_FUNC ) )

/* Run registered tests.
 * Arguments should be argc and argv passed to main.
 * If ARGC is less than or equal to 1 then all tests are run.
 * Otherwise only tests specified in the argument list are run.
 * Returns number of failed tests.
 */
#define RUN_TESTS( ARGC, ARGV ) runner_run_tests( ( ARGC ), ( ARGV ) )

/***************************************************************************************
 * NOTE: The remainder of this file contains the implementation of the above macros.
 *       The above macros constitute the entire intended API.
 ***************************************************************************************/

static void runner_register_test( const char* filename,
                                  int line_number,
                                  const char* name,
                                  test_func function,
                                  test_func requisite = 0 );
static int runner_run_tests( int argc, char* argv[] );

static void runner_usage( FILE* str, int argc, char* argv[] );
static void runner_list_tests( int long_format );
static void runner_help( int argc, char* argv[] );

enum RunnerStatus
{
    PASSED,
    FAILED,
    DESELECTED,
    SELECTED
};
struct RunnerTest
{
    test_func testFunc;
    char* testName;
    enum RunnerStatus testStatus;
    int* testRequisites;
    size_t numRequisites;
};

struct RunnerTest* RunnerTestList = 0;
size_t RunnerTestCount            = 0;
const size_t RUNNER_NOT_FOUND     = ~(size_t)0;
static size_t runner_find_test_func( test_func f );
static size_t runner_find_test_name( const char* name );
static size_t runner_add_test( test_func f, const char* name );
static void runner_add_requisite( size_t idx, size_t req );
static void free_test_list();

static size_t runner_find_test_func( test_func f )
{
    for( size_t i = 0; i < RunnerTestCount; ++i )
        if( RunnerTestList[i].testFunc == f ) return i;
    return RUNNER_NOT_FOUND;
}
static size_t runner_find_test_name( const char* name )
{
    for( size_t i = 0; i < RunnerTestCount; ++i )
        if( !strcmp( RunnerTestList[i].testName, name ) ) return i;
    return RUNNER_NOT_FOUND;
}
static size_t runner_add_test( test_func f, const char* name )
{
    size_t idx = runner_find_test_func( f );
    if( idx == RUNNER_NOT_FOUND )
    {
        if( !RunnerTestCount ) atexit( &free_test_list );
        idx                            = RunnerTestCount++;
        RunnerTest* new_RunnerTestList = (RunnerTest*)realloc( RunnerTestList, RunnerTestCount * sizeof( RunnerTest ) );
        if( !new_RunnerTestList )
        {
            fprintf( stderr, "TestRunner::runner_add_test(): reallocation of RunnerTestList failed\n" );
            atexit( &free_test_list );
        }
        else
            RunnerTestList = new_RunnerTestList;
        RunnerTestList[idx].testFunc       = f;
        RunnerTestList[idx].testName       = strdup( name );
        RunnerTestList[idx].testStatus     = SELECTED;
        RunnerTestList[idx].testRequisites = 0;
        RunnerTestList[idx].numRequisites  = 0;
    }
    return idx;
}
static void runner_add_requisite( size_t idx, size_t req )
{
    size_t i;
    for( i = 0; i < RunnerTestList[idx].numRequisites; ++i )
        if( RunnerTestList[idx].testRequisites[i] == (int)req ) return;
    ++RunnerTestList[idx].numRequisites;
    RunnerTestList[idx].testRequisites =
        (int*)realloc( RunnerTestList[idx].testRequisites,
                       RunnerTestList[idx].numRequisites * sizeof( *RunnerTestList[idx].testRequisites ) );
    RunnerTestList[idx].testRequisites[RunnerTestList[idx].numRequisites - 1] = req;
}
static void free_test_list()
{
    for( size_t i = 0; i < RunnerTestCount; ++i )
    {
        free( RunnerTestList[i].testName );
        free( RunnerTestList[i].testRequisites );
    }
    free( RunnerTestList );
}

void runner_register_test( const char* filename, int line_number, const char* name, test_func test, test_func req )
{
    size_t i = runner_add_test( test, name );
    size_t req_idx;
    if( req )
    {
        req_idx = runner_find_test_func( req );
        if( RUNNER_NOT_FOUND == req_idx )
        {
            fprintf( stderr,
                     "Error registering requisite for test: \"%s\"\n"
                     "\tat %s:%d\n"
                     "\tRequisite test function not registered.\n",
                     name, filename, line_number );
            abort();
        }
        runner_add_requisite( i, req_idx );
    }
}

void runner_usage( FILE* str, int /*argc*/, char* argv[] )
{
    fprintf( str, "%s [-l|-L] [-h] [-r] [<test_name> [<test_name> ...]]\n", argv[0] );
}

void runner_help( int argc, char* argv[] )
{
    runner_usage( stdout, argc, argv );
    fprintf( stdout, "-l : List test names and exit\n"
                     "-L : List test names and requisites and exit\n"
                     "-h : This help text\n"
                     "-r : Recursively run requisite tests for any specified test\n"
                     "\n" );
}

void runner_list_tests( int long_format )
{
    size_t i, j;
    printf( "Test List:\n" );
    for( i = 0; i < RunnerTestCount; ++i )
    {
        if( RunnerTestList[i].testStatus == DESELECTED ) continue;
        printf( " o %s\n", RunnerTestList[i].testName );
        if( !long_format || !RunnerTestList[i].numRequisites ) continue;
        if( RunnerTestList[i].numRequisites == 1 )
            printf( "  Requires : %s\n", RunnerTestList[RunnerTestList[i].testRequisites[0]].testName );
        else
        {
            printf( "  Requires : \n" );
            for( j = 0; j < RunnerTestList[i].numRequisites; ++j )
                printf( "    - %s\n", RunnerTestList[RunnerTestList[i].testRequisites[j]].testName );
        }
    }
}

int runner_run_tests( int argc, char* argv[] )
{
    /* Counters */
    int error_count  = 0;
    int fail_count   = 0;
    int num_selected = 0;
    int num_run      = 0;

    /* Flags from parsed arguments */
    int run_requisites = 0;
    int list_tests     = 0;
    int first_selected = 1;

    /* Misc iterator vars and such */
    int changed_some, ran_some, can_run, fail;
    int k;
    const char* c;
    size_t i, j;

    /* Process command line arguments */
    for( k = 1; k < argc; ++k )
    {
        if( argv[k][0] == '-' )
        {
            for( c = argv[k] + 1; *c; ++c )
            {
                switch( *c )
                {
                    case 'l':
                        list_tests = 1;
                        break;
                    case 'L':
                        list_tests = 2;
                        break;
                    case 'r':
                        run_requisites = true;
                        break;
                    case 'h':
                        runner_help( argc, argv );
                        return 0;
                    default:
                        runner_usage( stderr, argc, argv );
                        fprintf( stderr, "Unknown flag: '%c'\n", *c );
                        return 1;
                }
            }
        }
        else
        {
            // If user has specified some tests to run, begin
            // by marking all tests as de-selected.
            if( first_selected )
            {
                for( i = 0; i < RunnerTestCount; ++i )
                    RunnerTestList[i].testStatus = DESELECTED;
                first_selected = 0;
            }
            // Mark specified test as selected.
            i = runner_find_test_name( argv[k] );
            if( RUNNER_NOT_FOUND == i )
            {
                fprintf( stderr, "Unknown test name: \"%s\"\n", argv[k] );
                ++error_count;
            }
            else
            {
                RunnerTestList[i].testStatus = SELECTED;
            }
        }
    }

    /* If recursively running requisite tests, select those also. */
    if( run_requisites )
    {
        do
        {
            changed_some = 0;
            for( i = 0; i < RunnerTestCount; ++i )
            {
                if( RunnerTestList[i].testStatus == DESELECTED ) continue;

                for( j = 0; j < RunnerTestList[i].numRequisites; ++j )
                {
                    if( RunnerTestList[RunnerTestList[i].testRequisites[j]].testStatus == DESELECTED )
                    {
                        RunnerTestList[RunnerTestList[i].testRequisites[j]].testStatus = SELECTED;
                        changed_some                                                   = 1;
                    }
                }
            }
        } while( changed_some );
    }

    // Count number of selected tests
    num_selected = 0;
    for( i = 0; i < RunnerTestCount; ++i )
        if( RunnerTestList[i].testStatus == SELECTED ) ++num_selected;

    if( list_tests )
    {
        runner_list_tests( list_tests - 1 );
        return error_count;
    }

    // Now run the tests
    num_run = 0;
    do
    {
        ran_some = 0;
        for( i = 0; i < RunnerTestCount; ++i )
        {
            if( RunnerTestList[i].testStatus != SELECTED ) continue;
            can_run = 1;
            for( j = 0; j < RunnerTestList[i].numRequisites; ++j )
            {
                k = RunnerTestList[i].testRequisites[j];
                if( RunnerTestList[k].testStatus != PASSED && RunnerTestList[k].testStatus != DESELECTED )
                {
                    can_run = 0;
                    break;
                }
            }
            if( !can_run ) continue;

            ran_some = 1;
            ++num_run;
            fail = run_test( RunnerTestList[i].testFunc, RunnerTestList[i].testName );
            if( fail )
            {
                error_count++;
                fail_count++;
                RunnerTestList[i].testStatus = FAILED;
            }
            else
            {
                RunnerTestList[i].testStatus = PASSED;
            }
        }
    } while( ran_some );

    // check if we are running parallel MPI tests
    int rank = 0;
#ifdef MOAB_HAVE_MPI
    int isInit;
    MPI_Initialized( &isInit );
    if( isInit )
    {
        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    }
#endif

    // Print brief summary
    if( rank == 0 )
    {
        if( num_run == (int)RunnerTestCount && !fail_count )
        {
            printf( "All %d tests passed.\n", num_run );
        }
        else if( num_run == num_selected && !fail_count )
        {
            printf( "All %d selected tests passed.\n", num_run );
            printf( "Skipped %d non-selected tests\n", (int)( RunnerTestCount - num_selected ) );
        }
        else
        {
            printf( "%2d tests registered\n", (int)RunnerTestCount );
            if( num_selected == num_run )
                printf( "%2d tests selected\n", num_selected );
            else
                printf( "%2d of %2d tests ran\n", num_run, num_selected );
            if( num_run < (int)RunnerTestCount )
                printf( "%2d of %2d tests skipped\n", (int)RunnerTestCount - num_run, (int)RunnerTestCount );
            printf( "%2d of %2d tests passed\n", num_run - fail_count, num_run );
            if( fail_count ) printf( "%2d of %2d tests FAILED\n", fail_count, num_run );
        }
    }

    return error_count;
}

#endif