MOAB: Mesh Oriented datABase  (version 5.2.1)
mhdf_parallel.c
Go to the documentation of this file.
00001 /* Test mhdf library on top of Parallel HDF5.
00002  *
00003  * Call mhdf API similar to how WriteHDF5Parallel does,
00004  * but avoid any of our own parallel communiation.
00005  *
00006  * Output will be composed of:
00007  *  a 1-D array of hexes, one per processor, where the first
00008  *    process writes all the nodes for its hex and every other
00009  *    process writes the right 4 nodes of its hex.
00010  *  a set containing all of the hexes
00011  *  a set containing all of the nodes
00012  *  a set per processor containing all of the entities written by that proc
00013  *  a tag specifying the process ID that wrote each entity.
00014  */
00015 
00016 #include "mhdf.h"
00017 
00018 #include <time.h>
00019 #include <stdlib.h>
00020 #include <stdio.h>
00021 #include <assert.h>
00022 
00023 #include <H5Ppublic.h>
00024 #include <H5Tpublic.h>
00025 #include <H5Epublic.h>
00026 #include <H5FDmpi.h>
00027 #include <H5FDmpio.h>
00028 
00029 #ifdef H5_HAVE_PARALLEL
00030 
00031 const char filename[] = "mhdf_ll.h5m";
00032 const char proc_tag_name[] = "proc_id";
00033 const char elem_handle[] = "Hex";
00034 
00035 int RANK;
00036 int NUM_PROC;
00037 
00038 #define CHECK( A ) check( &(A), __LINE__ )
00039 void check( mhdf_Status* status, int line )
00040 {
00041   if (mhdf_isError( status )) {
00042     fprintf(stderr,"%d: ERROR at line %d: \"%s\"\n", RANK, line, mhdf_message( status ) );
00043     abort();
00044   }
00045 }
00046 
00047 typedef int handle_id_t;
00048 #define id_type H5T_NATIVE_INT
00049 
00050 /* create file layout in serial */
00051 void create_file()
00052 {
00053   const char* elem_types[1] = {elem_handle};
00054   const int num_elem_types = sizeof(elem_types)/sizeof(elem_types[0]);
00055   const int num_nodes = 4+4*NUM_PROC;
00056   const int num_hexes = NUM_PROC;
00057   const int num_sets = 2+NUM_PROC;
00058   const int set_data_size = num_hexes + num_nodes + num_hexes + num_nodes;
00059   const int num_tag_vals = num_hexes + num_nodes + num_sets - 2;
00060   const char* history[] = { __FILE__, NULL };
00061   const int history_size = sizeof(history) / sizeof(history[0]);
00062   int default_tag_val = NUM_PROC;
00063   hid_t data, tagdata[2];
00064   long junk;
00065   
00066   mhdf_Status status;
00067   mhdf_FileHandle file;
00068   
00069   time_t t;
00070   t = time(NULL);
00071   history[1] = ctime(&t);
00072   
00073   file = mhdf_createFile( filename, 1, elem_types, num_elem_types, id_type, &status );
00074   CHECK(status);
00075   
00076   mhdf_writeHistory( file, history, history_size, &status );
00077   CHECK(status);
00078   
00079   data = mhdf_createNodeCoords( file, 3, num_nodes, &junk, &status );
00080   CHECK(status);
00081   mhdf_closeData( file, data, &status );
00082   CHECK(status);
00083   
00084   mhdf_addElement( file, elem_types[0], 0, &status );
00085   CHECK(status);
00086   data = mhdf_createConnectivity( file, elem_handle, 8, num_hexes, &junk, &status );
00087   CHECK(status);
00088   mhdf_closeData( file, data, &status );
00089   CHECK(status);
00090   
00091   data = mhdf_createSetMeta( file, num_sets, &junk, &status );
00092   CHECK(status);
00093   mhdf_closeData( file, data, &status );
00094   CHECK(status);
00095 
00096   data = mhdf_createSetData( file, set_data_size, &status );
00097   CHECK(status);
00098   mhdf_closeData( file, data, &status );
00099   CHECK(status);
00100   
00101   mhdf_createTag( file, proc_tag_name, mhdf_INTEGER, 1, mhdf_DENSE_TYPE, 
00102                   &default_tag_val, &default_tag_val, 0, 0, &status );
00103   CHECK(status);
00104   
00105   mhdf_createSparseTagData( file, proc_tag_name, num_tag_vals, tagdata, &status );
00106   CHECK(status);
00107   mhdf_closeData( file, tagdata[0], &status );
00108   CHECK(status);
00109   mhdf_closeData( file, tagdata[1], &status );
00110   CHECK(status);
00111  
00112   mhdf_closeFile( file, &status );
00113   CHECK(status);
00114 }
00115 
00116 void write_file_data()
00117 {
00118   const int total_num_nodes = 4+4*NUM_PROC;
00119   const int total_num_hexes = NUM_PROC;
00120   long first_node, first_elem, first_set, count, ntag;
00121   unsigned long ucount;
00122   mhdf_index_t set_desc[4] = { 0, -1, -1, 0 };
00123   hid_t handle, handles[2];
00124   mhdf_Status status;
00125   mhdf_FileHandle file;
00126   int num_node, offset, dim;
00127   double coords[3*8] = { 0.0, 0.0, 0.0, 
00128                          0.0, 1.0, 0.0, 
00129                          0.0, 1.0, 1.0, 
00130                          0.0, 0.0, 1.0,
00131                          0.0, 0.0, 0.0, 
00132                          0.0, 1.0, 0.0, 
00133                          0.0, 1.0, 1.0, 
00134                          0.0, 0.0, 1.0 };
00135   handle_id_t list[10];
00136   int i, tagdata[10];
00137   for (i = 0; i < 10; i++) tagdata[i] = RANK;
00138   
00139   
00140     /* open file */
00141   handle = H5Pcreate( H5P_FILE_ACCESS );
00142   H5Pset_fapl_mpio( handle, MPI_COMM_WORLD, MPI_INFO_NULL );
00143   file = mhdf_openFileWithOpt( filename, 1, &ucount, id_type, handle, &status );
00144   CHECK(status);
00145   H5Pclose( handle );
00146   
00147     /* write node coordinates */
00148   if (0 == RANK) {
00149     num_node = 8;
00150     offset = 0;
00151     coords[12] = coords[15] = coords[18] = coords[21] = 1.0;
00152   }
00153   else {
00154     num_node = 4;
00155     offset = 4 + 4*RANK;
00156     coords[0] = coords[3] = coords[6] = coords[9] = 1.0 + RANK;
00157   }
00158   handle = mhdf_openNodeCoords( file, &count, &dim, &first_node, &status );
00159   CHECK(status);
00160   assert( count == total_num_nodes );
00161   assert( dim == 3 );
00162   mhdf_writeNodeCoords( handle, offset, num_node, coords, &status );
00163   CHECK(status);
00164   mhdf_closeData( file, handle, &status );
00165   CHECK(status);
00166   
00167     /* write hex connectivity */
00168   for (i = 0; i < 8; ++i)
00169     list[i] = 4*RANK + i + first_node;
00170   handle = mhdf_openConnectivity( file, elem_handle, &dim, &count, &first_elem, &status );
00171   CHECK(status);
00172   assert( count == total_num_hexes );
00173   assert( 8 == dim );
00174   mhdf_writeConnectivity( handle, RANK, 1, id_type, list, &status );
00175   CHECK(status);
00176   mhdf_closeData( file, handle, &status );
00177   CHECK(status);
00178   
00179     /* write set descriptions */
00180   handle = mhdf_openSetMeta( file, &count, &first_set, &status );
00181   CHECK(status);
00182   assert( count == 2+NUM_PROC );
00183   
00184     /* write set descriptions for per-proc sets */
00185   set_desc[0] = total_num_nodes + total_num_hexes + 9 + 5*RANK - 1;
00186   mhdf_writeSetMeta( handle, 2+RANK, 1, MHDF_INDEX_TYPE, set_desc, &status );
00187   CHECK(status);
00188   
00189     /* write set descriptions for multi-proc sets */
00190   if (0 == RANK) {
00191     set_desc[0] = total_num_nodes - 1;
00192     mhdf_writeSetMeta( handle, 0, 1, MHDF_INDEX_TYPE, set_desc, &status );
00193     CHECK(status);
00194     set_desc[0] += total_num_hexes;
00195     mhdf_writeSetMeta( handle, 1, 1, MHDF_INDEX_TYPE, set_desc, &status );
00196     CHECK(status);
00197   }
00198 
00199   mhdf_closeData( file, handle, &status );
00200   CHECK(status);
00201   
00202     /* write set contents */
00203   
00204   handle = mhdf_openSetData( file, &count, &status );
00205   CHECK(status);
00206   assert( 2*total_num_nodes + 2*total_num_hexes == count );
00207   
00208     /* write per-proc sets */
00209   if (0 == RANK) {
00210     count = 9;
00211     offset = total_num_nodes + total_num_hexes;
00212     for (i = 0; i < 8; ++i)
00213       list[i] = first_node + i;
00214     list[8] = first_elem;
00215   }
00216   else {
00217     count = 5;
00218     offset = total_num_nodes + total_num_hexes + 4 + 5*RANK;
00219     for (i = 0; i < 4; ++i)
00220       list[i] = 4 + 4*RANK + first_node + i;
00221     list[4] = RANK + first_elem;
00222   }
00223   mhdf_writeSetData( handle, offset, count, id_type, list, &status );
00224   CHECK(status);
00225   
00226     /* write multi-proc sets */
00227     /* write nodes */
00228   offset = (0 == RANK) ? 0 : 4 + 4*RANK;
00229   mhdf_writeSetData( handle, offset, count-1, id_type, list, &status );
00230   CHECK(status);
00231     /* write hex */
00232   offset = total_num_nodes + RANK;
00233   mhdf_writeSetData( handle, offset, 1, id_type, list + count - 1, &status );
00234   CHECK(status);
00235   
00236   mhdf_closeData( file, handle, &status );
00237   CHECK(status);
00238   
00239     /* write tag data */
00240   offset = (0 == RANK) ? 0 : 4 + 4*RANK;
00241   offset += 2*RANK;
00242   list[count++] = first_set + 2 + RANK;
00243   mhdf_openSparseTagData( file, proc_tag_name, &ntag, &ntag, handles, &status );
00244   CHECK(status);
00245   assert( ntag == total_num_nodes + total_num_hexes + NUM_PROC );
00246   mhdf_writeSparseTagEntities( handles[0], offset, count, id_type, list, &status );
00247   CHECK(status);
00248   mhdf_writeTagValues( handles[1], offset, count, H5T_NATIVE_INT, tagdata, &status );
00249   CHECK(status);
00250   mhdf_closeData( file, handles[0], &status );
00251   CHECK(status);
00252   mhdf_closeData( file, handles[1], &status );
00253   CHECK(status);
00254   
00255     /* done */
00256   mhdf_closeFile( file, &status );
00257   CHECK(status);
00258 }
00259 
00260 /* Define a dummy error handler to register with HDF5. 
00261  * This function doesn't do anything except pass the
00262  * error on to the default handler that would have been
00263  * called anyway.  It's only purpose is to provide a 
00264  * spot to set a break point so we can figure out where
00265  * (in our code) that we made an invalid call into HDF5
00266  */
00267 #if defined(H5E_auto_t_vers) && H5E_auto_t_vers > 1
00268 herr_t (*default_handler)( hid_t, void* );
00269 static herr_t handle_hdf5_error( hid_t stack, void* data )
00270 #else
00271 herr_t (*default_handler)( void* );
00272 static herr_t handle_hdf5_error( void* data )
00273 #endif
00274 {
00275   herr_t result = 0;
00276   if (default_handler)
00277 #if defined(H5E_auto_t_vers) && H5E_auto_t_vers > 1
00278     result = (default_handler)(stack,data);
00279 #else
00280     result = (default_handler)(data);
00281 #endif
00282   assert(0);
00283   return result;
00284 }
00285 
00286 #endif /* #ifdef H5_HAVE_PARALLEL */
00287  
00288 int main( int argc, char* argv[] )
00289 {
00290 #ifdef H5_HAVE_PARALLEL
00291   int rval;
00292   void* data;
00293   herr_t err;
00294 
00295 #if defined(H5Eget_auto_vers) && H5Eget_auto_vers > 1
00296   err = H5Eget_auto( H5E_DEFAULT, &default_handler, &data );
00297 #else
00298   err = H5Eget_auto( &default_handler, &data );
00299 #endif
00300   if (err >= 0) {
00301 #if defined(H5Eset_auto_vers) && H5Eset_auto_vers > 1
00302     H5Eset_auto( H5E_DEFAULT, &handle_hdf5_error, data );
00303 #else
00304     H5Eset_auto( &handle_hdf5_error, data );
00305 #endif
00306   }
00307   
00308   rval = MPI_Init( &argc, &argv );
00309   if (rval)
00310     return rval;
00311   rval = MPI_Comm_rank( MPI_COMM_WORLD, &RANK );
00312   if (rval)
00313     return rval;
00314   rval = MPI_Comm_size( MPI_COMM_WORLD, &NUM_PROC );
00315   if (rval)
00316     return rval;
00317   
00318   if (RANK == 0)
00319     create_file();
00320 
00321   /* Wait for rank 0 to finish creating the file, otherwise rank 1 may find it to be invalid */
00322   rval = MPI_Barrier(MPI_COMM_WORLD);
00323   if (rval)
00324     return rval;
00325 
00326   write_file_data();
00327   
00328   MPI_Finalize();
00329 #endif
00330   return 0;
00331 }
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines