MOAB: Mesh Oriented datABase  (version 5.4.0)
WriteHDF5Parallel.hpp
Go to the documentation of this file.
00001 #ifndef WRITE_HDF5_PARALLEL_HPP
00002 #define WRITE_HDF5_PARALLEL_HPP
00003 
00004 #include "WriteHDF5.hpp"
00005 #include <H5Spublic.h>
00006 #include <map>
00007 
00008 namespace moab
00009 {
00010 
00011 struct RemoteSetData;
00012 class ParallelComm;
00013 class IODebugTrack;
00014 
00015 /**
00016  * \brief Write MOAB HDF5 file in parallel.
00017  * \author Jason Kraftcheck
00018  * \data   22 July 2004
00019  */
00020 class WriteHDF5Parallel : public WriteHDF5
00021 {
00022   public:
00023     static WriterIface* factory( Interface* );
00024 
00025     /** Consturctor
00026      */
00027     WriteHDF5Parallel( Interface* iface );
00028 
00029     virtual ~WriteHDF5Parallel();
00030 
00031   protected:
00032     virtual void debug_barrier_line( int lineno );
00033 
00034     virtual void print_times( const double* times ) const;
00035 
00036     //! Called by normal (non-parallel) writer.  Sets up
00037     //! necessary data for parallel write.
00038     virtual ErrorCode parallel_create_file( const char* filename,
00039                                             bool overwrite,
00040                                             const std::vector< std::string >& qa_records,
00041                                             const FileOptions& opts,
00042                                             const Tag* user_tag_list = 0,
00043                                             int user_tag_count       = 0,
00044                                             int dimension            = 3,
00045                                             double* times            = 0 );
00046 
00047     //! Figure out which mesh local mesh is duplicated on
00048     //! remote processors and which processor will write
00049     //! that mesh.
00050     //!\param non_local_ents Output list of entities that are not to
00051     //!                  be written by this processor but are
00052     //!                  referenced by other entities that are
00053     //!                  to be written.
00054     ErrorCode gather_interface_meshes( Range& non_local_ents );
00055 
00056     //! For entities that will be written by another
00057     //! processor but are referenced by entities on this
00058     //! processor, get the file Ids that will be assigned
00059     //! to those so they can be referenced by
00060     //! entities to be written on this processor.
00061     //!\param non_local_ents List of entities that are not to
00062     //!                  be written by this processor but are
00063     //!                  referenced by other entities that are
00064     //!                  to be written.
00065     ErrorCode exchange_file_ids( const Range& non_local_ents );
00066 
00067     //! Get remote ids for shared sets
00068     ErrorCode communicate_shared_set_ids( const Range& owned, const Range& remote );
00069 
00070     //! Pack set data for communication.
00071     //!
00072     //! If set_data_length is insufficient for the set data,
00073     //! the length entries at indices 1, 2, and 3 of set_data
00074     //! will be set with the necessary lengths, but no data will
00075     //! be written to set_data beyond that.
00076     ErrorCode pack_set( Range::const_iterator set, unsigned long* set_data, size_t set_data_length );
00077 
00078     //! Unpack set data from communication
00079     ErrorCode unpack_set( EntityHandle set, const unsigned long* set_data, size_t set_data_length );
00080 
00081     //! Communicate set contents between processors such that each
00082     //! owner knows the contents, parents, & child lists from all
00083     //! processors that have a copy of the set.
00084     ErrorCode communicate_shared_set_data( const Range& owned, const Range& remote );
00085 
00086     //! Create the node table in the file.
00087     ErrorCode create_node_table( int dimension );
00088 
00089     //! Communicate with other processors to negotiate
00090     //! the types of elements that will be written
00091     //! (the union of the types defined on each proc.)
00092     ErrorCode negotiate_type_list();
00093 
00094     //! Create tables to hold element connectivity
00095     ErrorCode create_element_tables();
00096 
00097     //! Create tables to hold element adjacencies.
00098     ErrorCode create_adjacency_tables();
00099 
00100     //! Create tables for mesh sets
00101     ErrorCode create_meshset_tables( double* times );
00102 
00103     //! Write tag descriptions and create tables to hold tag data.
00104     ErrorCode create_tag_tables();
00105 
00106     //! Remove any remote mesh entities from the passed range.
00107     void remove_remote_entities( EntityHandle relative, Range& range );
00108     void remove_remote_entities( EntityHandle relative, std::vector< EntityHandle >& vect );
00109     void remove_remote_sets( EntityHandle relative, Range& range );
00110     void remove_remote_sets( EntityHandle relative, std::vector< EntityHandle >& vect );
00111 
00112     //! get any existing tags which aren't excluded and add to shared set tags
00113     ErrorCode get_sharedset_tags();
00114 
00115     ErrorCode append_serial_tag_data( std::vector< unsigned char >& buffer, const WriteHDF5::TagDesc& tag );
00116 
00117     //! helper function for create_tag_tables
00118     ErrorCode check_serial_tag_data( const std::vector< unsigned char >& buffer,
00119                                      std::vector< TagDesc* >* missing = 0,
00120                                      std::vector< TagDesc* >* newlist = 0 );
00121 
00122     /**\brief Argument ot create_dataset */
00123     struct DataSetCreator
00124     {
00125         virtual ErrorCode operator()( WriteHDF5* writer,
00126                                       long data_set_size,
00127                                       const ExportSet* group,
00128                                       long& start_id_out ) const = 0;
00129     };
00130     struct NoopDescCreator : public DataSetCreator
00131     {
00132         ErrorCode operator()( WriteHDF5*, long, const ExportSet*, long& start_id ) const
00133         {
00134             start_id = -1;
00135             return MB_SUCCESS;
00136         }
00137     };
00138 
00139     /**\brief Do typical communication for dataset creation
00140      *
00141      * Given the number of entities each processor intends to write,
00142      * do necessary communication and create dataset on root, passing
00143      * back misc info to each proc.
00144      *
00145      *\param creator             Functor to do actual dataset creation.  Used
00146      *                           only on root process.
00147      *\param num_datasets        The number of datasets to create.
00148      *\param groups              Third argument passed to DataSetCreator.
00149      *                           Array of length \c num_datasets pr NULL.
00150      *\param num_owned_entities  The number of entities this proc will write.
00151      *                           Array of length \c num_datasets .
00152      *\param offsets_out         Output: The offset in the dataset at which
00153      *                           this process should write.
00154      *                           Array of length \c num_datasets .
00155      *\param max_proc_ents_out   Output: The maximun number of entities that
00156      *                           any proc will write
00157      *                           Array of length \c num_datasets .
00158      *\param total_ents_out      Output: The size of the created dataset (sum
00159      *                           of counts over all procs)
00160      *                           Array of length \c num_datasets .
00161      *\param first_ids_out       Output: The first ID of the first entity in the
00162      *                           data set.  First ID for this proc's entities is
00163      *                           first_id_out+offset_out
00164      *                           Array of length \c num_datasets or NULL.
00165      */
00166     ErrorCode create_dataset( int num_datasets,
00167                               const long* num_owned_entities,
00168                               long* offsets_out,
00169                               long* max_proc_ents_out,
00170                               long* total_ents_out,
00171                               const DataSetCreator& creator = NoopDescCreator(),
00172                               ExportSet* groups[]           = 0,
00173                               wid_t* first_ids_out          = NULL );
00174 
00175     void print_shared_sets();
00176     void print_set_sharing_data( const Range& range, const char* label, Tag idt );
00177 
00178   private:
00179     //! pcomm controlling parallel nature of mesh
00180     ParallelComm* myPcomm;
00181 
00182     //! whether this instance allocated (and dtor should delete) the pcomm
00183     bool pcommAllocated;
00184 
00185     //! Operation to use to append hyperslab selections
00186     H5S_seloper_t hslabOp;
00187 };
00188 
00189 }  // namespace moab
00190 
00191 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines