1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
#ifndef WRITE_HDF5_PARALLEL_HPP
#define WRITE_HDF5_PARALLEL_HPP

#include "WriteHDF5.hpp"
#include <H5Spublic.h>
#include <map>

namespace moab
{

struct RemoteSetData;
class ParallelComm;
class IODebugTrack;

/**
 * \brief Write MOAB HDF5 file in parallel.
 * \author Jason Kraftcheck
 * \data   22 July 2004
 */
class WriteHDF5Parallel : public WriteHDF5
{
  public:
    static WriterIface* factory( Interface* );

    /** Consturctor
     */
    WriteHDF5Parallel( Interface* iface );

    virtual ~WriteHDF5Parallel();

  protected:
    virtual void debug_barrier_line( int lineno );

    virtual void print_times( const double* times ) const;

    //! Called by normal (non-parallel) writer.  Sets up
    //! necessary data for parallel write.
    virtual ErrorCode parallel_create_file( const char* filename,
                                            bool overwrite,
                                            const std::vector< std::string >& qa_records,
                                            const FileOptions& opts,
                                            const Tag* user_tag_list = 0,
                                            int user_tag_count       = 0,
                                            int dimension            = 3,
                                            double* times            = 0 );

    //! Figure out which mesh local mesh is duplicated on
    //! remote processors and which processor will write
    //! that mesh.
    //!\param non_local_ents Output list of entities that are not to
    //!                  be written by this processor but are
    //!                  referenced by other entities that are
    //!                  to be written.
    ErrorCode gather_interface_meshes( Range& non_local_ents );

    //! For entities that will be written by another
    //! processor but are referenced by entities on this
    //! processor, get the file Ids that will be assigned
    //! to those so they can be referenced by
    //! entities to be written on this processor.
    //!\param non_local_ents List of entities that are not to
    //!                  be written by this processor but are
    //!                  referenced by other entities that are
    //!                  to be written.
    ErrorCode exchange_file_ids( const Range& non_local_ents );

    //! Get remote ids for shared sets
    ErrorCode communicate_shared_set_ids( const Range& owned, const Range& remote );

    //! Pack set data for communication.
    //!
    //! If set_data_length is insufficient for the set data,
    //! the length entries at indices 1, 2, and 3 of set_data
    //! will be set with the necessary lengths, but no data will
    //! be written to set_data beyond that.
    ErrorCode pack_set( Range::const_iterator set, unsigned long* set_data, size_t set_data_length );

    //! Unpack set data from communication
    ErrorCode unpack_set( EntityHandle set, const unsigned long* set_data, size_t set_data_length );

    //! Communicate set contents between processors such that each
    //! owner knows the contents, parents, & child lists from all
    //! processors that have a copy of the set.
    ErrorCode communicate_shared_set_data( const Range& owned, const Range& remote );

    //! Create the node table in the file.
    ErrorCode create_node_table( int dimension );

    //! Communicate with other processors to negotiate
    //! the types of elements that will be written
    //! (the union of the types defined on each proc.)
    ErrorCode negotiate_type_list();

    //! Create tables to hold element connectivity
    ErrorCode create_element_tables();

    //! Create tables to hold element adjacencies.
    ErrorCode create_adjacency_tables();

    //! Create tables for mesh sets
    ErrorCode create_meshset_tables( double* times );

    //! Write tag descriptions and create tables to hold tag data.
    ErrorCode create_tag_tables();

    //! Remove any remote mesh entities from the passed range.
    void remove_remote_entities( EntityHandle relative, Range& range );
    void remove_remote_entities( EntityHandle relative, std::vector< EntityHandle >& vect );
    void remove_remote_sets( EntityHandle relative, Range& range );
    void remove_remote_sets( EntityHandle relative, std::vector< EntityHandle >& vect );

    //! get any existing tags which aren't excluded and add to shared set tags
    ErrorCode get_sharedset_tags();

    ErrorCode append_serial_tag_data( std::vector< unsigned char >& buffer, const WriteHDF5::TagDesc& tag );

    //! helper function for create_tag_tables
    ErrorCode check_serial_tag_data( const std::vector< unsigned char >& buffer,
                                     std::vector< TagDesc* >* missing = 0,
                                     std::vector< TagDesc* >* newlist = 0 );

    /**\brief Argument ot create_dataset */
    struct DataSetCreator
    {
        virtual ErrorCode operator()( WriteHDF5* writer,<--- Virtual function in base class
                                      long data_set_size,
                                      const ExportSet* group,
                                      long& start_id_out ) const = 0;
    };
    struct NoopDescCreator : public DataSetCreator
    {
        ErrorCode operator()( WriteHDF5*, long, const ExportSet*, long& start_id ) const<--- Function in derived class
        {
            start_id = -1;
            return MB_SUCCESS;
        }
    };

    /**\brief Do typical communication for dataset creation
     *
     * Given the number of entities each processor intends to write,
     * do necessary communication and create dataset on root, passing
     * back misc info to each proc.
     *
     *\param creator             Functor to do actual dataset creation.  Used
     *                           only on root process.
     *\param num_datasets        The number of datasets to create.
     *\param groups              Third argument passed to DataSetCreator.
     *                           Array of length \c num_datasets pr NULL.
     *\param num_owned_entities  The number of entities this proc will write.
     *                           Array of length \c num_datasets .
     *\param offsets_out         Output: The offset in the dataset at which
     *                           this process should write.
     *                           Array of length \c num_datasets .
     *\param max_proc_ents_out   Output: The maximun number of entities that
     *                           any proc will write
     *                           Array of length \c num_datasets .
     *\param total_ents_out      Output: The size of the created dataset (sum
     *                           of counts over all procs)
     *                           Array of length \c num_datasets .
     *\param first_ids_out       Output: The first ID of the first entity in the
     *                           data set.  First ID for this proc's entities is
     *                           first_id_out+offset_out
     *                           Array of length \c num_datasets or NULL.
     */
    ErrorCode create_dataset( int num_datasets,
                              const long* num_owned_entities,
                              long* offsets_out,
                              long* max_proc_ents_out,
                              long* total_ents_out,
                              const DataSetCreator& creator = NoopDescCreator(),
                              ExportSet* groups[]           = 0,
                              wid_t* first_ids_out          = NULL );

    void print_shared_sets();
    void print_set_sharing_data( const Range& range, const char* label, Tag idt );

  private:
    //! pcomm controlling parallel nature of mesh
    ParallelComm* myPcomm;

    //! whether this instance allocated (and dtor should delete) the pcomm
    bool pcommAllocated;

    //! Operation to use to append hyperslab selections
    H5S_seloper_t hslabOp;
};

}  // namespace moab

#endif