1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
/* WriteDamsel.cpp
 * The Damsel library provides mesh-aware parallel I/O; see
 * http://cucis.ece.northwestern.edu/projects/DAMSEL/ for details, though for now that site is
 * restricted to project participants.  Damsel uses a data model that's very similar to that used in
 * MOAB and ITAPS.  It uses the same basic data model concepts of entities, sets, tags, and
 * interface. In theory, we should be able to completely save/restore to/from Damsel any data that
 * can be saved/restored to/from our native HDF5-based reader/writer.
 *
 * Mapping between MOAB-Damsel data models
 * =======================================
 * Basic data model entities, MOAB <--> Damsel:
 *        Entity <--> Entity
 *     EntitySet <--> Collection
 *           Tag <--> Tag
 * API/data strutures:
 *          Range (n1) <--> Sequence container
 *         std::vector <--> Vector container
 *          Range (n2) <--> Tree container
 *
 * n1: single contiguous subrange
 * n2: multiple subranges
 *
 * Conventions
 * ===========
 * There are parts of MOAB data structures that need to be stored to Damsel that aren't represented
 * in the Damsel data model, e.g. dense vs. sparse storage type, set tracking flags.
 * - We need to store these as tags in Damsel.
 * - Since Damsel tags need to have a MOAB counterpart, we have to create those as tag data in MOAB
 * too (duplicating the data in the data structures, bummer).
 * - Because we may want to use these tags for multiple Damsel writes/models, we create the
 * MOAB-side tags in the WriteDamsel constructor, not in the init_tags function that's called for
 * every write
 * - Conventional tags have names prefixed with mbdmsl_ to avoid name conflicts with other MOAB
 * tags. Here we list the conventional tags used by MOAB's Damsel reader/writer.
 *
 * Tag name                   Tag char's (storage type, data type, length, def val)       Values,
 * used for what
 * --------                   -----------------------------------------------------
 * -------------------- mbdmsl_XCOORDS             dense; double[1]; 0.0 MOAB vertex x coordinate
 * mbdmsl_YCOORDS             dense; double[1]; 0.0                                       MOAB
 * vertex y coordinate mbdmsl_ZCOORDS             dense; double[1]; 0.0 MOAB vertex z coordinate
 * mbdmsl_COLL_FLAGS          sparse; char; 1; 0x0                                        bit 0:
 * 0=set-type, 1=vector-type 1: 1=tracking, 0=not tracking mbdmsl_PARENTS      sparse; handle; var;
 * (list of parent sets) mbdmsl_CHILDS       sparse; handle; var; (list of child sets)
 *
 *
 *
 */

#include "WriteDamsel.hpp"

#include "DamselUtil.hpp"
#include "damsel.h"
#include <cassert>
#include "moab/Interface.hpp"
#include "moab/Core.hpp"
#include "moab/Range.hpp"
#include "moab/Error.hpp"
#include "moab/WriteUtilIface.hpp"
#include "MBTagConventions.hpp"
#include "EntitySequence.hpp"
#include "Internals.hpp"
#include "DenseTag.hpp"
#include "SparseTag.hpp"

namespace moab
{

WriterIface* WriteDamsel::factory( Interface* iface )
{
    return new WriteDamsel( iface );
}

WriteDamsel::WriteDamsel( Interface* impl )
    : mbImpl( impl ), mWriteIface( NULL ), sequenceManager( NULL ), dU(), DAMSEL_FLAGS( DAMSEL_IS_TRACKING )
{
    assert( impl != NULL );

    impl->query_interface( mWriteIface );
    assert( mWriteIface );

    sequenceManager = dynamic_cast< Core* >( impl )->sequence_manager();
    assert( sequenceManager );

    ErrorCode rval =
        mbImpl->tag_get_handle( "mbdmsl_XCOORDS", 1, MB_TYPE_DOUBLE, dU.xcoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT );MB_CHK_SET_ERR_CONT( rval, "Failed to create_tag mbdmsl_XCOORDS" );
    dU.xcoordsTag.tagType = MB_TAG_ANY;
    dU.tagMap.push_back( dU.xcoordsTag );
    rval =
        mbImpl->tag_get_handle( "mbdmsl_YCOORDS", 1, MB_TYPE_DOUBLE, dU.ycoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT );MB_CHK_SET_ERR_CONT( rval, "Failed to create_tag mbdmsl_YCOORDS" );
    dU.ycoordsTag.tagType = MB_TAG_ANY;
    dU.tagMap.push_back( dU.ycoordsTag );

    rval =
        mbImpl->tag_get_handle( "mbdmsl_ZCOORDS", 1, MB_TYPE_DOUBLE, dU.zcoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT );MB_CHK_SET_ERR_CONT( rval, "Failed to create_tag mbdmsl_ZCOORDS" );
    dU.zcoordsTag.tagType = MB_TAG_ANY;
    dU.tagMap.push_back( dU.zcoordsTag );

    rval = mbImpl->tag_get_handle( "mbdmsl_COLL_FLAGS", 1, MB_TYPE_INTEGER, dU.collFlagsTag.mTagh,
                                   MB_TAG_DENSE | MB_TAG_CREAT );MB_CHK_SET_ERR_CONT( rval, "Failed to create_tag mbdmsl_COLL_FLAGS" );
    dU.collFlagsTag.tagType = MB_TAG_ANY;
    dU.tagMap.push_back( dU.collFlagsTag );

    /*
      rval = mbImpl->tag_get_handle("mbdmsl_PARENTS", 1, MB_TYPE_HANDLE,
                                   dU.parentsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT |
      MB_TAG_VARLEN);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_PARENTS");
      dU.parentsTag.tagType = MB_TAG_DENSE;
      dU.tagMap.push_back(dU.parentsTag);

      rval = mbImpl->tag_get_handle("mbdmsl_CHILDREN", 1, MB_TYPE_HANDLE,
                                   dU.childrenTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT |
      MB_TAG_VARLEN);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_CHILDREN");
      dU.childrenTag.tagType = MB_TAG_DENSE;
      dU.tagMap.push_back(dU.childrenTag);
    */

    dU.moabHandleType = ( sizeof( EntityHandle ) == 64 ? DAMSEL_HANDLE_TYPE_HANDLE64 : DAMSEL_HANDLE_TYPE_HANDLE32 );
}

WriteDamsel::~WriteDamsel()
{
    if( mWriteIface ) mbImpl->release_interface( mWriteIface );
}

ErrorCode WriteDamsel::write_file( const char* file_name,
                                   const bool /* overwrite */,
                                   const FileOptions& opts,
                                   const EntityHandle* meshset_list,
                                   const int num_sets,
                                   const std::vector< std::string >& /* qa_records */,
                                   const Tag* /* tag_list */,
                                   int /* num_tags */,
                                   int /* requested_output_dimension */ )
{
    // Gather all entities into one big range
    Range all_ents;
    ErrorCode rval;
    damsel_err_t err;

    dU.dmslLib = DMSLlib_init();

    // Create a damsel model
    dU.dmslModel =
        DMSLmodel_create( sizeof( EntityHandle ) == 8 ? DAMSEL_HANDLE_TYPE_HANDLE64 : DAMSEL_HANDLE_TYPE_HANDLE32 );

    // Attach to a file, since we need it for creating containers
    MPI_Comm comm = MPI_COMM_WORLD;
    unlink( file_name );
    err = DMSLmodel_attach( dU.dmslModel, file_name, comm, NULL );
    CHK_DMSL_ERR( err, "DMSLmodel_attach failed" );

    rval = mWriteIface->gather_entities( all_ents, meshset_list, num_sets );MB_CHK_SET_ERR( rval, "Gather entities failed in WriteDamsel" );

    if( all_ents.empty() ) return MB_SUCCESS;

    // Create damsel tags for MOAB dense, sparse, and conventional tags
    rval = init_tag_info();MB_CHK_ERR( rval );

    // Iterate through the groups of contiguous sequences of handles
    RangeSeqIntersectIter rsi( sequenceManager );
    rval = rsi.init( all_ents.begin(), all_ents.end() );

    while( MB_SUCCESS == rval )
    {
        // Write subrange of things to damsel: map handles, map entity definition data
        // (connectivity/coords/set contents), map dense tags
        rval = write_subrange( rsi );MB_CHK_SET_ERR( rval, "Failed to write subrange" );

        rval = rsi.step();
        while( MB_ENTITY_NOT_FOUND == rval )
            rval = rsi.step();
    }

    // Write sparse tags
    rval = map_sparse_tags();MB_CHK_SET_ERR( rval, "Failed to write sparse tags" );

    // damsel_request_t request;
    // err = DMSLmodel_transfer_async(dU.dmslModel, DAMSEL_TRANSFER_TYPE_WRITE, &request);
    err = DMSLmodel_transfer_sync( dU.dmslModel, DAMSEL_TRANSFER_TYPE_WRITE );
    CHK_DMSL_ERR( err, "DMSLmodel_transfer_asynch failed" );

    // damsel_status_t status;
    // err = DMSLmodel_wait(request, &status);CHK_DMSL_ERR(err, "DMSLmodel_wait failed");

    DMSLmodel_close( dU.dmslModel );

    DMSLlib_finalize( dU.dmslLib );

    // We should be done
    return MB_SUCCESS;
}

ErrorCode WriteDamsel::init_tag_info()
{
    // Initialize allTags and tagIndices
    std::vector< Tag > tmp_mtags;
    ErrorCode rval = mbImpl->tag_get_tags( tmp_mtags );MB_CHK_SET_ERR( rval, "Failed to get all tag handles." );
    int dum_size;
    damsel_err_t err;

    // Define damsel tag handles for all dense/sparse tags
    for( std::vector< Tag >::iterator vit = tmp_mtags.begin(); vit != tmp_mtags.end(); ++vit )
    {
        if( ( ( *vit )->get_storage_type() != MB_TAG_DENSE && ( *vit )->get_storage_type() != MB_TAG_SPARSE ) ||
            mbImpl->tag_get_length( *vit, dum_size ) == MB_VARIABLE_DATA_LENGTH || dum_size != 1 )
        {
            std::cerr << "Warning: tag " << ( *vit )->get_name()
                      << "is not of type dense or sparse, and is not currently supported by the "
                         "damsel writer."
                      << std::endl;
            continue;
        }

        std::vector< DamselUtil::tinfo >::iterator vit2 =
            std::find_if( dU.tagMap.begin(), dU.tagMap.end(), DamselUtil::MtagP< DamselUtil::tinfo >( *vit ) );

        if( vit2 != dU.tagMap.end() && ( *vit2 ).tagType == MB_TAG_ANY )
            // Conventional tag - skip
            continue;

        else if( vit2 == dU.tagMap.end() )
        {
            // Create a damsel counterpart for this tag
            Tag thandle = *vit;
            err         = DMSLtag_define( dU.dmslModel, (damsel_handle_ptr)&thandle,
                                          DamselUtil::mtod_data_type[( *vit )->get_data_type()], ( *vit )->get_name().c_str() );
            CHK_DMSL_ERR( err, "Failure to get Damsel tag for MOAB tag" );
            dU.tagMap.push_back( DamselUtil::tinfo( thandle, 0, ( *vit )->get_storage_type() ) );
        }
        else
        {
            // Assert there's a corresponding moab tag handle
            assert( ( *vit2 ).mTagh );
        }
    }

    // Do the same for conventional tags:
    // XCOORDS
    err = DMSLtag_define( dU.dmslModel, ( damsel_handle_ptr ) & ( dU.xcoordsTag.mTagh ),
                          DamselUtil::mtod_data_type[MB_TYPE_DOUBLE], dU.xcoordsTag.mTagh->get_name().c_str() );
    dU.tagMap.push_back( dU.xcoordsTag );
    CHK_DMSL_ERR( err, "Failure to get Damsel tag for MOAB tag" );

    // YCOORDS
    err = DMSLtag_define( dU.dmslModel, ( damsel_handle_ptr ) & ( dU.ycoordsTag.mTagh ),
                          DamselUtil::mtod_data_type[MB_TYPE_DOUBLE], dU.ycoordsTag.mTagh->get_name().c_str() );
    dU.tagMap.push_back( dU.ycoordsTag );
    CHK_DMSL_ERR( err, "Failure to get Damsel tag for MOAB tag" );

    // ZCOORDS
    err = DMSLtag_define( dU.dmslModel, ( damsel_handle_ptr ) & ( dU.zcoordsTag.mTagh ),
                          DamselUtil::mtod_data_type[MB_TYPE_DOUBLE], dU.zcoordsTag.mTagh->get_name().c_str() );
    dU.tagMap.push_back( dU.zcoordsTag );
    CHK_DMSL_ERR( err, "Failure to get Damsel tag for MOAB tag" );

    // COLL_FLAGS
    err = DMSLtag_define( dU.dmslModel, ( damsel_handle_ptr ) & ( dU.collFlagsTag.mTagh ),
                          DamselUtil::mtod_data_type[MB_TYPE_INTEGER], dU.collFlagsTag.mTagh->get_name().c_str() );
    dU.tagMap.push_back( dU.collFlagsTag );
    CHK_DMSL_ERR( err, "Failure to get Damsel tag for MOAB tag" );

    /*
      SKIP PARENTS/CHILDREN FOR NOW, UNTIL WE HAVE VAR LENGTH TAGS IN DAMSEL

      // PARENTS
      dU.parentsTagPair.second = DMSLtag_define(dU.dmslModel,
      (damsel_handle_ptr)&(dU.collFlagsTagPair.first),
                                               DamselUtil::mtod_data_type[(dU.collFlagsTagPair.first)->get_data_type()],
                                               (dU.parentsTagPair.first)->get_name().c_str());
      if (DAMSEL_TAG_INVALID == dtag)
        MB_SET_ERR(MB_FAILURE, "Failure to get Damsel tag for MOAB tag " <<
      (dU.parentsTagPair.first)->get_name());

      // CHILDREN
      dU.childrenTagPair.second = DMSLtag_define(dU.dmslModel,
      (damsel_handle_ptr)&(dU.collFlagsTagPair.first),
                                               DamselUtil::mtod_data_type[(dU.collFlagsTagPair.first)->get_data_type()],
                                               (dU.childrenTagPair.first)->get_name().c_str());
      if (DAMSEL_TAG_INVALID == dtag)
        MB_SET_ERR(MB_FAILURE, "Failure to get Damsel tag for MOAB tag " <<
      (dU.childrenTagPair.first)->get_name());
    */

    // Map the tag handles in one big call
    int num_tags = dU.tagMap.size();
    std::vector< Tag > moab_taghs;
    moab_taghs.reserve( num_tags );
    for( std::vector< DamselUtil::tinfo >::iterator vit = dU.tagMap.begin(); vit != dU.tagMap.end(); ++vit )
    {
        moab_taghs.push_back( ( *vit ).mTagh );
    }

    damsel_container mtags =
        DMSLcontainer_create_vector( dU.dmslModel, (damsel_handle_ptr)&moab_taghs[0], moab_taghs.size() );
    std::cerr << "MOAB: created model container: mtags = " << mtags << std::endl;

    err = DMSLmodel_map_handles_inventing_file_handles( mtags );
    CHK_DMSL_ERR( err, "Failed to map tag handles" );

    err = DMSLcontainer_release( mtags );
    CHK_DMSL_ERR( err, "Problem releasing tag handle container" );

    return MB_SUCCESS;
}

ErrorCode WriteDamsel::write_vertices( RangeSeqIntersectIter& rsi )
{
    // Write the vertices; these vertices will be in the same sequence and will be contiguous,
    // guaranteed
    EntityHandle start_vert = rsi.get_start_handle(), end_vert = rsi.get_end_handle();

    // Create a damsel container for these vertex handles
    damsel_container vertex_cont =
        DMSLcontainer_create_sequence( dU.dmslModel, start_vert, (int)( end_vert - start_vert + 1 ), 1 );
    std::cerr << "MOAB: created model container: vertex_cont = " << vertex_cont << std::endl;
    if( DAMSEL_CONTAINER_INVALID == vertex_cont )
        MB_SET_ERR( MB_FAILURE,
                    "Failed to create vertex sequence for vertices starting with handle " << rsi.get_start_handle() );

    damsel_err_t err = DMSLmodel_map_handles_inventing_file_handles( vertex_cont );
    CHK_DMSL_ERR( err, "Failed to map handles" );

    // Define the entities to damsel
    err = DMSLentity_define( vertex_cont, DAMSEL_ENTITY_TYPE_VERTEX, 1, vertex_cont );
    CHK_DMSL_ERR( err, "Failure in DMSLentity_define for vertices starting with handle " << rsi.get_start_handle() );

    // Get the vertex coordinates storage locations and pass to damsel
    Range vert_range( start_vert, end_vert );
    double *xcoords = NULL, *ycoords = NULL, *zcoords = NULL;
    int count;
    ErrorCode rval = mbImpl->coords_iterate( vert_range.begin(), vert_range.end(), xcoords, ycoords, zcoords, count );MB_CHK_SET_ERR( rval,
                    "Failed to get coordinate iterator for vertices starting with handle " << rsi.get_start_handle() );
    if( count != (int)vert_range.size() )
    {
        MB_SET_ERR( MB_FAILURE, "Vertex subrange not in the same sequence for vertices starting with handle "
                                    << rsi.get_start_handle() );
    }

    if( xcoords && !ycoords && !zcoords )
    {
        // Interleaved

        // Map the data to damsel
        err = DMSLmodel_map_tag( xcoords, vertex_cont, (damsel_handle_ptr)&dU.xcoordsTag.mTagh );
        CHK_DMSL_ERR( err, "Failed to assign vertex coordinates tag for vertices starting with handle "
                               << rsi.get_start_handle() );
    }
    else
    {
        // Map the data to damsel
        err = DMSLmodel_map_tag( xcoords, vertex_cont, (damsel_handle_ptr)&dU.xcoordsTag.mTagh );
        CHK_DMSL_ERR( err, "Failed to assign vertex x coordinates tag for vertices starting with handle "
                               << rsi.get_start_handle() );
        err = DMSLmodel_map_tag( ycoords, vertex_cont, (damsel_handle_ptr)&dU.ycoordsTag.mTagh );
        CHK_DMSL_ERR( err, "Failed to assign vertex y coordinates tag for vertices starting with handle "
                               << rsi.get_start_handle() );
        err = DMSLmodel_map_tag( zcoords, vertex_cont, (damsel_handle_ptr)&dU.zcoordsTag.mTagh );
        CHK_DMSL_ERR( err, "Failed to assign vertex z coordinates tag for vertices starting with handle "
                               << rsi.get_start_handle() );
    }

    // Write/map dense tags
    rval = map_dense_tags( rsi, vertex_cont );MB_CHK_ERR( rval );

    err = DMSLcontainer_release( vertex_cont );
    CHK_DMSL_ERR( err, "Problem releasing vertex handle container" );

    return MB_SUCCESS;
}

ErrorCode WriteDamsel::write_entities( RangeSeqIntersectIter& rsi )
{
    // Write the entities; these entities will be in the same sequence and will be contiguous,
    // guaranteed
    EntityHandle start_ent = rsi.get_start_handle(), end_ent = rsi.get_end_handle();

    // Create a damsel container for these entity handles
    damsel_container ent_cont;
    ent_cont = DMSLcontainer_create_sequence( dU.dmslModel, start_ent, (int)( end_ent - start_ent + 1 ), 1 );
    std::cerr << "MOAB: created model container: ent_cont = " << ent_cont << std::endl;
    if( DAMSEL_CONTAINER_INVALID == ent_cont ) MB_SET_ERR( MB_FAILURE, "Bad sequence returned by Damsel" );

    damsel_err_t err = DMSLmodel_map_handles_inventing_file_handles( ent_cont );
    CHK_DMSL_ERR( err, "Failed to map handles" );

    // Get # verts per entity and entity type
    EntityType etype = mbImpl->type_from_handle( start_ent );
    assert( MBMAXTYPE != etype );
    int num_connect = rsi.get_sequence()->values_per_entity();
    assert( 0 < num_connect );

    // Get the connectivity storage location and pass to damsel
    Range ent_range( start_ent, end_ent );
    int count;
    EntityHandle* connect;
    int verts_per_ent;
    ErrorCode rval = mbImpl->connect_iterate( ent_range.begin(), ent_range.end(), connect, verts_per_ent, count );MB_CHK_SET_ERR( rval,
                    "Failed to get connect iterator for entities starting with handle " << rsi.get_start_handle() );
    if( count != (int)ent_range.size() )
        MB_SET_ERR( MB_FAILURE, "Entity subrange not in the same sequence for entities starting with handle "
                                    << rsi.get_start_handle() );

    // Define the entities to damsel
    err = DMSLentity_define_fast( ent_cont, DamselUtil::mtod_entity_type[etype], num_connect, (damsel_handle*)connect );
    CHK_DMSL_ERR( err, "DMSLentity_define failed for entities starting with handle " << rsi.get_start_handle() );

    // Write dense tags
    rval = map_dense_tags( rsi, ent_cont );MB_CHK_ERR( rval );

    err = DMSLcontainer_release( ent_cont );
    CHK_DMSL_ERR( err, "Problem releasing entity handle container" );

    return MB_SUCCESS;
}

ErrorCode WriteDamsel::map_dense_tags( RangeSeqIntersectIter& rsi, damsel_container& ent_cont )
{
    // All dense_tags have been initialized before this, so here we just go through
    // them and map data if there is any
    const unsigned char* val_ptr;
    ErrorCode rval = MB_SUCCESS;
    std::vector< DamselUtil::tinfo >::iterator tagit;
    damsel_err_t err;
    for( tagit = dU.tagMap.begin(); tagit != dU.tagMap.end(); ++tagit )
    {
        if( ( *tagit ).tagType != MB_TAG_DENSE ) continue;

        // Get a ptr to memory for this tag/sequence
        DenseTag* dtag = dynamic_cast< DenseTag* >( ( *tagit ).mTagh );
        assert( dtag );
        rval = dtag->get_array( rsi.get_sequence(), val_ptr );MB_CHK_SET_ERR( rval, "Failed to get tag coordinates pointer for vertices starting with handle "
                                  << rsi.get_start_handle() );

        // If ptr is NULL, no data for this tag in this sequence
        if( !val_ptr ) continue;

        // Else, register with damsel
        err = DMSLmodel_map_tag( (void*)val_ptr, ent_cont, (damsel_handle_ptr)&dtag );
        CHK_DMSL_ERR( err,
                      "Failed to write coordinates tag for vertices starting with handle " << rsi.get_start_handle() );
    }

    return rval;
}

ErrorCode WriteDamsel::map_sparse_tags()
{
    // All sparse_tags have been initialized before this, so here we just go through
    // them and map data if there is any
    ErrorCode rval = MB_SUCCESS;
    damsel_err_t err;
    std::vector< DamselUtil::tinfo >::iterator tagit;
    std::vector< unsigned char > tag_values;
    std::vector< EntityHandle > tagged_ents;
    damsel_container ent_cont;
    for( tagit = dU.tagMap.begin(); tagit != dU.tagMap.end(); ++tagit )
    {
        if( ( *tagit ).tagType != MB_TAG_SPARSE ) continue;
        // Get a ptr to memory for this tag/sequence
        SparseTag* stag = dynamic_cast< SparseTag* >( ( *tagit ).mTagh );
        assert( stag );
        Range output_ents;
        rval = stag->get_tagged_entities( sequenceManager, output_ents );MB_CHK_SET_ERR( rval, "Trouble getting tagged entities for tag " << stag->get_name() );

        // If no entities have this tag set, don't map it
        if( output_ents.empty() ) continue;

        // Else, register with damsel
        // Allocate space for and get values
        tag_values.resize( stag->get_size() * output_ents.size() );
        rval = mbImpl->tag_get_data( stag, output_ents, &tag_values[0] );MB_CHK_SET_ERR( rval, "Trouble getting tag values for tag " << stag->get_name() );

        // Build a vector of entity handles from the range, and a container from that
        tagged_ents.resize( output_ents.size() );
        std::copy( output_ents.begin(), output_ents.end(), tagged_ents.begin() );
        ent_cont = DMSLcontainer_create_vector( dU.dmslModel, (damsel_handle_ptr)&tagged_ents[0], tagged_ents.size() );
        std::cerr << "MOAB: created model container: sparse_tag_ent_cont = " << ent_cont << std::endl;
        if( DAMSEL_CONTAINER_INVALID == ent_cont )
            MB_SET_ERR( MB_FAILURE, "Trouble creating entity handle container for tag " << stag->get_name() );

        // Now map it
        err = DMSLmodel_map_tag( (void*)&tag_values[0], ent_cont, (damsel_handle_ptr)&stag );
        CHK_DMSL_ERR( err, "Failed to write tag " << stag->get_name() );

        err = DMSLcontainer_release( ent_cont );
        CHK_DMSL_ERR( err, "Problem releasing entity handle container" );
    }

    return rval;
}

ErrorCode WriteDamsel::write_sets( RangeSeqIntersectIter& rsi )
{
    // Write the sets
    ErrorCode rval = MB_SUCCESS;
    std::vector< EntityHandle > ents;
    damsel_container mcont;
    damsel_err_t err;
    unsigned int i, num_sets = rsi.get_end_handle() - rsi.get_start_handle() + 1;
    std::vector< unsigned int > set_flags( num_sets, 0 );
    EntityHandle seth;
    for( seth = rsi.get_start_handle(), i = 0; seth <= rsi.get_end_handle(); seth++, i++ )
    {
        // Get all the entities in the set
        ents.clear();
        rval = mbImpl->get_entities_by_handle( seth, ents );MB_CHK_SET_ERR( rval, "get_entities_by_handle failed for set " << seth );
        if( !ents.empty() )
        {
            mcont = DMSLcontainer_create_vector( dU.dmslModel, (damsel_handle*)&ents[0], ents.size() );
        }
        else
        {
            mcont = DMSLcontainer_create_vector( dU.dmslModel, (damsel_handle*)NULL, 0 );
        }
        std::cerr << "MOAB: created model container: sets_cont = " << mcont << std::endl;

        // Get the set type (range or set)
        unsigned int opts;
        rval = mbImpl->get_meshset_options( seth, opts );MB_CHK_SET_ERR( rval, "Failed to get options for meshset " << seth );
        damsel_collection_type coll_type =
            ( opts & MESHSET_SET ? DAMSEL_HANDLE_COLLECTION_TYPE_SET : DAMSEL_HANDLE_COLLECTION_TYPE_VECTOR );

        // Parents/children...

        // Set flags
        if( opts & MESHSET_TRACK_OWNER )
            set_flags[i] |= MESHSET_TRACK_OWNER;
        else
            set_flags[i] &= !MESHSET_TRACK_OWNER;

        // Create the collection
        DMSLcoll_create( dU.dmslModel, (damsel_handle_ptr)&seth, mcont, coll_type );

        // Release the container
        err = DMSLcontainer_release( mcont );
        CHK_DMSL_ERR( err, "Problem releasing set entity handle container" );
    }

    // Set the COLL_FLAGS tag, using assign (direct)
    // Make a container of set handles...
    mcont = DMSLcontainer_create_sequence( dU.dmslModel, rsi.get_start_handle(), num_sets, 1 );
    std::cerr << "MOAB: created model container: sets_cont = " << mcont << std::endl;

    // Assign the tags on them
    err = DMSLmodel_map_tag( &set_flags[0], mcont, ( damsel_handle_ptr ) & ( dU.collFlagsTag.mTagh ) );
    CHK_DMSL_ERR( err, "Failed to assign COLL_FLAGS tag for sets" );

    // Map set handles
    err = DMSLmodel_map_handles_inventing_file_handles( mcont );
    CHK_DMSL_ERR( err, "Failed to map set handles" );

    // Map other dense tags
    rval = map_dense_tags( rsi, mcont );MB_CHK_SET_ERR( rval, "Failed to map dense tags for sets" );

    return rval;
}

}  // namespace moab