Actual source code: dmmoab.cxx

petsc-dev 2014-02-02
Report Typos and Errors
  1: #include <petsc-private/dmimpl.h> /*I  "petscdm.h"   I*/
  2: #include <petsc-private/vecimpl.h> /*I  "petscdm.h"   I*/

  4: #include <petscdmmoab.h>
  5: #include <MBTagConventions.hpp>
  6: #include <sstream>

  8: typedef struct {
  9:   PetscInt bs; /* Number of degrees of freedom on each entity, aka tag size in moab */
 10:   PetscBool icreatedinstance; /* true if DM created moab instance internally, will destroy instance in DMDestroy */
 11:   moab::ParallelComm *pcomm;
 12:   moab::Interface *mbiface;
 13:   moab::Tag ltog_tag; /* moab supports "global id" tags, which are usually local to global numbering */
 14:   moab::Range range;
 15: } DM_Moab;

 17: typedef struct {
 18:   moab::Interface    *mbiface;
 19:   moab::ParallelComm *pcomm;
 20:   moab::Range         tag_range; /* entities to which this tag applies */
 21:   moab::Tag           tag;
 22:   moab::Tag           ltog_tag;
 23:   PetscInt            tag_size;
 24:   PetscBool           new_tag;
 25:   PetscBool           serial;

 27: } Vec_MOAB;

 31: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
 32: {
 33:   PetscErrorCode  ierr;
 34:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

 39:   PetscInt block_size = ((DM_Moab*)dm->data)->bs;
 40:   moab::Tag tag = 0;
 41:   DMMoabCreateVector(dm,tag,block_size,dmmoab->range,PETSC_FALSE,PETSC_TRUE,gvec);
 42:   return(0);
 43: }


 48: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *gvec)
 49: {
 50:   PetscErrorCode  ierr;
 51:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

 54:   PetscInt bs = 1;
 57:   moab::Tag tag = 0;
 58:   DMMoabCreateVector(dm,tag,bs,dmmoab->range,PETSC_TRUE,PETSC_TRUE,gvec);
 59:   return(0);
 60: }

 64: PetscErrorCode DMDestroy_Moab(DM dm)
 65: {

 70:   if (((DM_Moab*)dm->data)->icreatedinstance) {
 71:     delete ((DM_Moab*)dm->data)->mbiface;
 72:     ((DM_Moab*)dm->data)->mbiface = NULL;
 73:     ((DM_Moab*)dm->data)->pcomm = NULL;
 74:     ((DM_Moab*)dm->data)->range.~Range();
 75:   }
 76:   PetscFree(dm->data);
 77:   return(0);
 78: }

 82: PETSC_EXTERN PetscErrorCode DMCreate_Moab(DM dm)
 83: {
 84:   DM_Moab        *moab;

 89:   PetscNewLog(dm,&moab);
 90:   dm->data = moab;
 91:   new (moab) DM_Moab();

 93:   dm->ops->createglobalvector              = DMCreateGlobalVector_Moab;
 94:   dm->ops->createlocalvector               = DMCreateLocalVector_Moab;
 95:   dm->ops->destroy                         = DMDestroy_Moab;
 96:   return(0);
 97: }

101: /*@
102:   DMMoabCreate - Creates a DMMoab object, which encapsulates a moab instance

104:   Collective on MPI_Comm

106:   Input Parameter:
107: . comm - The communicator for the DMMoab object

109:   Output Parameter:
110: . moab  - The DMMoab object

112:   Level: beginner

114: .keywords: DMMoab, create
115: @*/
116: PetscErrorCode DMMoabCreate(MPI_Comm comm, DM *moab)
117: {

122:   DMCreate(comm, moab);
123:   DMSetType(*moab, DMMOAB);
124:   return(0);
125: }

129: /*@
130:   DMMoabCreate - Creates a DMMoab object, optionally from an instance and other data

132:   Collective on MPI_Comm

134:   Input Parameter:
135: . comm - The communicator for the DMMoab object
136: . moab - (ptr to) the MOAB Instance; if passed in NULL, MOAB instance is created inside PETSc, and destroyed
137:          along with the DMMoab
138: . pcomm - (ptr to) a ParallelComm; if NULL, creates one internally for the whole communicator
139: . ltog_tag - A tag to use to retrieve global id for an entity; if 0, will use GLOBAL_ID_TAG_NAME/tag
140: . range - If non-NULL, contains range of entities to which DOFs will be assigned

142:   Output Parameter:
143: . moab  - The DMMoab object

145:   Level: beginner

147: .keywords: DMMoab, create
148: @*/
149: PetscErrorCode DMMoabCreateMoab(MPI_Comm comm, moab::Interface *mbiface, moab::ParallelComm *pcomm, moab::Tag ltog_tag, moab::Range *range, DM *moab)
150: {
152:   DM_Moab        *dmmoab;

156:   DMMoabCreate(comm, moab);
157:   dmmoab = (DM_Moab*)(*moab)->data;

159:   if (!mbiface) {
160:     mbiface = new moab::Core();
161:     dmmoab->icreatedinstance = PETSC_TRUE;
162:   }
163:   else
164:     dmmoab->icreatedinstance = PETSC_FALSE;

166:   if (!pcomm) {
167:     PetscInt rank, nprocs;
168:     MPI_Comm_rank(comm, &rank);
169:     MPI_Comm_size(comm, &nprocs);
170:     pcomm = new moab::ParallelComm(mbiface, comm);
171:   }

173:     // do the initialization of the DM
174:   dmmoab->bs       = 0;
175:   dmmoab->pcomm    = pcomm;
176:   dmmoab->mbiface    = mbiface;
177:   dmmoab->ltog_tag = ltog_tag;

179:   DMMoabSetInterface(*moab, mbiface);
180:   if (!pcomm) pcomm = new moab::ParallelComm(mbiface, comm);
181:   DMMoabSetParallelComm(*moab, pcomm);
182:   if (!ltog_tag) {
183:     moab::ErrorCode merr = mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, ltog_tag);MBERRNM(merr);
184:   }
185:   if (ltog_tag) {
186:     DMMoabSetLocalToGlobalTag(*moab, ltog_tag);
187:   }
188:   if (range) {
189:     DMMoabSetRange(*moab, *range);
190:   }
191:   return(0);
192: }

196: /*@
197:   DMMoabSetParallelComm - Set the ParallelComm used with this DMMoab

199:   Collective on MPI_Comm

201:   Input Parameter:
202: . dm    - The DMMoab object being set
203: . pcomm - The ParallelComm being set on the DMMoab

205:   Level: beginner

207: .keywords: DMMoab, create
208: @*/
209: PetscErrorCode DMMoabSetParallelComm(DM dm,moab::ParallelComm *pcomm)
210: {
213:   ((DM_Moab*)dm->data)->pcomm = pcomm;
214:   ((DM_Moab*)dm->data)->mbiface = pcomm->get_moab();
215:   return(0);
216: }


221: /*@
222:   DMMoabGetParallelComm - Get the ParallelComm used with this DMMoab

224:   Collective on MPI_Comm

226:   Input Parameter:
227: . dm    - The DMMoab object being set

229:   Output Parameter:
230: . pcomm - The ParallelComm for the DMMoab

232:   Level: beginner

234: .keywords: DMMoab, create
235: @*/
236: PetscErrorCode DMMoabGetParallelComm(DM dm,moab::ParallelComm **pcomm)
237: {
240:   *pcomm = ((DM_Moab*)dm->data)->pcomm;
241:   return(0);
242: }


247: /*@
248:   DMMoabSetInterface - Set the MOAB instance used with this DMMoab

250:   Collective on MPI_Comm

252:   Input Parameter:
253: . dm      - The DMMoab object being set
254: . mbiface - The MOAB instance being set on this DMMoab

256:   Level: beginner

258: .keywords: DMMoab, create
259: @*/
260: PetscErrorCode DMMoabSetInterface(DM dm,moab::Interface *mbiface)
261: {
264:   ((DM_Moab*)dm->data)->pcomm = NULL;
265:   ((DM_Moab*)dm->data)->mbiface = mbiface;
266:   return(0);
267: }


272: /*@
273:   DMMoabGetInterface - Get the MOAB instance used with this DMMoab

275:   Collective on MPI_Comm

277:   Input Parameter:
278: . dm      - The DMMoab object being set

280:   Output Parameter:
281: . mbiface - The MOAB instance set on this DMMoab

283:   Level: beginner

285: .keywords: DMMoab, create
286: @*/
287: PetscErrorCode DMMoabGetInterface(DM dm,moab::Interface **mbiface)
288: {
289:   PetscErrorCode   ierr;
290:   static PetscBool cite = PETSC_FALSE;

294:   PetscCitationsRegister("@techreport{tautges_moab:_2004,\n  type = {{SAND2004-1592}},\n  title = {{MOAB:} A Mesh-Oriented Database},  institution = {Sandia National Laboratories},\n  author = {Tautges, T. J. and Meyers, R. and Merkley, K. and Stimpson, C. and Ernst, C.},\n  year = {2004},  note = {Report}\n}\n",&cite);
295:   *mbiface = ((DM_Moab*)dm->data)->mbiface;
296:   return(0);
297: }


302: /*@
303:   DMMoabSetRange - Set the entities having DOFs on this DMMoab

305:   Collective on MPI_Comm

307:   Input Parameter:
308: . dm    - The DMMoab object being set
309: . range - The entities treated by this DMMoab

311:   Level: beginner

313: .keywords: DMMoab, create
314: @*/
315: PetscErrorCode DMMoabSetRange(DM dm,moab::Range range)
316: {
319:   ((DM_Moab*)dm->data)->range = range;
320:   return(0);
321: }


326: /*@
327:   DMMoabGetRange - Get the entities having DOFs on this DMMoab

329:   Collective on MPI_Comm

331:   Input Parameter:
332: . dm    - The DMMoab object being set

334:   Output Parameter:
335: . range - The entities treated by this DMMoab

337:   Level: beginner

339: .keywords: DMMoab, create
340: @*/
341: PetscErrorCode DMMoabGetRange(DM dm,moab::Range *range)
342: {
345:   *range = ((DM_Moab*)dm->data)->range;
346:   return(0);
347: }

351: /*@
352:   DMMoabSetLocalToGlobalTag - Set the tag used for local to global numbering

354:   Collective on MPI_Comm

356:   Input Parameter:
357: . dm      - The DMMoab object being set
358: . ltogtag - The MOAB tag used for local to global ids

360:   Level: beginner

362: .keywords: DMMoab, create
363: @*/
364: PetscErrorCode DMMoabSetLocalToGlobalTag(DM dm,moab::Tag ltogtag)
365: {
368:   ((DM_Moab*)dm->data)->ltog_tag = ltogtag;
369:   return(0);
370: }


375: /*@
376:   DMMoabGetLocalToGlobalTag - Get the tag used for local to global numbering

378:   Collective on MPI_Comm

380:   Input Parameter:
381: . dm      - The DMMoab object being set

383:   Output Parameter:
384: . ltogtag - The MOAB tag used for local to global ids

386:   Level: beginner

388: .keywords: DMMoab, create
389: @*/
390: PetscErrorCode DMMoabGetLocalToGlobalTag(DM dm,moab::Tag *ltog_tag)
391: {
394:   *ltog_tag = ((DM_Moab*)dm->data)->ltog_tag;
395:   return(0);
396: }


401: /*@
402:   DMMoabSetBlockSize - Set the block size used with this DMMoab

404:   Collective on MPI_Comm

406:   Input Parameter:
407: . dm - The DMMoab object being set
408: . bs - The block size used with this DMMoab

410:   Level: beginner

412: .keywords: DMMoab, create
413: @*/
414: PetscErrorCode DMMoabSetBlockSize(DM dm,PetscInt bs)
415: {
418:   ((DM_Moab*)dm->data)->bs = bs;
419:   return(0);
420: }


425: /*@
426:   DMMoabGetBlockSize - Get the block size used with this DMMoab

428:   Collective on MPI_Comm

430:   Input Parameter:
431: . dm - The DMMoab object being set

433:   Output Parameter:
434: . bs - The block size used with this DMMoab

436:   Level: beginner

438: .keywords: DMMoab, create
439: @*/
440: PetscErrorCode DMMoabGetBlockSize(DM dm,PetscInt *bs)
441: {
444:   *bs = ((DM_Moab*)dm->data)->bs;
445:   return(0);
446: }


449: // declare for use later but before they're defined
450: PetscErrorCode DMMoab_VecUserDestroy(void *user);
451: PetscErrorCode DMMoab_VecDuplicate(Vec x,Vec *y);
452: PetscErrorCode DMMoab_CreateTagName(const moab::ParallelComm *pcomm,std::string& tag_name);
453: PetscErrorCode DMMoab_CreateVector(moab::Interface *iface,moab::ParallelComm *pcomm,moab::Tag tag,PetscInt tag_size,moab::Tag ltog_tag,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec);

457: /*@
458:   DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities

460:   Collective on MPI_Comm

462:   Input Parameter:
463: . dm          - The DMMoab object being set
464: . tag         - If non-zero, block size will be taken from the tag size
465: . tag_size    - If tag was zero, this parameter specifies the block size; unique tag name will be generated automatically
466: . range       - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
467: . serial      - If true, this is a serial Vec, otherwise a parallel one
468: . destroy_tag - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB

470:   Output Parameter:
471: . vec         - The created vector

473:   Level: beginner

475: .keywords: DMMoab, create
476: @*/
477: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,PetscInt tag_size,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec)
478: {
479:   PetscErrorCode     ierr;


483:   DM_Moab *dmmoab = (DM_Moab*)dm->data;
484:   moab::ParallelComm *pcomm = dmmoab->pcomm;
485:   moab::Interface *mbiface = dmmoab->mbiface;
486:   moab::Tag ltog_tag = dmmoab->ltog_tag;

488:   if (!tag && !tag_size) {
489:     PetscFunctionReturn(PETSC_ERR_ARG_WRONG);
490:   }
491:   else {
492:     DMMoab_CreateVector(mbiface,pcomm,tag,tag_size,ltog_tag,range,serial,destroy_tag,vec);
493:   }
494:   return(0);
495: }


500: PetscErrorCode DMMoab_CreateVector(moab::Interface *mbiface,moab::ParallelComm *pcomm,moab::Tag tag,PetscInt tag_size,moab::Tag ltog_tag,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec)
501: {
502:   PetscErrorCode     ierr;
503:   moab::ErrorCode    merr;


507:   if (!tag) {
508:     std::string tag_name;
509:     DMMoab_CreateTagName(pcomm,tag_name);

511:       // Create the default value for the tag (all zeros):
512:     std::vector<PetscScalar> default_value(tag_size, 0.0);

514:       // Create the tag:
515:     merr = mbiface->tag_get_handle(tag_name.c_str(),tag_size,moab::MB_TYPE_DOUBLE,tag,
516:                                    moab::MB_TAG_DENSE | moab::MB_TAG_CREAT,default_value.data());MBERRNM(merr);
517:   }
518:   else {

520:       // Make sure the tag data is of type "double":
521:     moab::DataType tag_type;
522:     merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
523:     if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
524:   }

526:     // Create the MOAB internal data object
527:   Vec_MOAB *vmoab;
528:   PetscMalloc(sizeof(Vec_MOAB),&vmoab);
529:   new (vmoab) Vec_MOAB();
530:   vmoab->tag = tag;
531:   vmoab->ltog_tag = ltog_tag;
532:   vmoab->mbiface = mbiface;
533:   vmoab->pcomm = pcomm;
534:   vmoab->tag_range = range;
535:   vmoab->new_tag = destroy_tag;
536:   vmoab->serial = serial;
537:   merr = mbiface->tag_get_length(tag,vmoab->tag_size);MBERR("tag_get_size", merr);

539:     // Call tag_iterate. This will cause MOAB to allocate memory for the
540:     // tag data if it hasn't already happened:
541:   int  count;
542:   void *void_ptr;
543:   merr = mbiface->tag_iterate(tag,range.begin(),range.end(),count,void_ptr);MBERRNM(merr);

545:     // Check to make sure the tag data is in a single sequence:
546:   if ((unsigned)count != range.size()) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Can only create MOAB Vector for single sequence");
547:   PetscScalar *data_ptr = (PetscScalar*)void_ptr;

549:     // Create the PETSc Vector:
550:   if(!serial) {
551:       // This is an MPI Vector:
552:     VecCreateMPIWithArray(vmoab->pcomm->comm(),vmoab->tag_size,vmoab->tag_size*range.size(),
553:                                  PETSC_DECIDE,data_ptr,vec);

555:       // Vector created, manually set local to global mapping:
556:     ISLocalToGlobalMapping ltog;
557:     PetscInt               *gindices = new PetscInt[range.size()];
558:     PetscInt               count = 0;
559:     moab::Range::iterator  iter;
560:     for(iter = range.begin(); iter != range.end(); iter++) {
561:       int dof;
562:       merr = mbiface->tag_get_data(ltog_tag,&(*iter),1,&dof);MBERRNM(merr);
563:       gindices[count] = dof;
564:       count++;
565:     }

567:     ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,range.size(),gindices,
568:                                         PETSC_COPY_VALUES,&ltog);
569:     VecSetLocalToGlobalMappingBlock(*vec,ltog);

571:       // Clean up:
572:     ISLocalToGlobalMappingDestroy(&ltog);
573:     delete [] gindices;
574:   } else {
575:       // This is a serial vector:
576:     VecCreateSeqWithArray(PETSC_COMM_SELF,vmoab->tag_size,vmoab->tag_size*range.size(),data_ptr,vec);
577:   }


580:   PetscContainer moabdata;
581:   PetscContainerCreate(PETSC_COMM_SELF,&moabdata);
582:   PetscContainerSetPointer(moabdata,vmoab);
583:   PetscContainerSetUserDestroy(moabdata,DMMoab_VecUserDestroy);
584:   PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
585:   (*vec)->ops->duplicate = DMMoab_VecDuplicate;

587:   PetscContainerDestroy(&moabdata);
588:   return(0);
589: }

593: /*@
594:   DMMoabGetVecTag - Get the MOAB tag associated with this Vec

596:   Collective on MPI_Comm

598:   Input Parameter:
599: . vec - Vec being queried

601:   Output Parameter:
602: . tag - Tag associated with this Vec

604:   Level: beginner

606: .keywords: DMMoab, create
607: @*/
608: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
609: {
610:   PetscContainer  moabdata;
611:   Vec_MOAB        *vmoab;
612:   PetscErrorCode  ierr;


616:   // Get the MOAB private data:
617:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
618:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

620:   *tag = vmoab->tag;

622:   return(0);
623: }


628: /*@
629:   DMMoabGetVecRange - Get the MOAB entities associated with this Vec

631:   Collective on MPI_Comm

633:   Input Parameter:
634: . vec   - Vec being queried

636:   Output Parameter:
637: . range - Entities associated with this Vec

639:   Level: beginner

641: .keywords: DMMoab, create
642: @*/
643: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
644: {
645:   PetscContainer  moabdata;
646:   Vec_MOAB        *vmoab;
647:   PetscErrorCode  ierr;


651:   // Get the MOAB private data:
652:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
653:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

655:   *range = vmoab->tag_range;

657:   return(0);
658: }


663: PetscErrorCode DMMoab_VecDuplicate(Vec x,Vec *y)
664: {

670:   // Get the Vec_MOAB struct for the original vector:
671:   PetscContainer  moabdata;
672:   Vec_MOAB        *vmoab;
673:   PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
674:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

676:   DMMoab_CreateVector(vmoab->mbiface,vmoab->pcomm,0,vmoab->tag_size,vmoab->ltog_tag,vmoab->tag_range,vmoab->serial,PETSC_TRUE,y);
677:   return(0);
678: }


683: /*  DMMoab_CreateTagName
684:  *
685:  *  Creates a unique tag name that will be shared across processes. If
686:  *  pcomm is NULL, then this is a serial vector. A unique tag name
687:  *  will be returned in tag_name in either case.
688:  *
689:  *  The tag names have the format _PETSC_VEC_N where N is some integer.
690:  */
691: PetscErrorCode DMMoab_CreateTagName(const moab::ParallelComm *pcomm,std::string& tag_name)
692: {
693:   moab::ErrorCode mberr;
694:   PetscErrorCode  ierr;

697:   const std::string PVEC_PREFIX      = "_PETSC_VEC_";
698:   const PetscInt    PVEC_PREFIX_SIZE = PVEC_PREFIX.size();

700:   // Check to see if there are any PETSc vectors defined:
701:   const moab::Interface  *mbiface = pcomm->get_moab();
702:   std::vector<moab::Tag> tags;
703:   PetscInt               n = 0;
704:   mberr = mbiface->tag_get_tags(tags);MBERRNM(mberr);
705:   for(unsigned i = 0; i < tags.size(); i++) {
706:     std::string s;
707:     mberr = mbiface->tag_get_name(tags[i],s);MBERRNM(mberr);
708:     if(s.find(PVEC_PREFIX) != std::string::npos){
709:       // This tag represents a PETSc vector. Find the vector number:
710:       PetscInt m;
711:       std::istringstream(s.substr(PVEC_PREFIX_SIZE)) >> m;
712:       if(m >= n) n = m+1;
713:     }
714:   }

716:   // Make sure that n is consistent across all processes:
717:   PetscInt global_n;
718:   MPI_Comm comm = PETSC_COMM_SELF;
719:   if(pcomm) comm = pcomm->comm();
720:   MPI_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,comm);

722:   // Set the answer and return:
723:   std::ostringstream ss;
724:   ss << PVEC_PREFIX << global_n;
725:   tag_name = ss.str();
726:   return(0);
727: }


732: PetscErrorCode DMMoab_VecUserDestroy(void *user)
733: {
734:   Vec_MOAB        *vmoab;
735:   PetscErrorCode  ierr;
736:   moab::ErrorCode merr;

739:   vmoab = (Vec_MOAB*)user;
740:   vmoab->tag_range.~Range();
741:   if(vmoab->new_tag) {
742:     // Tag created via a call to VecDuplicate, delete the underlying tag in MOAB...
743:     merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
744:   }

746:   PetscFree(vmoab);
747:   return(0);
748: }