Actual source code: dmmoab.cxx
petsc-dev 2014-02-02
1: #include <petsc-private/dmimpl.h> /*I "petscdm.h" I*/
2: #include <petsc-private/vecimpl.h> /*I "petscdm.h" I*/
4: #include <petscdmmoab.h>
5: #include <MBTagConventions.hpp>
6: #include <sstream>
8: typedef struct {
9: PetscInt bs; /* Number of degrees of freedom on each entity, aka tag size in moab */
10: PetscBool icreatedinstance; /* true if DM created moab instance internally, will destroy instance in DMDestroy */
11: moab::ParallelComm *pcomm;
12: moab::Interface *mbiface;
13: moab::Tag ltog_tag; /* moab supports "global id" tags, which are usually local to global numbering */
14: moab::Range range;
15: } DM_Moab;
17: typedef struct {
18: moab::Interface *mbiface;
19: moab::ParallelComm *pcomm;
20: moab::Range tag_range; /* entities to which this tag applies */
21: moab::Tag tag;
22: moab::Tag ltog_tag;
23: PetscInt tag_size;
24: PetscBool new_tag;
25: PetscBool serial;
27: } Vec_MOAB;
31: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
32: {
33: PetscErrorCode ierr;
34: DM_Moab *dmmoab = (DM_Moab*)dm->data;
39: PetscInt block_size = ((DM_Moab*)dm->data)->bs;
40: moab::Tag tag = 0;
41: DMMoabCreateVector(dm,tag,block_size,dmmoab->range,PETSC_FALSE,PETSC_TRUE,gvec);
42: return(0);
43: }
48: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *gvec)
49: {
50: PetscErrorCode ierr;
51: DM_Moab *dmmoab = (DM_Moab*)dm->data;
54: PetscInt bs = 1;
57: moab::Tag tag = 0;
58: DMMoabCreateVector(dm,tag,bs,dmmoab->range,PETSC_TRUE,PETSC_TRUE,gvec);
59: return(0);
60: }
64: PetscErrorCode DMDestroy_Moab(DM dm)
65: {
70: if (((DM_Moab*)dm->data)->icreatedinstance) {
71: delete ((DM_Moab*)dm->data)->mbiface;
72: ((DM_Moab*)dm->data)->mbiface = NULL;
73: ((DM_Moab*)dm->data)->pcomm = NULL;
74: ((DM_Moab*)dm->data)->range.~Range();
75: }
76: PetscFree(dm->data);
77: return(0);
78: }
82: PETSC_EXTERN PetscErrorCode DMCreate_Moab(DM dm)
83: {
84: DM_Moab *moab;
89: PetscNewLog(dm,&moab);
90: dm->data = moab;
91: new (moab) DM_Moab();
93: dm->ops->createglobalvector = DMCreateGlobalVector_Moab;
94: dm->ops->createlocalvector = DMCreateLocalVector_Moab;
95: dm->ops->destroy = DMDestroy_Moab;
96: return(0);
97: }
101: /*@
102: DMMoabCreate - Creates a DMMoab object, which encapsulates a moab instance
104: Collective on MPI_Comm
106: Input Parameter:
107: . comm - The communicator for the DMMoab object
109: Output Parameter:
110: . moab - The DMMoab object
112: Level: beginner
114: .keywords: DMMoab, create
115: @*/
116: PetscErrorCode DMMoabCreate(MPI_Comm comm, DM *moab)
117: {
122: DMCreate(comm, moab);
123: DMSetType(*moab, DMMOAB);
124: return(0);
125: }
129: /*@
130: DMMoabCreate - Creates a DMMoab object, optionally from an instance and other data
132: Collective on MPI_Comm
134: Input Parameter:
135: . comm - The communicator for the DMMoab object
136: . moab - (ptr to) the MOAB Instance; if passed in NULL, MOAB instance is created inside PETSc, and destroyed
137: along with the DMMoab
138: . pcomm - (ptr to) a ParallelComm; if NULL, creates one internally for the whole communicator
139: . ltog_tag - A tag to use to retrieve global id for an entity; if 0, will use GLOBAL_ID_TAG_NAME/tag
140: . range - If non-NULL, contains range of entities to which DOFs will be assigned
142: Output Parameter:
143: . moab - The DMMoab object
145: Level: beginner
147: .keywords: DMMoab, create
148: @*/
149: PetscErrorCode DMMoabCreateMoab(MPI_Comm comm, moab::Interface *mbiface, moab::ParallelComm *pcomm, moab::Tag ltog_tag, moab::Range *range, DM *moab)
150: {
152: DM_Moab *dmmoab;
156: DMMoabCreate(comm, moab);
157: dmmoab = (DM_Moab*)(*moab)->data;
159: if (!mbiface) {
160: mbiface = new moab::Core();
161: dmmoab->icreatedinstance = PETSC_TRUE;
162: }
163: else
164: dmmoab->icreatedinstance = PETSC_FALSE;
166: if (!pcomm) {
167: PetscInt rank, nprocs;
168: MPI_Comm_rank(comm, &rank);
169: MPI_Comm_size(comm, &nprocs);
170: pcomm = new moab::ParallelComm(mbiface, comm);
171: }
173: // do the initialization of the DM
174: dmmoab->bs = 0;
175: dmmoab->pcomm = pcomm;
176: dmmoab->mbiface = mbiface;
177: dmmoab->ltog_tag = ltog_tag;
179: DMMoabSetInterface(*moab, mbiface);
180: if (!pcomm) pcomm = new moab::ParallelComm(mbiface, comm);
181: DMMoabSetParallelComm(*moab, pcomm);
182: if (!ltog_tag) {
183: moab::ErrorCode merr = mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, ltog_tag);MBERRNM(merr);
184: }
185: if (ltog_tag) {
186: DMMoabSetLocalToGlobalTag(*moab, ltog_tag);
187: }
188: if (range) {
189: DMMoabSetRange(*moab, *range);
190: }
191: return(0);
192: }
196: /*@
197: DMMoabSetParallelComm - Set the ParallelComm used with this DMMoab
199: Collective on MPI_Comm
201: Input Parameter:
202: . dm - The DMMoab object being set
203: . pcomm - The ParallelComm being set on the DMMoab
205: Level: beginner
207: .keywords: DMMoab, create
208: @*/
209: PetscErrorCode DMMoabSetParallelComm(DM dm,moab::ParallelComm *pcomm)
210: {
213: ((DM_Moab*)dm->data)->pcomm = pcomm;
214: ((DM_Moab*)dm->data)->mbiface = pcomm->get_moab();
215: return(0);
216: }
221: /*@
222: DMMoabGetParallelComm - Get the ParallelComm used with this DMMoab
224: Collective on MPI_Comm
226: Input Parameter:
227: . dm - The DMMoab object being set
229: Output Parameter:
230: . pcomm - The ParallelComm for the DMMoab
232: Level: beginner
234: .keywords: DMMoab, create
235: @*/
236: PetscErrorCode DMMoabGetParallelComm(DM dm,moab::ParallelComm **pcomm)
237: {
240: *pcomm = ((DM_Moab*)dm->data)->pcomm;
241: return(0);
242: }
247: /*@
248: DMMoabSetInterface - Set the MOAB instance used with this DMMoab
250: Collective on MPI_Comm
252: Input Parameter:
253: . dm - The DMMoab object being set
254: . mbiface - The MOAB instance being set on this DMMoab
256: Level: beginner
258: .keywords: DMMoab, create
259: @*/
260: PetscErrorCode DMMoabSetInterface(DM dm,moab::Interface *mbiface)
261: {
264: ((DM_Moab*)dm->data)->pcomm = NULL;
265: ((DM_Moab*)dm->data)->mbiface = mbiface;
266: return(0);
267: }
272: /*@
273: DMMoabGetInterface - Get the MOAB instance used with this DMMoab
275: Collective on MPI_Comm
277: Input Parameter:
278: . dm - The DMMoab object being set
280: Output Parameter:
281: . mbiface - The MOAB instance set on this DMMoab
283: Level: beginner
285: .keywords: DMMoab, create
286: @*/
287: PetscErrorCode DMMoabGetInterface(DM dm,moab::Interface **mbiface)
288: {
289: PetscErrorCode ierr;
290: static PetscBool cite = PETSC_FALSE;
294: PetscCitationsRegister("@techreport{tautges_moab:_2004,\n type = {{SAND2004-1592}},\n title = {{MOAB:} A Mesh-Oriented Database}, institution = {Sandia National Laboratories},\n author = {Tautges, T. J. and Meyers, R. and Merkley, K. and Stimpson, C. and Ernst, C.},\n year = {2004}, note = {Report}\n}\n",&cite);
295: *mbiface = ((DM_Moab*)dm->data)->mbiface;
296: return(0);
297: }
302: /*@
303: DMMoabSetRange - Set the entities having DOFs on this DMMoab
305: Collective on MPI_Comm
307: Input Parameter:
308: . dm - The DMMoab object being set
309: . range - The entities treated by this DMMoab
311: Level: beginner
313: .keywords: DMMoab, create
314: @*/
315: PetscErrorCode DMMoabSetRange(DM dm,moab::Range range)
316: {
319: ((DM_Moab*)dm->data)->range = range;
320: return(0);
321: }
326: /*@
327: DMMoabGetRange - Get the entities having DOFs on this DMMoab
329: Collective on MPI_Comm
331: Input Parameter:
332: . dm - The DMMoab object being set
334: Output Parameter:
335: . range - The entities treated by this DMMoab
337: Level: beginner
339: .keywords: DMMoab, create
340: @*/
341: PetscErrorCode DMMoabGetRange(DM dm,moab::Range *range)
342: {
345: *range = ((DM_Moab*)dm->data)->range;
346: return(0);
347: }
351: /*@
352: DMMoabSetLocalToGlobalTag - Set the tag used for local to global numbering
354: Collective on MPI_Comm
356: Input Parameter:
357: . dm - The DMMoab object being set
358: . ltogtag - The MOAB tag used for local to global ids
360: Level: beginner
362: .keywords: DMMoab, create
363: @*/
364: PetscErrorCode DMMoabSetLocalToGlobalTag(DM dm,moab::Tag ltogtag)
365: {
368: ((DM_Moab*)dm->data)->ltog_tag = ltogtag;
369: return(0);
370: }
375: /*@
376: DMMoabGetLocalToGlobalTag - Get the tag used for local to global numbering
378: Collective on MPI_Comm
380: Input Parameter:
381: . dm - The DMMoab object being set
383: Output Parameter:
384: . ltogtag - The MOAB tag used for local to global ids
386: Level: beginner
388: .keywords: DMMoab, create
389: @*/
390: PetscErrorCode DMMoabGetLocalToGlobalTag(DM dm,moab::Tag *ltog_tag)
391: {
394: *ltog_tag = ((DM_Moab*)dm->data)->ltog_tag;
395: return(0);
396: }
401: /*@
402: DMMoabSetBlockSize - Set the block size used with this DMMoab
404: Collective on MPI_Comm
406: Input Parameter:
407: . dm - The DMMoab object being set
408: . bs - The block size used with this DMMoab
410: Level: beginner
412: .keywords: DMMoab, create
413: @*/
414: PetscErrorCode DMMoabSetBlockSize(DM dm,PetscInt bs)
415: {
418: ((DM_Moab*)dm->data)->bs = bs;
419: return(0);
420: }
425: /*@
426: DMMoabGetBlockSize - Get the block size used with this DMMoab
428: Collective on MPI_Comm
430: Input Parameter:
431: . dm - The DMMoab object being set
433: Output Parameter:
434: . bs - The block size used with this DMMoab
436: Level: beginner
438: .keywords: DMMoab, create
439: @*/
440: PetscErrorCode DMMoabGetBlockSize(DM dm,PetscInt *bs)
441: {
444: *bs = ((DM_Moab*)dm->data)->bs;
445: return(0);
446: }
449: // declare for use later but before they're defined
450: PetscErrorCode DMMoab_VecUserDestroy(void *user);
451: PetscErrorCode DMMoab_VecDuplicate(Vec x,Vec *y);
452: PetscErrorCode DMMoab_CreateTagName(const moab::ParallelComm *pcomm,std::string& tag_name);
453: PetscErrorCode DMMoab_CreateVector(moab::Interface *iface,moab::ParallelComm *pcomm,moab::Tag tag,PetscInt tag_size,moab::Tag ltog_tag,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec);
457: /*@
458: DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities
460: Collective on MPI_Comm
462: Input Parameter:
463: . dm - The DMMoab object being set
464: . tag - If non-zero, block size will be taken from the tag size
465: . tag_size - If tag was zero, this parameter specifies the block size; unique tag name will be generated automatically
466: . range - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
467: . serial - If true, this is a serial Vec, otherwise a parallel one
468: . destroy_tag - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB
470: Output Parameter:
471: . vec - The created vector
473: Level: beginner
475: .keywords: DMMoab, create
476: @*/
477: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,PetscInt tag_size,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec)
478: {
479: PetscErrorCode ierr;
483: DM_Moab *dmmoab = (DM_Moab*)dm->data;
484: moab::ParallelComm *pcomm = dmmoab->pcomm;
485: moab::Interface *mbiface = dmmoab->mbiface;
486: moab::Tag ltog_tag = dmmoab->ltog_tag;
488: if (!tag && !tag_size) {
489: PetscFunctionReturn(PETSC_ERR_ARG_WRONG);
490: }
491: else {
492: DMMoab_CreateVector(mbiface,pcomm,tag,tag_size,ltog_tag,range,serial,destroy_tag,vec);
493: }
494: return(0);
495: }
500: PetscErrorCode DMMoab_CreateVector(moab::Interface *mbiface,moab::ParallelComm *pcomm,moab::Tag tag,PetscInt tag_size,moab::Tag ltog_tag,moab::Range range,PetscBool serial, PetscBool destroy_tag,Vec *vec)
501: {
502: PetscErrorCode ierr;
503: moab::ErrorCode merr;
507: if (!tag) {
508: std::string tag_name;
509: DMMoab_CreateTagName(pcomm,tag_name);
511: // Create the default value for the tag (all zeros):
512: std::vector<PetscScalar> default_value(tag_size, 0.0);
514: // Create the tag:
515: merr = mbiface->tag_get_handle(tag_name.c_str(),tag_size,moab::MB_TYPE_DOUBLE,tag,
516: moab::MB_TAG_DENSE | moab::MB_TAG_CREAT,default_value.data());MBERRNM(merr);
517: }
518: else {
520: // Make sure the tag data is of type "double":
521: moab::DataType tag_type;
522: merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
523: if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
524: }
526: // Create the MOAB internal data object
527: Vec_MOAB *vmoab;
528: PetscMalloc(sizeof(Vec_MOAB),&vmoab);
529: new (vmoab) Vec_MOAB();
530: vmoab->tag = tag;
531: vmoab->ltog_tag = ltog_tag;
532: vmoab->mbiface = mbiface;
533: vmoab->pcomm = pcomm;
534: vmoab->tag_range = range;
535: vmoab->new_tag = destroy_tag;
536: vmoab->serial = serial;
537: merr = mbiface->tag_get_length(tag,vmoab->tag_size);MBERR("tag_get_size", merr);
539: // Call tag_iterate. This will cause MOAB to allocate memory for the
540: // tag data if it hasn't already happened:
541: int count;
542: void *void_ptr;
543: merr = mbiface->tag_iterate(tag,range.begin(),range.end(),count,void_ptr);MBERRNM(merr);
545: // Check to make sure the tag data is in a single sequence:
546: if ((unsigned)count != range.size()) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Can only create MOAB Vector for single sequence");
547: PetscScalar *data_ptr = (PetscScalar*)void_ptr;
549: // Create the PETSc Vector:
550: if(!serial) {
551: // This is an MPI Vector:
552: VecCreateMPIWithArray(vmoab->pcomm->comm(),vmoab->tag_size,vmoab->tag_size*range.size(),
553: PETSC_DECIDE,data_ptr,vec);
555: // Vector created, manually set local to global mapping:
556: ISLocalToGlobalMapping ltog;
557: PetscInt *gindices = new PetscInt[range.size()];
558: PetscInt count = 0;
559: moab::Range::iterator iter;
560: for(iter = range.begin(); iter != range.end(); iter++) {
561: int dof;
562: merr = mbiface->tag_get_data(ltog_tag,&(*iter),1,&dof);MBERRNM(merr);
563: gindices[count] = dof;
564: count++;
565: }
567: ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,range.size(),gindices,
568: PETSC_COPY_VALUES,<og);
569: VecSetLocalToGlobalMappingBlock(*vec,ltog);
571: // Clean up:
572: ISLocalToGlobalMappingDestroy(<og);
573: delete [] gindices;
574: } else {
575: // This is a serial vector:
576: VecCreateSeqWithArray(PETSC_COMM_SELF,vmoab->tag_size,vmoab->tag_size*range.size(),data_ptr,vec);
577: }
580: PetscContainer moabdata;
581: PetscContainerCreate(PETSC_COMM_SELF,&moabdata);
582: PetscContainerSetPointer(moabdata,vmoab);
583: PetscContainerSetUserDestroy(moabdata,DMMoab_VecUserDestroy);
584: PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
585: (*vec)->ops->duplicate = DMMoab_VecDuplicate;
587: PetscContainerDestroy(&moabdata);
588: return(0);
589: }
593: /*@
594: DMMoabGetVecTag - Get the MOAB tag associated with this Vec
596: Collective on MPI_Comm
598: Input Parameter:
599: . vec - Vec being queried
601: Output Parameter:
602: . tag - Tag associated with this Vec
604: Level: beginner
606: .keywords: DMMoab, create
607: @*/
608: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
609: {
610: PetscContainer moabdata;
611: Vec_MOAB *vmoab;
612: PetscErrorCode ierr;
616: // Get the MOAB private data:
617: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
618: PetscContainerGetPointer(moabdata, (void**) &vmoab);
620: *tag = vmoab->tag;
622: return(0);
623: }
628: /*@
629: DMMoabGetVecRange - Get the MOAB entities associated with this Vec
631: Collective on MPI_Comm
633: Input Parameter:
634: . vec - Vec being queried
636: Output Parameter:
637: . range - Entities associated with this Vec
639: Level: beginner
641: .keywords: DMMoab, create
642: @*/
643: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
644: {
645: PetscContainer moabdata;
646: Vec_MOAB *vmoab;
647: PetscErrorCode ierr;
651: // Get the MOAB private data:
652: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
653: PetscContainerGetPointer(moabdata, (void**) &vmoab);
655: *range = vmoab->tag_range;
657: return(0);
658: }
663: PetscErrorCode DMMoab_VecDuplicate(Vec x,Vec *y)
664: {
670: // Get the Vec_MOAB struct for the original vector:
671: PetscContainer moabdata;
672: Vec_MOAB *vmoab;
673: PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
674: PetscContainerGetPointer(moabdata, (void**)&vmoab);
676: DMMoab_CreateVector(vmoab->mbiface,vmoab->pcomm,0,vmoab->tag_size,vmoab->ltog_tag,vmoab->tag_range,vmoab->serial,PETSC_TRUE,y);
677: return(0);
678: }
683: /* DMMoab_CreateTagName
684: *
685: * Creates a unique tag name that will be shared across processes. If
686: * pcomm is NULL, then this is a serial vector. A unique tag name
687: * will be returned in tag_name in either case.
688: *
689: * The tag names have the format _PETSC_VEC_N where N is some integer.
690: */
691: PetscErrorCode DMMoab_CreateTagName(const moab::ParallelComm *pcomm,std::string& tag_name)
692: {
693: moab::ErrorCode mberr;
694: PetscErrorCode ierr;
697: const std::string PVEC_PREFIX = "_PETSC_VEC_";
698: const PetscInt PVEC_PREFIX_SIZE = PVEC_PREFIX.size();
700: // Check to see if there are any PETSc vectors defined:
701: const moab::Interface *mbiface = pcomm->get_moab();
702: std::vector<moab::Tag> tags;
703: PetscInt n = 0;
704: mberr = mbiface->tag_get_tags(tags);MBERRNM(mberr);
705: for(unsigned i = 0; i < tags.size(); i++) {
706: std::string s;
707: mberr = mbiface->tag_get_name(tags[i],s);MBERRNM(mberr);
708: if(s.find(PVEC_PREFIX) != std::string::npos){
709: // This tag represents a PETSc vector. Find the vector number:
710: PetscInt m;
711: std::istringstream(s.substr(PVEC_PREFIX_SIZE)) >> m;
712: if(m >= n) n = m+1;
713: }
714: }
716: // Make sure that n is consistent across all processes:
717: PetscInt global_n;
718: MPI_Comm comm = PETSC_COMM_SELF;
719: if(pcomm) comm = pcomm->comm();
720: MPI_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,comm);
722: // Set the answer and return:
723: std::ostringstream ss;
724: ss << PVEC_PREFIX << global_n;
725: tag_name = ss.str();
726: return(0);
727: }
732: PetscErrorCode DMMoab_VecUserDestroy(void *user)
733: {
734: Vec_MOAB *vmoab;
735: PetscErrorCode ierr;
736: moab::ErrorCode merr;
739: vmoab = (Vec_MOAB*)user;
740: vmoab->tag_range.~Range();
741: if(vmoab->new_tag) {
742: // Tag created via a call to VecDuplicate, delete the underlying tag in MOAB...
743: merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
744: }
746: PetscFree(vmoab);
747: return(0);
748: }