Actual source code: dmmbvec.cxx
petsc-3.5.4 2015-05-23
1: #include <petsc-private/dmmbimpl.h> /*I "petscdmmoab.h" I*/
2: #include <petsc-private/vecimpl.h>
4: #include <petscdmmoab.h>
5: #include <MBTagConventions.hpp>
7: /* declare some private DMMoab specific overrides */
8: static PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec);
9: static PetscErrorCode DMVecUserDestroy_Moab(void *user);
10: static PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y);
11: static PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name);
15: /*@C
16: DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities
18: Collective on MPI_Comm
20: Input Parameter:
21: + dm - The DMMoab object being set
22: . tag - If non-zero, block size will be taken from the tag size
23: . range - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
24: . is_global_vec - If true, this is a local representation of the Vec (including ghosts in parallel), otherwise a truly parallel one
25: . destroy_tag - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB
27: Output Parameter:
28: . vec - The created vector
30: Level: beginner
32: .keywords: Vec, create
34: .seealso: VecCreate()
35: @*/
36: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,const moab::Range* range,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
37: {
38: PetscErrorCode ierr;
41: if(!tag && (!range || range->empty())) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Both tag and range cannot be null.");
43: DMCreateVector_Moab_Private(dm,tag,range,is_global_vec,destroy_tag,vec);
44: return(0);
45: }
50: /*@C
51: DMMoabGetVecTag - Get the MOAB tag associated with this Vec
53: Input Parameter:
54: . vec - Vec being queried
56: Output Parameter:
57: . tag - Tag associated with this Vec. NULL if vec is a native PETSc Vec.
59: Level: beginner
61: .keywords: DMMoab, MOAB tag
63: .seealso: DMMoabCreateVector(), DMMoabGetVecRange()
64: @*/
65: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
66: {
67: PetscContainer moabdata;
68: Vec_MOAB *vmoab;
69: PetscErrorCode ierr;
74: /* Get the MOAB private data */
75: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
76: PetscContainerGetPointer(moabdata, (void**) &vmoab);
78: *tag = vmoab->tag;
79: return(0);
80: }
85: /*@C
86: DMMoabGetVecRange - Get the MOAB entities associated with this Vec
88: Input Parameter:
89: . vec - Vec being queried
91: Output Parameter:
92: . range - Entities associated with this Vec. NULL if vec is a native PETSc Vec.
94: Level: beginner
96: .keywords: DMMoab, MOAB range
98: .seealso: DMMoabCreateVector(), DMMoabGetVecTag()
99: @*/
100: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
101: {
102: PetscContainer moabdata;
103: Vec_MOAB *vmoab;
104: PetscErrorCode ierr;
109: /* Get the MOAB private data handle */
110: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
111: PetscContainerGetPointer(moabdata, (void**) &vmoab);
113: *range = *vmoab->tag_range;
114: return(0);
115: }
120: /*@C
121: DMMoabVecGetArray - Returns the writable direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities
123: Collective on MPI_Comm
125: Input Parameter:
126: . dm - The DMMoab object being set
127: . vec - The Vector whose underlying data is requested
129: Output Parameter:
130: . array - The local data array
132: Level: intermediate
134: .keywords: MOAB, distributed array
136: .seealso: DMMoabVecRestoreArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
137: @*/
138: PetscErrorCode DMMoabVecGetArray(DM dm,Vec vec,void* array)
139: {
140: DM_Moab *dmmoab;
141: moab::ErrorCode merr;
142: PetscErrorCode ierr;
143: PetscInt count,i,f;
144: moab::Tag vtag;
145: PetscScalar **varray;
146: PetscScalar *marray;
147: PetscContainer moabdata;
148: Vec_MOAB *vmoab,*xmoab;
154: dmmoab=(DM_Moab*)dm->data;
156: /* Get the Vec_MOAB struct for the original vector */
157: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
158: PetscContainerGetPointer(moabdata, (void**)&vmoab);
160: /* Get the real scalar array handle */
161: varray = reinterpret_cast<PetscScalar**>(array);
163: if (vmoab->is_native_vec) {
165: /* get the local representation of the arrays from Vectors */
166: DMGetLocalVector(dm,&vmoab->local);
167: DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
168: DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);
170: /* Get the Vec_MOAB struct for the original vector */
171: PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
172: PetscContainerGetPointer(moabdata, (void**)&xmoab);
174: /* get the local representation of the arrays from Vectors */
175: VecGhostGetLocalForm(vmoab->local,&xmoab->local);
176: VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
177: VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
179: VecGetArray(xmoab->local, varray);
180: }
181: else {
183: /* Get the MOAB private data */
184: DMMoabGetVecTag(vec,&vtag);
186: /* exchange the data into ghost cells first */
187: merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);
189: PetscMalloc(sizeof(PetscScalar)*(dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);
191: /* Get the array data for local entities */
192: merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
193: if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
195: i=0;
196: for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
197: for (f=0;f<dmmoab->numFields;f++,i++)
198: (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
199: }
200: }
201: return(0);
202: }
207: /*@C
208: DMMoabVecRestoreArray - Restores the writable direct access array obtained via DMMoabVecGetArray
210: Collective on MPI_Comm
212: Input Parameter:
213: + dm - The DMMoab object being set
214: . vec - The Vector whose underlying data is requested
215: . array - The local data array
217: Level: intermediate
219: .keywords: MOAB, distributed array
221: .seealso: DMMoabVecGetArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
222: @*/
223: PetscErrorCode DMMoabVecRestoreArray(DM dm,Vec vec,void* array)
224: {
225: DM_Moab *dmmoab;
226: moab::ErrorCode merr;
227: PetscErrorCode ierr;
228: moab::Tag vtag;
229: PetscInt count,i,f;
230: PetscScalar **varray;
231: PetscScalar *marray;
232: PetscContainer moabdata;
233: Vec_MOAB *vmoab,*xmoab;
239: dmmoab=(DM_Moab*)dm->data;
241: /* Get the Vec_MOAB struct for the original vector */
242: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
243: PetscContainerGetPointer(moabdata, (void**)&vmoab);
245: /* Get the real scalar array handle */
246: varray = reinterpret_cast<PetscScalar**>(array);
248: if (vmoab->is_native_vec) {
250: /* Get the Vec_MOAB struct for the original vector */
251: PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
252: PetscContainerGetPointer(moabdata, (void**)&xmoab);
254: /* get the local representation of the arrays from Vectors */
255: VecRestoreArray(xmoab->local, varray);
256: VecGhostUpdateBegin(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
257: VecGhostUpdateEnd(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
258: VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);
260: /* restore local pieces */
261: DMLocalToGlobalBegin(dm,vmoab->local,INSERT_VALUES,vec);
262: DMLocalToGlobalEnd(dm,vmoab->local,INSERT_VALUES,vec);
263: DMRestoreLocalVector(dm, &vmoab->local);
264: }
265: else {
267: /* Get the MOAB private data */
268: DMMoabGetVecTag(vec,&vtag);
270: /* Get the array data for local entities */
271: merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
272: if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
274: i=0;
275: for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
276: for (f=0;f<dmmoab->numFields;f++,i++)
277: marray[i] = (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f];
278: }
280: /* reduce the tags correctly -> should probably let the user choose how to reduce in the future
281: For all FEM residual based assembly calculations, MPI_SUM should serve well */
282: merr = dmmoab->pcomm->reduce_tags(vtag,MPI_SUM,*dmmoab->vlocal);MBERRV(dmmoab->mbiface,merr);
284: PetscFree(*varray);
285: }
286: return(0);
287: }
291: /*@C
292: DMMoabVecGetArrayRead - Returns the read-only direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities
294: Collective on MPI_Comm
296: Input Parameter:
297: + dm - The DMMoab object being set
298: . vec - The Vector whose underlying data is requested
300: Output Parameter:
301: . array - The local data array
303: Level: intermediate
305: .keywords: MOAB, distributed array
307: .seealso: DMMoabVecRestoreArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
308: @*/
309: PetscErrorCode DMMoabVecGetArrayRead(DM dm,Vec vec,void* array)
310: {
311: DM_Moab *dmmoab;
312: moab::ErrorCode merr;
313: PetscErrorCode ierr;
314: PetscInt count,i,f;
315: moab::Tag vtag;
316: PetscScalar **varray;
317: PetscScalar *marray;
318: PetscContainer moabdata;
319: Vec_MOAB *vmoab,*xmoab;
325: dmmoab=(DM_Moab*)dm->data;
327: /* Get the Vec_MOAB struct for the original vector */
328: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
329: PetscContainerGetPointer(moabdata, (void**)&vmoab);
331: /* Get the real scalar array handle */
332: varray = reinterpret_cast<PetscScalar**>(array);
334: if (vmoab->is_native_vec) {
335: /* get the local representation of the arrays from Vectors */
336: DMGetLocalVector(dm,&vmoab->local);
337: DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
338: DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);
340: /* Get the Vec_MOAB struct for the original vector */
341: PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
342: PetscContainerGetPointer(moabdata, (void**)&xmoab);
344: /* get the local representation of the arrays from Vectors */
345: VecGhostGetLocalForm(vmoab->local,&xmoab->local);
346: VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
347: VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
348: VecGetArray(xmoab->local, varray);
349: }
350: else {
351: /* Get the MOAB private data */
352: DMMoabGetVecTag(vec,&vtag);
354: /* exchange the data into ghost cells first */
355: merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);
357: PetscMalloc(sizeof(PetscScalar)*(dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);
359: /* Get the array data for local entities */
360: merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
361: if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
363: i=0;
364: for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
365: for (f=0;f<dmmoab->numFields;f++,i++)
366: (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
367: }
368: }
369: return(0);
370: }
375: /*@C
376: DMMoabVecRestoreArray - Restores the read-only direct access array obtained via DMMoabVecGetArray
378: Collective on MPI_Comm
380: Input Parameter:
381: + dm - The DMMoab object being set
382: . vec - The Vector whose underlying data is requested
383: . array - The local data array
385: Level: intermediate
387: .keywords: MOAB, distributed array
389: .seealso: DMMoabVecGetArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
390: @*/
391: PetscErrorCode DMMoabVecRestoreArrayRead(DM dm,Vec vec,void* array)
392: {
393: PetscErrorCode ierr;
394: PetscScalar **varray;
395: PetscContainer moabdata;
396: Vec_MOAB *vmoab,*xmoab;
403: /* Get the Vec_MOAB struct for the original vector */
404: PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
405: PetscContainerGetPointer(moabdata, (void**)&vmoab);
407: /* Get the real scalar array handle */
408: varray = reinterpret_cast<PetscScalar**>(array);
410: if (vmoab->is_native_vec) {
411: /* Get the Vec_MOAB struct for the original vector */
412: PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
413: PetscContainerGetPointer(moabdata, (void**)&xmoab);
415: /* restore the local representation of the arrays from Vectors */
416: VecRestoreArray(xmoab->local, varray);
417: VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);
419: /* restore local pieces */
420: DMRestoreLocalVector(dm, &vmoab->local);
421: }
422: else {
423: /* Nothing to do but just free the memory allocated before */
424: PetscFree(*varray);
426: }
427: return(0);
428: }
433: PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
434: {
435: PetscErrorCode ierr;
436: moab::ErrorCode merr;
437: PetscBool is_newtag;
438: const moab::Range *range;
439: PetscInt count,lnative_vec,gnative_vec;
440: std::string ttname;
441: PetscScalar *data_ptr;
443: Vec_MOAB *vmoab;
444: DM_Moab *dmmoab = (DM_Moab*)dm->data;
445: moab::ParallelComm *pcomm = dmmoab->pcomm;
446: moab::Interface *mbiface = dmmoab->mbiface;
449: if(!userrange) range = dmmoab->vowned;
450: else range = userrange;
451: if(!range) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Input range cannot be empty or call DMSetUp first.");
453: /* If the tag data is in a single sequence, use PETSc native vector since tag_iterate isn't useful anymore */
454: lnative_vec=(range->psize()-1);
456: #if 0
457: // lnative_vec=1; /* NOTE: Testing PETSc vector: will force to create native vector all the time */
458: // lnative_vec=0; /* NOTE: Testing MOAB vector: will force to create MOAB tag_iterate based vector all the time */
459: #endif
460: MPI_Allreduce(&lnative_vec, &gnative_vec, 1, MPI_INT, MPI_MAX, pcomm->comm());
462: /* Create the MOAB internal data object */
463: PetscNew(&vmoab);
464: vmoab->is_native_vec=(gnative_vec>0?PETSC_TRUE:PETSC_FALSE);
466: if (!vmoab->is_native_vec) {
467: merr = mbiface->tag_get_name(tag, ttname);
468: if (!ttname.length() && merr !=moab::MB_SUCCESS) {
469: /* get the new name for the anonymous MOABVec -> the tag_name will be destroyed along with Tag */
470: char *tag_name = PETSC_NULL;
471: DMVecCreateTagName_Moab_Private(pcomm,&tag_name);
472: is_newtag = PETSC_TRUE;
474: /* Create the default value for the tag (all zeros) */
475: std::vector<PetscScalar> default_value(dmmoab->numFields, 0.0);
477: /* Create the tag */
478: merr = mbiface->tag_get_handle(tag_name,dmmoab->numFields,moab::MB_TYPE_DOUBLE,tag,
479: moab::MB_TAG_DENSE|moab::MB_TAG_CREAT,default_value.data());MBERRNM(merr);
480: PetscFree(tag_name);
481: }
482: else {
483: /* Make sure the tag data is of type "double" */
484: moab::DataType tag_type;
485: merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
486: if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
487: is_newtag = destroy_tag;
488: }
490: vmoab->tag = tag;
491: vmoab->new_tag = is_newtag;
492: }
493: vmoab->mbiface = mbiface;
494: vmoab->pcomm = pcomm;
495: vmoab->is_global_vec = is_global_vec;
496: vmoab->tag_size=dmmoab->bs;
497:
498: if (vmoab->is_native_vec) {
500: /* Create the PETSc Vector directly and attach our functions accordingly */
501: if (!is_global_vec) {
502: /* This is an MPI Vector with ghosted padding */
503: VecCreateGhostBlock(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
504: dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],vec);
505: }
506: else {
507: /* This is an MPI/SEQ Vector */
508: VecCreate(pcomm->comm(),vec);
509: VecSetSizes(*vec,dmmoab->numFields*dmmoab->nloc,PETSC_DECIDE);
510: VecSetBlockSize(*vec,dmmoab->bs);
511: VecSetType(*vec, VECMPI);
512: }
513: }
514: else {
515: /* Call tag_iterate. This will cause MOAB to allocate memory for the
516: tag data if it hasn't already happened */
517: merr = mbiface->tag_iterate(tag,range->begin(),range->end(),count,(void*&)data_ptr);MBERRNM(merr);
519: /* set the reference for vector range */
520: vmoab->tag_range = new moab::Range(*range);
521: merr = mbiface->tag_get_length(tag,dmmoab->numFields);MBERRNM(merr);
523: /* Create the PETSc Vector
524: Query MOAB mesh to check if there are any ghosted entities
525: -> if we do, create a ghosted vector to map correctly to the same layout
526: -> else, create a non-ghosted parallel vector */
527: if (!is_global_vec) {
528: /* This is an MPI Vector with ghosted padding */
529: VecCreateGhostBlockWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
530: dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],data_ptr,vec);
531: }
532: else {
533: /* This is an MPI Vector without ghosted padding */
534: VecCreateMPIWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*range->size(),
535: PETSC_DECIDE,data_ptr,vec);
536: }
537: }
538: VecSetFromOptions(*vec);
540: /* create a container and store the internal MOAB data for faster access based on Entities etc */
541: PetscContainer moabdata;
542: PetscContainerCreate(PETSC_COMM_WORLD,&moabdata);
543: PetscContainerSetPointer(moabdata,vmoab);
544: PetscContainerSetUserDestroy(moabdata,DMVecUserDestroy_Moab);
545: PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
546: (*vec)->ops->duplicate = DMVecDuplicate_Moab;
547: PetscContainerDestroy(&moabdata);
549: /* Vector created, manually set local to global mapping */
550: if (dmmoab->ltog_map) {
551: VecSetLocalToGlobalMapping(*vec,dmmoab->ltog_map);
552: }
554: /* set the DM reference to the vector */
555: VecSetDM(*vec, dm);
556: return(0);
557: }
562: /* DMVecCreateTagName_Moab_Private
563: *
564: * Creates a unique tag name that will be shared across processes. If
565: * pcomm is NULL, then this is a serial vector. A unique tag name
566: * will be returned in tag_name in either case.
567: *
568: * The tag names have the format _PETSC_VEC_N where N is some integer.
569: *
570: * NOTE: The tag_name is allocated in this routine; The user needs to free
571: * the character array.
572: */
573: PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name)
574: {
575: moab::ErrorCode mberr;
576: PetscErrorCode ierr;
577: PetscInt n,global_n;
578: moab::Tag indexTag;
581: const char* PVEC_PREFIX = "__PETSC_VEC_";
582: PetscMalloc(PETSC_MAX_PATH_LEN, tag_name);
584: /* Check to see if there are any PETSc vectors defined */
585: moab::Interface *mbiface = pcomm->get_moab();
586: moab::EntityHandle rootset = mbiface->get_root_set();
587:
588: /* Create a tag in MOAB mesh to index and keep track of number of Petsc vec tags */
589: mberr = mbiface->tag_get_handle("__PETSC_VECS__",1,moab::MB_TYPE_INTEGER,indexTag,
590: moab::MB_TAG_SPARSE | moab::MB_TAG_CREAT,0);MBERRNM(mberr);
591: mberr = mbiface->tag_get_data(indexTag, &rootset, 1, &n);
592: if (mberr == moab::MB_TAG_NOT_FOUND) n=0; /* this is the first temporary vector */
593: else MBERRNM(mberr);
595: /* increment the new value of n */
596: ++n;
598: /* Make sure that n is consistent across all processes */
599: MPI_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,pcomm->comm());
601: /* Set the new name accordingly and return */
602: PetscSNPrintf(*tag_name, PETSC_MAX_PATH_LEN-1, "%s_%D", PVEC_PREFIX, global_n);
603: mberr = mbiface->tag_set_data(indexTag, &rootset, 1, (const void*)&global_n);MBERRNM(mberr);
604: return(0);
605: }
610: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
611: {
612: PetscErrorCode ierr;
613: DM_Moab *dmmoab = (DM_Moab*)dm->data;
618: DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vowned,PETSC_TRUE,PETSC_TRUE,gvec);
619: return(0);
620: }
625: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *lvec)
626: {
627: PetscErrorCode ierr;
628: DM_Moab *dmmoab = (DM_Moab*)dm->data;
633: DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vlocal,PETSC_FALSE,PETSC_TRUE,lvec);
634: return(0);
635: }
640: PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y)
641: {
643: DM dm;
644: PetscContainer moabdata;
645: Vec_MOAB *vmoab;
651: /* Get the Vec_MOAB struct for the original vector */
652: PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
653: PetscContainerGetPointer(moabdata, (void**)&vmoab);
655: VecGetDM(x, &dm);
658: DMCreateVector_Moab_Private(dm,PETSC_NULL,vmoab->tag_range,vmoab->is_global_vec,PETSC_TRUE,y);
659: VecSetDM(*y, dm);
660: return(0);
661: }
666: PetscErrorCode DMVecUserDestroy_Moab(void *user)
667: {
668: Vec_MOAB *vmoab=(Vec_MOAB*)user;
669: PetscErrorCode ierr;
670: moab::ErrorCode merr;
673: if(vmoab->new_tag && vmoab->tag) {
674: /* Tag was created via a call to VecDuplicate, delete the underlying tag in MOAB */
675: merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
676: }
677: delete vmoab->tag_range;
678: vmoab->tag = PETSC_NULL;
679: vmoab->mbiface = PETSC_NULL;
680: vmoab->pcomm = PETSC_NULL;
681: PetscFree(vmoab);
682: return(0);
683: }
688: PetscErrorCode DMGlobalToLocalBegin_Moab(DM dm,Vec g,InsertMode mode,Vec l)
689: {
690: PetscErrorCode ierr;
691: DM_Moab *dmmoab = (DM_Moab*)dm->data;
694: VecScatterBegin(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
695: return(0);
696: }
701: PetscErrorCode DMGlobalToLocalEnd_Moab(DM dm,Vec g,InsertMode mode,Vec l)
702: {
703: PetscErrorCode ierr;
704: DM_Moab *dmmoab = (DM_Moab*)dm->data;
707: VecScatterEnd(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
708: return(0);
709: }
714: PetscErrorCode DMLocalToGlobalBegin_Moab(DM dm,Vec l,InsertMode mode,Vec g)
715: {
716: PetscErrorCode ierr;
717: DM_Moab *dmmoab = (DM_Moab*)dm->data;
720: VecScatterBegin(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
721: return(0);
722: }
727: PetscErrorCode DMLocalToGlobalEnd_Moab(DM dm,Vec l,InsertMode mode,Vec g)
728: {
729: PetscErrorCode ierr;
730: DM_Moab *dmmoab = (DM_Moab*)dm->data;
733: VecScatterEnd(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
734: return(0);
735: }