Actual source code: dmmbvec.cxx

petsc-3.5.4 2015-05-23
Report Typos and Errors
  1: #include <petsc-private/dmmbimpl.h> /*I  "petscdmmoab.h"   I*/
  2: #include <petsc-private/vecimpl.h>

  4: #include <petscdmmoab.h>
  5: #include <MBTagConventions.hpp>

  7: /* declare some private DMMoab specific overrides */
  8: static PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec);
  9: static PetscErrorCode DMVecUserDestroy_Moab(void *user);
 10: static PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y);
 11: static PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name);

 15: /*@C
 16:   DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities

 18:   Collective on MPI_Comm

 20:   Input Parameter:
 21: + dm              - The DMMoab object being set
 22: . tag             - If non-zero, block size will be taken from the tag size
 23: . range           - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
 24: . is_global_vec   - If true, this is a local representation of the Vec (including ghosts in parallel), otherwise a truly parallel one
 25: . destroy_tag     - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB

 27:   Output Parameter:
 28: . vec             - The created vector

 30:   Level: beginner

 32: .keywords: Vec, create

 34: .seealso: VecCreate()
 35: @*/
 36: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,const moab::Range* range,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
 37: {
 38:   PetscErrorCode     ierr;

 41:   if(!tag && (!range || range->empty())) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Both tag and range cannot be null.");

 43:   DMCreateVector_Moab_Private(dm,tag,range,is_global_vec,destroy_tag,vec);
 44:   return(0);
 45: }


 50: /*@C
 51:   DMMoabGetVecTag - Get the MOAB tag associated with this Vec

 53:   Input Parameter:
 54: . vec - Vec being queried

 56:   Output Parameter:
 57: . tag - Tag associated with this Vec. NULL if vec is a native PETSc Vec.

 59:   Level: beginner

 61: .keywords: DMMoab, MOAB tag

 63: .seealso: DMMoabCreateVector(), DMMoabGetVecRange()
 64: @*/
 65: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
 66: {
 67:   PetscContainer  moabdata;
 68:   Vec_MOAB        *vmoab;
 69:   PetscErrorCode  ierr;


 74:   /* Get the MOAB private data */
 75:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
 76:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

 78:   *tag = vmoab->tag;
 79:   return(0);
 80: }


 85: /*@C
 86:   DMMoabGetVecRange - Get the MOAB entities associated with this Vec

 88:   Input Parameter:
 89: . vec   - Vec being queried

 91:   Output Parameter:
 92: . range - Entities associated with this Vec. NULL if vec is a native PETSc Vec.

 94:   Level: beginner

 96: .keywords: DMMoab, MOAB range

 98: .seealso: DMMoabCreateVector(), DMMoabGetVecTag()
 99: @*/
100: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
101: {
102:   PetscContainer  moabdata;
103:   Vec_MOAB        *vmoab;
104:   PetscErrorCode  ierr;


109:   /* Get the MOAB private data handle */
110:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
111:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

113:   *range = *vmoab->tag_range;
114:   return(0);
115: }


120: /*@C
121:   DMMoabVecGetArray - Returns the writable direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities

123:   Collective on MPI_Comm

125:   Input Parameter:
126: . dm              - The DMMoab object being set
127: . vec             - The Vector whose underlying data is requested

129:   Output Parameter:
130: . array           - The local data array

132:   Level: intermediate

134: .keywords: MOAB, distributed array

136: .seealso: DMMoabVecRestoreArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
137: @*/
138: PetscErrorCode  DMMoabVecGetArray(DM dm,Vec vec,void* array)
139: {
140:   DM_Moab        *dmmoab;
141:   moab::ErrorCode merr;
142:   PetscErrorCode  ierr;
143:   PetscInt        count,i,f;
144:   moab::Tag       vtag;
145:   PetscScalar     **varray;
146:   PetscScalar     *marray;
147:   PetscContainer  moabdata;
148:   Vec_MOAB        *vmoab,*xmoab;

154:   dmmoab=(DM_Moab*)dm->data;

156:   /* Get the Vec_MOAB struct for the original vector */
157:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
158:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

160:   /* Get the real scalar array handle */
161:   varray = reinterpret_cast<PetscScalar**>(array);

163:   if (vmoab->is_native_vec) {

165:     /* get the local representation of the arrays from Vectors */
166:     DMGetLocalVector(dm,&vmoab->local);
167:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
168:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);

170:     /* Get the Vec_MOAB struct for the original vector */
171:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
172:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

174:     /* get the local representation of the arrays from Vectors */
175:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
176:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
177:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);

179:     VecGetArray(xmoab->local, varray);
180:   }
181:   else {

183:     /* Get the MOAB private data */
184:     DMMoabGetVecTag(vec,&vtag);

186:     /* exchange the data into ghost cells first */
187:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);

189:     PetscMalloc(sizeof(PetscScalar)*(dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);

191:     /* Get the array data for local entities */
192:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
193:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

195:     i=0;
196:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
197:       for (f=0;f<dmmoab->numFields;f++,i++)
198:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
199:     }
200:   }
201:   return(0);
202: }


207: /*@C
208:   DMMoabVecRestoreArray - Restores the writable direct access array obtained via DMMoabVecGetArray

210:   Collective on MPI_Comm

212:   Input Parameter:
213: + dm              - The DMMoab object being set
214: . vec             - The Vector whose underlying data is requested
215: . array           - The local data array

217:   Level: intermediate

219: .keywords: MOAB, distributed array

221: .seealso: DMMoabVecGetArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
222: @*/
223: PetscErrorCode  DMMoabVecRestoreArray(DM dm,Vec vec,void* array)
224: {
225:   DM_Moab        *dmmoab;
226:   moab::ErrorCode merr;
227:   PetscErrorCode  ierr;
228:   moab::Tag       vtag;
229:   PetscInt        count,i,f;
230:   PetscScalar     **varray;
231:   PetscScalar     *marray;
232:   PetscContainer  moabdata;
233:   Vec_MOAB        *vmoab,*xmoab;

239:   dmmoab=(DM_Moab*)dm->data;

241:   /* Get the Vec_MOAB struct for the original vector */
242:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
243:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

245:   /* Get the real scalar array handle */
246:   varray = reinterpret_cast<PetscScalar**>(array);

248:   if (vmoab->is_native_vec) {

250:     /* Get the Vec_MOAB struct for the original vector */
251:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
252:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

254:     /* get the local representation of the arrays from Vectors */
255:     VecRestoreArray(xmoab->local, varray);
256:     VecGhostUpdateBegin(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
257:     VecGhostUpdateEnd(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
258:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);

260:     /* restore local pieces */
261:     DMLocalToGlobalBegin(dm,vmoab->local,INSERT_VALUES,vec);
262:     DMLocalToGlobalEnd(dm,vmoab->local,INSERT_VALUES,vec);
263:     DMRestoreLocalVector(dm, &vmoab->local);
264:   }
265:   else {

267:     /* Get the MOAB private data */
268:     DMMoabGetVecTag(vec,&vtag);

270:     /* Get the array data for local entities */
271:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
272:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

274:     i=0;
275:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
276:       for (f=0;f<dmmoab->numFields;f++,i++)
277:       marray[i] = (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f];
278:     }

280:     /* reduce the tags correctly -> should probably let the user choose how to reduce in the future
281:       For all FEM residual based assembly calculations, MPI_SUM should serve well */
282:     merr = dmmoab->pcomm->reduce_tags(vtag,MPI_SUM,*dmmoab->vlocal);MBERRV(dmmoab->mbiface,merr);

284:     PetscFree(*varray);
285:   }
286:   return(0);
287: }

291: /*@C
292:   DMMoabVecGetArrayRead - Returns the read-only direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities

294:   Collective on MPI_Comm

296:   Input Parameter:
297: + dm              - The DMMoab object being set
298: . vec             - The Vector whose underlying data is requested

300:   Output Parameter:
301: . array           - The local data array

303:   Level: intermediate

305: .keywords: MOAB, distributed array

307: .seealso: DMMoabVecRestoreArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
308: @*/
309: PetscErrorCode  DMMoabVecGetArrayRead(DM dm,Vec vec,void* array)
310: {
311:   DM_Moab        *dmmoab;
312:   moab::ErrorCode merr;
313:   PetscErrorCode  ierr;
314:   PetscInt        count,i,f;
315:   moab::Tag       vtag;
316:   PetscScalar     **varray;
317:   PetscScalar     *marray;
318:   PetscContainer  moabdata;
319:   Vec_MOAB        *vmoab,*xmoab;

325:   dmmoab=(DM_Moab*)dm->data;

327:   /* Get the Vec_MOAB struct for the original vector */
328:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
329:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

331:   /* Get the real scalar array handle */
332:   varray = reinterpret_cast<PetscScalar**>(array);

334:   if (vmoab->is_native_vec) {
335:     /* get the local representation of the arrays from Vectors */
336:     DMGetLocalVector(dm,&vmoab->local);
337:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
338:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);

340:     /* Get the Vec_MOAB struct for the original vector */
341:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
342:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

344:     /* get the local representation of the arrays from Vectors */
345:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
346:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
347:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
348:     VecGetArray(xmoab->local, varray);
349:   }
350:   else {
351:     /* Get the MOAB private data */
352:     DMMoabGetVecTag(vec,&vtag);

354:     /* exchange the data into ghost cells first */
355:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);

357:     PetscMalloc(sizeof(PetscScalar)*(dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);

359:     /* Get the array data for local entities */
360:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
361:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

363:     i=0;
364:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
365:       for (f=0;f<dmmoab->numFields;f++,i++)
366:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
367:     }
368:   }
369:   return(0);
370: }


375: /*@C
376:   DMMoabVecRestoreArray - Restores the read-only direct access array obtained via DMMoabVecGetArray

378:   Collective on MPI_Comm

380:   Input Parameter:
381: + dm              - The DMMoab object being set
382: . vec             - The Vector whose underlying data is requested
383: . array           - The local data array

385:   Level: intermediate

387: .keywords: MOAB, distributed array

389: .seealso: DMMoabVecGetArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
390: @*/
391: PetscErrorCode  DMMoabVecRestoreArrayRead(DM dm,Vec vec,void* array)
392: {
393:   PetscErrorCode  ierr;
394:   PetscScalar     **varray;
395:   PetscContainer  moabdata;
396:   Vec_MOAB        *vmoab,*xmoab;


403:   /* Get the Vec_MOAB struct for the original vector */
404:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
405:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

407:   /* Get the real scalar array handle */
408:   varray = reinterpret_cast<PetscScalar**>(array);

410:   if (vmoab->is_native_vec) {
411:     /* Get the Vec_MOAB struct for the original vector */
412:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
413:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

415:     /* restore the local representation of the arrays from Vectors */
416:     VecRestoreArray(xmoab->local, varray);
417:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);

419:     /* restore local pieces */
420:     DMRestoreLocalVector(dm, &vmoab->local);
421:   }
422:   else {
423:     /* Nothing to do but just free the memory allocated before */
424:     PetscFree(*varray);

426:   }
427:   return(0);
428: }


433: PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
434: {
435:   PetscErrorCode         ierr;
436:   moab::ErrorCode        merr;
437:   PetscBool              is_newtag;
438:   const moab::Range      *range;
439:   PetscInt               count,lnative_vec,gnative_vec;
440:   std::string ttname;
441:   PetscScalar *data_ptr;

443:   Vec_MOAB *vmoab;
444:   DM_Moab *dmmoab = (DM_Moab*)dm->data;
445:   moab::ParallelComm *pcomm = dmmoab->pcomm;
446:   moab::Interface *mbiface = dmmoab->mbiface;

449:   if(!userrange) range = dmmoab->vowned;
450:   else range = userrange;
451:   if(!range) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Input range cannot be empty or call DMSetUp first.");

453:   /* If the tag data is in a single sequence, use PETSc native vector since tag_iterate isn't useful anymore */
454:   lnative_vec=(range->psize()-1);

456: #if 0
457: //  lnative_vec=1; /* NOTE: Testing PETSc vector: will force to create native vector all the time */
458: //  lnative_vec=0; /* NOTE: Testing MOAB vector: will force to create MOAB tag_iterate based vector all the time */
459: #endif
460:   MPI_Allreduce(&lnative_vec, &gnative_vec, 1, MPI_INT, MPI_MAX, pcomm->comm());

462:   /* Create the MOAB internal data object */
463:   PetscNew(&vmoab);
464:   vmoab->is_native_vec=(gnative_vec>0?PETSC_TRUE:PETSC_FALSE);

466:   if (!vmoab->is_native_vec) {
467:     merr = mbiface->tag_get_name(tag, ttname);
468:     if (!ttname.length() && merr !=moab::MB_SUCCESS) {
469:       /* get the new name for the anonymous MOABVec -> the tag_name will be destroyed along with Tag */
470:       char *tag_name = PETSC_NULL;
471:       DMVecCreateTagName_Moab_Private(pcomm,&tag_name);
472:       is_newtag = PETSC_TRUE;

474:       /* Create the default value for the tag (all zeros) */
475:       std::vector<PetscScalar> default_value(dmmoab->numFields, 0.0);

477:       /* Create the tag */
478:       merr = mbiface->tag_get_handle(tag_name,dmmoab->numFields,moab::MB_TYPE_DOUBLE,tag,
479:                                     moab::MB_TAG_DENSE|moab::MB_TAG_CREAT,default_value.data());MBERRNM(merr);
480:       PetscFree(tag_name);
481:     }
482:     else {
483:       /* Make sure the tag data is of type "double" */
484:       moab::DataType tag_type;
485:       merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
486:       if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
487:       is_newtag = destroy_tag;
488:     }

490:     vmoab->tag = tag;
491:     vmoab->new_tag = is_newtag;
492:   }
493:   vmoab->mbiface = mbiface;
494:   vmoab->pcomm = pcomm;
495:   vmoab->is_global_vec = is_global_vec;
496:   vmoab->tag_size=dmmoab->bs;
497: 
498:   if (vmoab->is_native_vec) {

500:     /* Create the PETSc Vector directly and attach our functions accordingly */
501:     if (!is_global_vec) {
502:       /* This is an MPI Vector with ghosted padding */
503:       VecCreateGhostBlock(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
504:                                  dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],vec);
505:     }
506:     else {
507:       /* This is an MPI/SEQ Vector */
508:       VecCreate(pcomm->comm(),vec);
509:       VecSetSizes(*vec,dmmoab->numFields*dmmoab->nloc,PETSC_DECIDE);
510:       VecSetBlockSize(*vec,dmmoab->bs);
511:       VecSetType(*vec, VECMPI);
512:     }
513:   }
514:   else {
515:     /* Call tag_iterate. This will cause MOAB to allocate memory for the
516:        tag data if it hasn't already happened */
517:     merr = mbiface->tag_iterate(tag,range->begin(),range->end(),count,(void*&)data_ptr);MBERRNM(merr);

519:     /* set the reference for vector range */
520:     vmoab->tag_range = new moab::Range(*range);
521:     merr = mbiface->tag_get_length(tag,dmmoab->numFields);MBERRNM(merr);

523:     /* Create the PETSc Vector
524:       Query MOAB mesh to check if there are any ghosted entities
525:         -> if we do, create a ghosted vector to map correctly to the same layout
526:         -> else, create a non-ghosted parallel vector */
527:     if (!is_global_vec) {
528:       /* This is an MPI Vector with ghosted padding */
529:       VecCreateGhostBlockWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
530:                                 dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],data_ptr,vec);
531:     }
532:     else {
533:       /* This is an MPI Vector without ghosted padding */
534:       VecCreateMPIWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*range->size(),
535:                                 PETSC_DECIDE,data_ptr,vec);
536:     }
537:   }
538:   VecSetFromOptions(*vec);

540:   /* create a container and store the internal MOAB data for faster access based on Entities etc */
541:   PetscContainer moabdata;
542:   PetscContainerCreate(PETSC_COMM_WORLD,&moabdata);
543:   PetscContainerSetPointer(moabdata,vmoab);
544:   PetscContainerSetUserDestroy(moabdata,DMVecUserDestroy_Moab);
545:   PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
546:   (*vec)->ops->duplicate = DMVecDuplicate_Moab;
547:   PetscContainerDestroy(&moabdata);

549:   /* Vector created, manually set local to global mapping */
550:   if (dmmoab->ltog_map) {
551:     VecSetLocalToGlobalMapping(*vec,dmmoab->ltog_map);
552:   }

554:   /* set the DM reference to the vector */
555:   VecSetDM(*vec, dm);
556:   return(0);
557: }


562: /*  DMVecCreateTagName_Moab_Private
563:  *
564:  *  Creates a unique tag name that will be shared across processes. If
565:  *  pcomm is NULL, then this is a serial vector. A unique tag name
566:  *  will be returned in tag_name in either case.
567:  *
568:  *  The tag names have the format _PETSC_VEC_N where N is some integer.
569:  *
570:  *  NOTE: The tag_name is allocated in this routine; The user needs to free 
571:  *        the character array.
572:  */
573: PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name)
574: {
575:   moab::ErrorCode mberr;
576:   PetscErrorCode  ierr;
577:   PetscInt        n,global_n;
578:   moab::Tag indexTag;

581:   const char*       PVEC_PREFIX      = "__PETSC_VEC_";
582:   PetscMalloc(PETSC_MAX_PATH_LEN, tag_name);

584:   /* Check to see if there are any PETSc vectors defined */
585:   moab::Interface  *mbiface = pcomm->get_moab();
586:   moab::EntityHandle rootset = mbiface->get_root_set();
587: 
588:   /* Create a tag in MOAB mesh to index and keep track of number of Petsc vec tags */
589:   mberr = mbiface->tag_get_handle("__PETSC_VECS__",1,moab::MB_TYPE_INTEGER,indexTag,
590:                                   moab::MB_TAG_SPARSE | moab::MB_TAG_CREAT,0);MBERRNM(mberr);
591:   mberr = mbiface->tag_get_data(indexTag, &rootset, 1, &n);
592:   if (mberr == moab::MB_TAG_NOT_FOUND) n=0;  /* this is the first temporary vector */
593:   else MBERRNM(mberr);

595:   /* increment the new value of n */
596:   ++n;

598:   /* Make sure that n is consistent across all processes */
599:   MPI_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,pcomm->comm());

601:   /* Set the new name accordingly and return */
602:   PetscSNPrintf(*tag_name, PETSC_MAX_PATH_LEN-1, "%s_%D", PVEC_PREFIX, global_n);
603:   mberr = mbiface->tag_set_data(indexTag, &rootset, 1, (const void*)&global_n);MBERRNM(mberr);
604:   return(0);
605: }


610: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
611: {
612:   PetscErrorCode  ierr;
613:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

618:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vowned,PETSC_TRUE,PETSC_TRUE,gvec);
619:   return(0);
620: }


625: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *lvec)
626: {
627:   PetscErrorCode  ierr;
628:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

633:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vlocal,PETSC_FALSE,PETSC_TRUE,lvec);
634:   return(0);
635: }


640: PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y)
641: {
643:   DM             dm;
644:   PetscContainer  moabdata;
645:   Vec_MOAB        *vmoab;


651:   /* Get the Vec_MOAB struct for the original vector */
652:   PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
653:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

655:   VecGetDM(x, &dm);

658:   DMCreateVector_Moab_Private(dm,PETSC_NULL,vmoab->tag_range,vmoab->is_global_vec,PETSC_TRUE,y);
659:   VecSetDM(*y, dm);
660:   return(0);
661: }


666: PetscErrorCode DMVecUserDestroy_Moab(void *user)
667: {
668:   Vec_MOAB        *vmoab=(Vec_MOAB*)user;
669:   PetscErrorCode  ierr;
670:   moab::ErrorCode merr;

673:   if(vmoab->new_tag && vmoab->tag) {
674:     /* Tag was created via a call to VecDuplicate, delete the underlying tag in MOAB */
675:     merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
676:   }
677:   delete vmoab->tag_range;
678:   vmoab->tag = PETSC_NULL;
679:   vmoab->mbiface = PETSC_NULL;
680:   vmoab->pcomm = PETSC_NULL;
681:   PetscFree(vmoab);
682:   return(0);
683: }


688: PetscErrorCode  DMGlobalToLocalBegin_Moab(DM dm,Vec g,InsertMode mode,Vec l)
689: {
690:   PetscErrorCode    ierr;
691:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

694:   VecScatterBegin(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
695:   return(0);
696: }


701: PetscErrorCode  DMGlobalToLocalEnd_Moab(DM dm,Vec g,InsertMode mode,Vec l)
702: {
703:   PetscErrorCode    ierr;
704:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

707:   VecScatterEnd(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
708:   return(0);
709: }


714: PetscErrorCode  DMLocalToGlobalBegin_Moab(DM dm,Vec l,InsertMode mode,Vec g)
715: {
716:   PetscErrorCode    ierr;
717:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

720:   VecScatterBegin(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
721:   return(0);
722: }


727: PetscErrorCode  DMLocalToGlobalEnd_Moab(DM dm,Vec l,InsertMode mode,Vec g)
728: {
729:   PetscErrorCode    ierr;
730:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

733:   VecScatterEnd(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
734:   return(0);
735: }