Actual source code: pack.c

  1: #define PETSCDM_DLL
  2: 
 3:  #include petscda.h
 4:  #include src/dm/dmimpl.h
 5:  #include petscmat.h

  7: typedef struct _DMCompositeOps *DMCompositeOps;
  8: struct _DMCompositeOps {
  9:   DMOPS(DMComposite)
 10: };

 12: /*
 13:    rstart is where an array/subvector starts in the global parallel vector, so arrays
 14:    rstarts are meaningless (and set to the previous one) except on processor 0
 15: */

 17: typedef enum {DMCOMPOSITE_ARRAY, DMCOMPOSITE_DA, DMCOMPOSITE_VECSCATTER} DMCompositeLinkType;

 19: struct DMCompositeLink {
 20:   DMCompositeLinkType    type;
 21:   struct DMCompositeLink *next;
 22:   PetscInt               n,rstart;      /* rstart is relative to this processor */

 24:   /* only used for DMCOMPOSITE_DA */
 25:   PetscInt               *grstarts;     /* global row for first unknown of this DA on each process */
 26:   DA                     da;

 28:   /* only used for DMCOMPOSITE_ARRAY */
 29:   PetscInt               grstart;        /* global row for first array unknown */
 30:   PetscMPIInt            rank;          /* process where array unknowns live */
 31: };

 33: struct _p_DMComposite {
 34:   PETSCHEADER(struct _DMCompositeOps);
 35:   PetscInt               n,N,rstart;     /* rstart is relative to all processors, n unknowns owned by this process, N is total unknowns */
 36:   PetscInt               nghost;         /* number of all local entries include DA ghost points and any shared redundant arrays */
 37:   PetscInt               nDA,nredundant; /* how many DA's and seperate redundant arrays used to build DMComposite */
 38:   PetscTruth             setup;          /* after this is set, cannot add new links to the DMComposite */
 39:   struct DMCompositeLink *next;
 40: };


 45: /*@C
 46:     DMCompositeCreate - Creates a vector packer, used to generate "composite"
 47:       vectors made up of several subvectors.

 49:     Collective on MPI_Comm

 51:     Input Parameter:
 52: .   comm - the processors that will share the global vector

 54:     Output Parameters:
 55: .   packer - the packer object

 57:     Level: advanced

 59: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeScatter(),
 60:          DMCompositeGather(), DMCompositeCreateGlobalVector(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess()
 61:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

 63: @*/
 64: PetscErrorCode  DMCompositeCreate(MPI_Comm comm,DMComposite *packer)
 65: {
 67:   DMComposite    p;

 71:   *packer = PETSC_NULL;
 72: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
 73:   DMInitializePackage(PETSC_NULL);
 74: #endif

 76:   PetscHeaderCreate(p,_p_DMComposite,struct _DMCompositeOps,DA_COOKIE,0,"DMComposite",comm,DMCompositeDestroy,0);
 77:   p->n            = 0;
 78:   p->next         = PETSC_NULL;
 79:   /* dalcinl: line below seems to be wrong ...
 80:   ((PetscObject)p)->comm         = comm;
 81:   */
 82:   p->nredundant   = 0;
 83:   p->nDA          = 0;

 85:   p->ops->createglobalvector = DMCompositeCreateGlobalVector;
 86:   p->ops->refine             = DMCompositeRefine;
 87:   p->ops->getinterpolation   = DMCompositeGetInterpolation;
 88:   p->ops->getmatrix          = DMCompositeGetMatrix;
 89:   p->ops->getcoloring        = DMCompositeGetColoring;

 91:   *packer = p;
 92:   return(0);
 93: }

 97: /*@C
 98:     DMCompositeDestroy - Destroys a vector packer.

100:     Collective on DMComposite

102:     Input Parameter:
103: .   packer - the packer object

105:     Level: advanced

107: .seealso DMCompositeCreate(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeScatter(),DMCompositeGetEntries()
108:          DMCompositeGather(), DMCompositeCreateGlobalVector(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess()

110: @*/
111: PetscErrorCode  DMCompositeDestroy(DMComposite packer)
112: {
113:   PetscErrorCode         ierr;
114:   struct DMCompositeLink *next, *prev;

118:   next = packer->next;
119:   if (--((PetscObject)packer)->refct > 0) return(0);
120:   while (next) {
121:     prev = next;
122:     next = next->next;
123:     if (prev->type == DMCOMPOSITE_DA) {
124:       DADestroy(prev->da);
125:     }
126:     if (prev->grstarts) {
127:       PetscFree(prev->grstarts);
128:     }
129:     PetscFree(prev);
130:   }
131:   PetscHeaderDestroy(packer);
132:   return(0);
133: }

135: /* --------------------------------------------------------------------------------------*/
138: PetscErrorCode  DMCompositeSetUp(DMComposite packer)
139: {
140:   PetscErrorCode         ierr;
141:   PetscInt               nprev = 0;
142:   PetscMPIInt            rank,size;
143:   struct DMCompositeLink *next = packer->next;
144:   PetscMap               map;

147:   if (packer->setup) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
148:   PetscMapInitialize(((PetscObject)packer)->comm,&map);
149:   PetscMapSetLocalSize(&map,packer->n);
150:   PetscMapSetSize(&map,PETSC_DETERMINE);
151:   PetscMapSetBlockSize(&map,1);
152:   PetscMapSetUp(&map);
153:   PetscMapGetSize(&map,&packer->N);
154:   PetscMapGetLocalRange(&map,&packer->rstart,PETSC_NULL);
155:   PetscFree(map.range);
156: 
157:   /* now set the rstart for each linked array/vector */
158:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
159:   MPI_Comm_size(((PetscObject)packer)->comm,&size);
160:   while (next) {
161:     next->rstart = nprev;
162:     if ((rank == next->rank) || next->type != DMCOMPOSITE_ARRAY) nprev += next->n;
163:     next->grstart = packer->rstart + next->rstart;
164:     if (next->type == DMCOMPOSITE_ARRAY) {
165:       MPI_Bcast(&next->grstart,1,MPIU_INT,next->rank,((PetscObject)packer)->comm);
166:     } else {
167:       PetscMalloc(size*sizeof(PetscInt),&next->grstarts);
168:       MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)packer)->comm);
169:     }
170:     next = next->next;
171:   }
172:   packer->setup = PETSC_TRUE;
173:   return(0);
174: }


179: PetscErrorCode DMCompositeGetAccess_Array(DMComposite packer,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
180: {
182:   PetscScalar    *varray;
183:   PetscMPIInt    rank;

186:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
187:   if (array) {
188:     if (rank == mine->rank) {
189:       VecGetArray(vec,&varray);
190:       *array  = varray + mine->rstart;
191:       VecRestoreArray(vec,&varray);
192:     } else {
193:       *array = 0;
194:     }
195:   }
196:   return(0);
197: }

201: PetscErrorCode DMCompositeGetAccess_DA(DMComposite packer,struct DMCompositeLink *mine,Vec vec,Vec *global)
202: {
204:   PetscScalar    *array;

207:   if (global) {
208:     DAGetGlobalVector(mine->da,global);
209:     VecGetArray(vec,&array);
210:     VecPlaceArray(*global,array+mine->rstart);
211:     VecRestoreArray(vec,&array);
212:   }
213:   return(0);
214: }

218: PetscErrorCode DMCompositeRestoreAccess_Array(DMComposite packer,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
219: {
221:   return(0);
222: }

226: PetscErrorCode DMCompositeRestoreAccess_DA(DMComposite packer,struct DMCompositeLink *mine,Vec vec,Vec *global)
227: {

231:   if (global) {
232:     VecResetArray(*global);
233:     DARestoreGlobalVector(mine->da,global);
234:   }
235:   return(0);
236: }

240: PetscErrorCode DMCompositeScatter_Array(DMComposite packer,struct DMCompositeLink *mine,Vec vec,PetscScalar *array)
241: {
243:   PetscScalar    *varray;
244:   PetscMPIInt    rank;

247:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
248:   if (rank == mine->rank) {
249:     VecGetArray(vec,&varray);
250:     PetscMemcpy(array,varray+mine->rstart,mine->n*sizeof(PetscScalar));
251:     VecRestoreArray(vec,&varray);
252:   }
253:   MPI_Bcast(array,mine->n,MPIU_SCALAR,mine->rank,((PetscObject)packer)->comm);
254:   return(0);
255: }

259: PetscErrorCode DMCompositeScatter_DA(DMComposite packer,struct DMCompositeLink *mine,Vec vec,Vec local)
260: {
262:   PetscScalar    *array;
263:   Vec            global;

266:   DAGetGlobalVector(mine->da,&global);
267:   VecGetArray(vec,&array);
268:   VecPlaceArray(global,array+mine->rstart);
269:   DAGlobalToLocalBegin(mine->da,global,INSERT_VALUES,local);
270:   DAGlobalToLocalEnd(mine->da,global,INSERT_VALUES,local);
271:   VecRestoreArray(vec,&array);
272:   VecResetArray(global);
273:   DARestoreGlobalVector(mine->da,&global);
274:   return(0);
275: }

279: PetscErrorCode DMCompositeGather_Array(DMComposite packer,struct DMCompositeLink *mine,Vec vec,PetscScalar *array)
280: {
282:   PetscScalar    *varray;
283:   PetscMPIInt    rank;

286:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
287:   if (rank == mine->rank) {
288:     VecGetArray(vec,&varray);
289:     if (varray+mine->rstart == array) SETERRQ(PETSC_ERR_ARG_WRONG,"You need not DMCompositeGather() into objects obtained via DMCompositeGetAccess()");
290:     PetscMemcpy(varray+mine->rstart,array,mine->n*sizeof(PetscScalar));
291:     VecRestoreArray(vec,&varray);
292:   }
293:   return(0);
294: }

298: PetscErrorCode DMCompositeGather_DA(DMComposite packer,struct DMCompositeLink *mine,Vec vec,Vec local)
299: {
301:   PetscScalar    *array;
302:   Vec            global;

305:   DAGetGlobalVector(mine->da,&global);
306:   VecGetArray(vec,&array);
307:   VecPlaceArray(global,array+mine->rstart);
308:   DALocalToGlobal(mine->da,local,INSERT_VALUES,global);
309:   VecRestoreArray(vec,&array);
310:   VecResetArray(global);
311:   DARestoreGlobalVector(mine->da,&global);
312:   return(0);
313: }

315: /* ----------------------------------------------------------------------------------*/

317: #include <stdarg.h>

321: /*@C
322:     DMCompositeGetAccess - Allows one to access the individual packed vectors in their global
323:        representation.

325:     Collective on DMComposite

327:     Input Parameter:
328: +    packer - the packer object
329: .    gvec - the global vector
330: -    ... - the individual sequential or parallel objects (arrays or vectors)

332:     Notes: Use DMCompositeRestoreAccess() to return the vectors when you no longer need them
333:  
334:     Level: advanced

336: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
337:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeScatter(),
338:          DMCompositeRestoreAccess(), DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),
339:          DMCompositeGetEntries()

341: @*/
342: PetscErrorCode  DMCompositeGetAccess(DMComposite packer,Vec gvec,...)
343: {
344:   va_list                Argp;
345:   PetscErrorCode         ierr;
346:   struct DMCompositeLink *next;

351:   next = packer->next;
352:   if (!packer->setup) {
353:     DMCompositeSetUp(packer);
354:   }

356:   /* loop over packed objects, handling one at at time */
357:   va_start(Argp,gvec);
358:   while (next) {
359:     if (next->type == DMCOMPOSITE_ARRAY) {
360:       PetscScalar **array;
361:       array = va_arg(Argp, PetscScalar**);
362:       DMCompositeGetAccess_Array(packer,next,gvec,array);
363:     } else if (next->type == DMCOMPOSITE_DA) {
364:       Vec *vec;
365:       vec  = va_arg(Argp, Vec*);
366:       DMCompositeGetAccess_DA(packer,next,gvec,vec);
367:     } else {
368:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
369:     }
370:     next = next->next;
371:   }
372:   va_end(Argp);
373:   return(0);
374: }

378: /*@C
379:     DMCompositeRestoreAccess - Returns the vectors obtained with DACompositeGetAccess()
380:        representation.

382:     Collective on DMComposite

384:     Input Parameter:
385: +    packer - the packer object
386: .    gvec - the global vector
387: -    ... - the individual sequential or parallel objects (arrays or vectors)
388:  
389:     Level: advanced

391: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
392:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeScatter(),
393:          DMCompositeRestoreAccess(), DACompositeGetAccess()

395: @*/
396: PetscErrorCode  DMCompositeRestoreAccess(DMComposite packer,Vec gvec,...)
397: {
398:   va_list                Argp;
399:   PetscErrorCode         ierr;
400:   struct DMCompositeLink *next;

405:   next = packer->next;
406:   if (!packer->setup) {
407:     DMCompositeSetUp(packer);
408:   }

410:   /* loop over packed objects, handling one at at time */
411:   va_start(Argp,gvec);
412:   while (next) {
413:     if (next->type == DMCOMPOSITE_ARRAY) {
414:       PetscScalar **array;
415:       array = va_arg(Argp, PetscScalar**);
416:       DMCompositeRestoreAccess_Array(packer,next,gvec,array);
417:     } else if (next->type == DMCOMPOSITE_DA) {
418:       Vec *vec;
419:       vec  = va_arg(Argp, Vec*);
420:       DMCompositeRestoreAccess_DA(packer,next,gvec,vec);
421:     } else {
422:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
423:     }
424:     next = next->next;
425:   }
426:   va_end(Argp);
427:   return(0);
428: }

432: /*@C
433:     DMCompositeScatter - Scatters from a global packed vector into its individual local vectors

435:     Collective on DMComposite

437:     Input Parameter:
438: +    packer - the packer object
439: .    gvec - the global vector
440: -    ... - the individual sequential objects (arrays or vectors)
441:  
442:     Level: advanced

444: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
445:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
446:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

448: @*/
449: PetscErrorCode  DMCompositeScatter(DMComposite packer,Vec gvec,...)
450: {
451:   va_list                Argp;
452:   PetscErrorCode         ierr;
453:   struct DMCompositeLink *next;
454:   PetscInt               cnt = 3;

459:   next = packer->next;
460:   if (!packer->setup) {
461:     DMCompositeSetUp(packer);
462:   }

464:   /* loop over packed objects, handling one at at time */
465:   va_start(Argp,gvec);
466:   while (next) {
467:     if (next->type == DMCOMPOSITE_ARRAY) {
468:       PetscScalar *array;
469:       array = va_arg(Argp, PetscScalar*);
470:       DMCompositeScatter_Array(packer,next,gvec,array);
471:     } else if (next->type == DMCOMPOSITE_DA) {
472:       Vec vec;
473:       vec = va_arg(Argp, Vec);
475:       DMCompositeScatter_DA(packer,next,gvec,vec);
476:     } else {
477:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
478:     }
479:     cnt++;
480:     next = next->next;
481:   }
482:   va_end(Argp);
483:   return(0);
484: }

488: /*@C
489:     DMCompositeGather - Gathers into a global packed vector from its individual local vectors

491:     Collective on DMComposite

493:     Input Parameter:
494: +    packer - the packer object
495: .    gvec - the global vector
496: -    ... - the individual sequential objects (arrays or vectors)
497:  
498:     Level: advanced

500: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
501:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
502:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

504: @*/
505: PetscErrorCode  DMCompositeGather(DMComposite packer,Vec gvec,...)
506: {
507:   va_list                Argp;
508:   PetscErrorCode         ierr;
509:   struct DMCompositeLink *next;

514:   next = packer->next;
515:   if (!packer->setup) {
516:     DMCompositeSetUp(packer);
517:   }

519:   /* loop over packed objects, handling one at at time */
520:   va_start(Argp,gvec);
521:   while (next) {
522:     if (next->type == DMCOMPOSITE_ARRAY) {
523:       PetscScalar *array;
524:       array = va_arg(Argp, PetscScalar*);
525:       DMCompositeGather_Array(packer,next,gvec,array);
526:     } else if (next->type == DMCOMPOSITE_DA) {
527:       Vec vec;
528:       vec = va_arg(Argp, Vec);
530:       DMCompositeGather_DA(packer,next,gvec,vec);
531:     } else {
532:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
533:     }
534:     next = next->next;
535:   }
536:   va_end(Argp);
537:   return(0);
538: }

542: /*@C
543:     DMCompositeAddArray - adds an "redundant" array to a DMComposite. The array values will 
544:        be stored in part of the array on process orank.

546:     Collective on DMComposite

548:     Input Parameter:
549: +    packer - the packer object
550: .    orank - the process on which the array entries officially live, this number must be
551:              the same on all processes.
552: -    n - the length of the array
553:  
554:     Level: advanced

556: .seealso DMCompositeDestroy(), DMCompositeGather(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
557:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
558:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

560: @*/
561: PetscErrorCode  DMCompositeAddArray(DMComposite packer,PetscMPIInt orank,PetscInt n)
562: {
563:   struct DMCompositeLink *mine,*next;
564:   PetscErrorCode         ierr;
565:   PetscMPIInt            rank;

569:   next = packer->next;
570:   if (packer->setup) {
571:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Cannot add an array once you have used the DMComposite");
572:   }
573: #if defined(PETSC_USE_DEBUG)
574:   {
575:     PetscMPIInt        orankmax;
576:     MPI_Allreduce(&orank,&orankmax,1,MPI_INT,MPI_MAX,((PetscObject)packer)->comm);
577:     if (orank != orankmax) SETERRQ2(PETSC_ERR_ARG_INCOMP,"orank %d must be equal on all processes, another process has value %d",orank,orankmax);
578:   }
579: #endif

581:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
582:   /* create new link */
583:   PetscNew(struct DMCompositeLink,&mine);
584:   mine->n             = n;
585:   mine->rank          = orank;
586:   mine->da            = PETSC_NULL;
587:   mine->type          = DMCOMPOSITE_ARRAY;
588:   mine->next          = PETSC_NULL;
589:   if (rank == mine->rank) packer->n += n;

591:   /* add to end of list */
592:   if (!next) {
593:     packer->next = mine;
594:   } else {
595:     while (next->next) next = next->next;
596:     next->next = mine;
597:   }
598:   packer->nredundant++;
599:   return(0);
600: }

604: /*@C
605:     DMCompositeAddDA - adds a DA vector to a DMComposite

607:     Collective on DMComposite

609:     Input Parameter:
610: +    packer - the packer object
611: -    da - the DA object
612:  
613:     Level: advanced

615: .seealso DMCompositeDestroy(), DMCompositeGather(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
616:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
617:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

619: @*/
620: PetscErrorCode  DMCompositeAddDA(DMComposite packer,DA da)
621: {
622:   PetscErrorCode         ierr;
623:   PetscInt               n;
624:   struct DMCompositeLink *mine,*next;
625:   Vec                    global;

630:   next = packer->next;
631:   if (packer->setup) {
632:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Cannot add a DA once you have used the DMComposite");
633:   }

635:   /* create new link */
636:   PetscNew(struct DMCompositeLink,&mine);
637:   PetscObjectReference((PetscObject)da);
638:   DAGetGlobalVector(da,&global);
639:   VecGetLocalSize(global,&n);
640:   DARestoreGlobalVector(da,&global);
641:   mine->n      = n;
642:   mine->da     = da;
643:   mine->type   = DMCOMPOSITE_DA;
644:   mine->next   = PETSC_NULL;
645:   packer->n   += n;

647:   /* add to end of list */
648:   if (!next) {
649:     packer->next = mine;
650:   } else {
651:     while (next->next) next = next->next;
652:     next->next = mine;
653:   }
654:   packer->nDA++;
655:   return(0);
656: }

662: PetscErrorCode  VecView_DMComposite(Vec gvec,PetscViewer viewer)
663: {
664:   DMComposite            packer;
665:   PetscErrorCode         ierr;
666:   struct DMCompositeLink *next;
667:   PetscTruth             isdraw;

670:   PetscObjectQuery((PetscObject)gvec,"DMComposite",(PetscObject*)&packer);
671:   if (!packer) SETERRQ(PETSC_ERR_ARG_WRONG,"Vector not generated from a DMComposite");
672:   next = packer->next;

674:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
675:   if (!isdraw) {
676:     /* do I really want to call this? */
677:     VecView_MPI(gvec,viewer);
678:   } else {
679:     PetscInt cnt = 0;

681:     /* loop over packed objects, handling one at at time */
682:     while (next) {
683:       if (next->type == DMCOMPOSITE_ARRAY) {
684:         PetscScalar *array;
685:         DMCompositeGetAccess_Array(packer,next,gvec,&array);

687:         /*skip it for now */
688:       } else if (next->type == DMCOMPOSITE_DA) {
689:         Vec      vec;
690:         PetscInt bs;

692:         DMCompositeGetAccess_DA(packer,next,gvec,&vec);
693:         VecView(vec,viewer);
694:         VecGetBlockSize(vec,&bs);
695:         DMCompositeRestoreAccess_DA(packer,next,gvec,&vec);
696:         PetscViewerDrawBaseAdd(viewer,bs);
697:         cnt += bs;
698:       } else {
699:         SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
700:       }
701:       next = next->next;
702:     }
703:     PetscViewerDrawBaseAdd(viewer,-cnt);
704:   }
705:   return(0);
706: }

711: /*@C
712:     DMCompositeCreateGlobalVector - Creates a vector of the correct size to be gathered into 
713:         by the packer.

715:     Collective on DMComposite

717:     Input Parameter:
718: .    packer - the packer object

720:     Output Parameters:
721: .   gvec - the global vector

723:     Level: advanced

725:     Notes: Once this has been created you cannot add additional arrays or vectors to be packed.

727: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeScatter(),
728:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
729:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

731: @*/
732: PetscErrorCode  DMCompositeCreateGlobalVector(DMComposite packer,Vec *gvec)
733: {
734:   PetscErrorCode         ierr;

738:   if (!packer->setup) {
739:     DMCompositeSetUp(packer);
740:   }
741:   VecCreateMPI(((PetscObject)packer)->comm,packer->n,packer->N,gvec);
742:   PetscObjectCompose((PetscObject)*gvec,"DMComposite",(PetscObject)packer);
743:   VecSetOperation(*gvec,VECOP_VIEW,(void(*)(void))VecView_DMComposite);
744:   return(0);
745: }

749: /*@C
750:     DMCompositeGetGlobalIndices - Gets the global indices for all the entries in the packed
751:       vectors.

753:     Collective on DMComposite

755:     Input Parameter:
756: .    packer - the packer object

758:     Output Parameters:
759: .    idx - the individual indices for each packed vector/array. Note that this includes
760:            all the ghost points that individual ghosted DA's may have.
761:  
762:     Level: advanced

764:     Notes:
765:        The idx parameters should be freed by the calling routine with PetscFree()

767: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
768:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
769:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()

771: @*/
772: PetscErrorCode  DMCompositeGetGlobalIndices(DMComposite packer,...)
773: {
774:   va_list                Argp;
775:   PetscErrorCode         ierr;
776:   PetscInt               i,**idx,n;
777:   struct DMCompositeLink *next;
778:   Vec                    global,dglobal;
779:   PF                     pf;
780:   PetscScalar            *array;
781:   PetscMPIInt            rank;

785:   next = packer->next;
786:   DMCompositeCreateGlobalVector(packer,&global);
787:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);

789:   /* put 0 to N-1 into the global vector */
790:   PFCreate(PETSC_COMM_WORLD,1,1,&pf);
791:   PFSetType(pf,PFIDENTITY,PETSC_NULL);
792:   PFApplyVec(pf,PETSC_NULL,global);
793:   PFDestroy(pf);

795:   /* loop over packed objects, handling one at at time */
796:   va_start(Argp,packer);
797:   while (next) {
798:     idx = va_arg(Argp, PetscInt**);

800:     if (next->type == DMCOMPOSITE_ARRAY) {
801: 
802:       PetscMalloc(next->n*sizeof(PetscInt),idx);
803:       if (rank == next->rank) {
804:         VecGetArray(global,&array);
805:         array += next->rstart;
806:         for (i=0; i<next->n; i++) (*idx)[i] = (PetscInt)PetscRealPart(array[i]);
807:         array -= next->rstart;
808:         VecRestoreArray(global,&array);
809:       }
810:       MPI_Bcast(*idx,next->n,MPIU_INT,next->rank,((PetscObject)packer)->comm);

812:     } else if (next->type == DMCOMPOSITE_DA) {
813:       Vec local;

815:       DACreateLocalVector(next->da,&local);
816:       VecGetArray(global,&array);
817:       array += next->rstart;
818:       DAGetGlobalVector(next->da,&dglobal);
819:       VecPlaceArray(dglobal,array);
820:       DAGlobalToLocalBegin(next->da,dglobal,INSERT_VALUES,local);
821:       DAGlobalToLocalEnd(next->da,dglobal,INSERT_VALUES,local);
822:       array -= next->rstart;
823:       VecRestoreArray(global,&array);
824:       VecResetArray(dglobal);
825:       DARestoreGlobalVector(next->da,&dglobal);

827:       VecGetArray(local,&array);
828:       VecGetSize(local,&n);
829:       PetscMalloc(n*sizeof(PetscInt),idx);
830:       for (i=0; i<n; i++) (*idx)[i] = (PetscInt)PetscRealPart(array[i]);
831:       VecRestoreArray(local,&array);
832:       VecDestroy(local);

834:     } else {
835:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
836:     }
837:     next = next->next;
838:   }
839:   va_end(Argp);
840:   VecDestroy(global);
841:   return(0);
842: }

844: /* -------------------------------------------------------------------------------------*/
847: PetscErrorCode DMCompositeGetLocalVectors_Array(DMComposite packer,struct DMCompositeLink *mine,PetscScalar **array)
848: {
851:   if (array) {
852:     PetscMalloc(mine->n*sizeof(PetscScalar),array);
853:   }
854:   return(0);
855: }

859: PetscErrorCode DMCompositeGetLocalVectors_DA(DMComposite packer,struct DMCompositeLink *mine,Vec *local)
860: {
863:   if (local) {
864:     DAGetLocalVector(mine->da,local);
865:   }
866:   return(0);
867: }

871: PetscErrorCode DMCompositeRestoreLocalVectors_Array(DMComposite packer,struct DMCompositeLink *mine,PetscScalar **array)
872: {
875:   if (array) {
876:     PetscFree(*array);
877:   }
878:   return(0);
879: }

883: PetscErrorCode DMCompositeRestoreLocalVectors_DA(DMComposite packer,struct DMCompositeLink *mine,Vec *local)
884: {
887:   if (local) {
888:     DARestoreLocalVector(mine->da,local);
889:   }
890:   return(0);
891: }

895: /*@C
896:     DMCompositeGetLocalVectors - Gets local vectors and arrays for each part of a DMComposite.'
897:        Use VecPakcRestoreLocalVectors() to return them.

899:     Collective on DMComposite

901:     Input Parameter:
902: .    packer - the packer object
903:  
904:     Output Parameter:
905: .    ... - the individual sequential objects (arrays or vectors)
906:  
907:     Level: advanced

909: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
910:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(), 
911:          DMCompositeRestoreLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()

913: @*/
914: PetscErrorCode  DMCompositeGetLocalVectors(DMComposite packer,...)
915: {
916:   va_list                Argp;
917:   PetscErrorCode         ierr;
918:   struct DMCompositeLink *next;

922:   next = packer->next;
923:   /* loop over packed objects, handling one at at time */
924:   va_start(Argp,packer);
925:   while (next) {
926:     if (next->type == DMCOMPOSITE_ARRAY) {
927:       PetscScalar **array;
928:       array = va_arg(Argp, PetscScalar**);
929:       DMCompositeGetLocalVectors_Array(packer,next,array);
930:     } else if (next->type == DMCOMPOSITE_DA) {
931:       Vec *vec;
932:       vec = va_arg(Argp, Vec*);
933:       DMCompositeGetLocalVectors_DA(packer,next,vec);
934:     } else {
935:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
936:     }
937:     next = next->next;
938:   }
939:   va_end(Argp);
940:   return(0);
941: }

945: /*@C
946:     DMCompositeRestoreLocalVectors - Restores local vectors and arrays for each part of a DMComposite.'
947:        Use VecPakcRestoreLocalVectors() to return them.

949:     Collective on DMComposite

951:     Input Parameter:
952: .    packer - the packer object
953:  
954:     Output Parameter:
955: .    ... - the individual sequential objects (arrays or vectors)
956:  
957:     Level: advanced

959: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
960:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(), 
961:          DMCompositeGetLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()

963: @*/
964: PetscErrorCode  DMCompositeRestoreLocalVectors(DMComposite packer,...)
965: {
966:   va_list                Argp;
967:   PetscErrorCode         ierr;
968:   struct DMCompositeLink *next;

972:   next = packer->next;
973:   /* loop over packed objects, handling one at at time */
974:   va_start(Argp,packer);
975:   while (next) {
976:     if (next->type == DMCOMPOSITE_ARRAY) {
977:       PetscScalar **array;
978:       array = va_arg(Argp, PetscScalar**);
979:       DMCompositeRestoreLocalVectors_Array(packer,next,array);
980:     } else if (next->type == DMCOMPOSITE_DA) {
981:       Vec *vec;
982:       vec = va_arg(Argp, Vec*);
983:       DMCompositeRestoreLocalVectors_DA(packer,next,vec);
984:     } else {
985:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
986:     }
987:     next = next->next;
988:   }
989:   va_end(Argp);
990:   return(0);
991: }

993: /* -------------------------------------------------------------------------------------*/
996: PetscErrorCode DMCompositeGetEntries_Array(DMComposite packer,struct DMCompositeLink *mine,PetscInt *n)
997: {
999:   if (n) *n = mine->n;
1000:   return(0);
1001: }

1005: PetscErrorCode DMCompositeGetEntries_DA(DMComposite packer,struct DMCompositeLink *mine,DA *da)
1006: {
1008:   if (da) *da = mine->da;
1009:   return(0);
1010: }

1014: /*@C
1015:     DMCompositeGetEntries - Gets the DA, redundant size, etc for each entry in a DMComposite.

1017:     Collective on DMComposite

1019:     Input Parameter:
1020: .    packer - the packer object
1021:  
1022:     Output Parameter:
1023: .    ... - the individual entries, DAs or integer sizes)
1024:  
1025:     Level: advanced

1027: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
1028:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(), 
1029:          DMCompositeRestoreLocalVectors(), DMCompositeGetLocalVectors(),  DMCompositeScatter(),
1030:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors()

1032: @*/
1033: PetscErrorCode  DMCompositeGetEntries(DMComposite packer,...)
1034: {
1035:   va_list                Argp;
1036:   PetscErrorCode         ierr;
1037:   struct DMCompositeLink *next;

1041:   next = packer->next;
1042:   /* loop over packed objects, handling one at at time */
1043:   va_start(Argp,packer);
1044:   while (next) {
1045:     if (next->type == DMCOMPOSITE_ARRAY) {
1046:       PetscInt *n;
1047:       n = va_arg(Argp, PetscInt*);
1048:       DMCompositeGetEntries_Array(packer,next,n);
1049:     } else if (next->type == DMCOMPOSITE_DA) {
1050:       DA *da;
1051:       da = va_arg(Argp, DA*);
1052:       DMCompositeGetEntries_DA(packer,next,da);
1053:     } else {
1054:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1055:     }
1056:     next = next->next;
1057:   }
1058:   va_end(Argp);
1059:   return(0);
1060: }

1064: /*@C
1065:     DMCompositeRefine - Refines a DMComposite by refining all of its DAs

1067:     Collective on DMComposite

1069:     Input Parameters:
1070: +    packer - the packer object
1071: -    comm - communicator to contain the new DM object, usually PETSC_NULL

1073:     Output Parameter:
1074: .    fine - new packer
1075:  
1076:     Level: advanced

1078: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
1079:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
1080:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),  DMCompositeScatter(),
1081:          DMCompositeGetEntries()

1083: @*/
1084: PetscErrorCode  DMCompositeRefine(DMComposite packer,MPI_Comm comm,DMComposite *fine)
1085: {
1086:   PetscErrorCode         ierr;
1087:   struct DMCompositeLink *next;
1088:   DA                     da;

1092:   next = packer->next;
1093:   DMCompositeCreate(comm,fine);

1095:   /* loop over packed objects, handling one at at time */
1096:   while (next) {
1097:     if (next->type == DMCOMPOSITE_ARRAY) {
1098:       DMCompositeAddArray(*fine,next->rank,next->n);
1099:     } else if (next->type == DMCOMPOSITE_DA) {
1100:       DARefine(next->da,comm,&da);
1101:       DMCompositeAddDA(*fine,da);
1102:       PetscObjectDereference((PetscObject)da);
1103:     } else {
1104:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1105:     }
1106:     next = next->next;
1107:   }
1108:   return(0);
1109: }

1111:  #include petscmat.h

1113: struct MatPackLink {
1114:   Mat                A;
1115:   struct MatPackLink *next;
1116: };

1118: struct MatPack {
1119:   DMComposite            right,left;
1120:   struct MatPackLink *next;
1121: };

1125: PetscErrorCode MatMultBoth_Shell_Pack(Mat A,Vec x,Vec y,PetscTruth add)
1126: {
1127:   struct MatPack         *mpack;
1128:   struct DMCompositeLink *xnext,*ynext;
1129:   struct MatPackLink     *anext;
1130:   PetscScalar            *xarray,*yarray;
1131:   PetscErrorCode         ierr;
1132:   PetscInt               i;
1133:   Vec                    xglobal,yglobal;
1134:   PetscMPIInt            rank;

1137:   MatShellGetContext(A,(void**)&mpack);
1138:   MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1139:   xnext = mpack->right->next;
1140:   ynext = mpack->left->next;
1141:   anext = mpack->next;

1143:   while (xnext) {
1144:     if (xnext->type == DMCOMPOSITE_ARRAY) {
1145:       if (rank == xnext->rank) {
1146:         VecGetArray(x,&xarray);
1147:         VecGetArray(y,&yarray);
1148:         if (add) {
1149:           for (i=0; i<xnext->n; i++) {
1150:             yarray[ynext->rstart+i] += xarray[xnext->rstart+i];
1151:           }
1152:         } else {
1153:           PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1154:         }
1155:         VecRestoreArray(x,&xarray);
1156:         VecRestoreArray(y,&yarray);
1157:       }
1158:     } else if (xnext->type == DMCOMPOSITE_DA) {
1159:       VecGetArray(x,&xarray);
1160:       VecGetArray(y,&yarray);
1161:       DAGetGlobalVector(xnext->da,&xglobal);
1162:       DAGetGlobalVector(ynext->da,&yglobal);
1163:       VecPlaceArray(xglobal,xarray+xnext->rstart);
1164:       VecPlaceArray(yglobal,yarray+ynext->rstart);
1165:       if (add) {
1166:         MatMultAdd(anext->A,xglobal,yglobal,yglobal);
1167:       } else {
1168:         MatMult(anext->A,xglobal,yglobal);
1169:       }
1170:       VecRestoreArray(x,&xarray);
1171:       VecRestoreArray(y,&yarray);
1172:       VecResetArray(xglobal);
1173:       VecResetArray(yglobal);
1174:       DARestoreGlobalVector(xnext->da,&xglobal);
1175:       DARestoreGlobalVector(ynext->da,&yglobal);
1176:       anext = anext->next;
1177:     } else {
1178:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1179:     }
1180:     xnext = xnext->next;
1181:     ynext = ynext->next;
1182:   }
1183:   return(0);
1184: }

1188: PetscErrorCode MatMultAdd_Shell_Pack(Mat A,Vec x,Vec y,Vec z)
1189: {
1192:   if (z != y) SETERRQ(PETSC_ERR_SUP,"Handles y == z only");
1193:   MatMultBoth_Shell_Pack(A,x,y,PETSC_TRUE);
1194:   return(0);
1195: }

1199: PetscErrorCode MatMult_Shell_Pack(Mat A,Vec x,Vec y)
1200: {
1203:   MatMultBoth_Shell_Pack(A,x,y,PETSC_FALSE);
1204:   return(0);
1205: }

1209: PetscErrorCode MatMultTranspose_Shell_Pack(Mat A,Vec x,Vec y)
1210: {
1211:   struct MatPack         *mpack;
1212:   struct DMCompositeLink *xnext,*ynext;
1213:   struct MatPackLink     *anext;
1214:   PetscScalar            *xarray,*yarray;
1215:   PetscErrorCode         ierr;
1216:   Vec                    xglobal,yglobal;
1217:   PetscMPIInt            rank;

1220:   MatShellGetContext(A,(void**)&mpack);
1221:   MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1222:   xnext = mpack->left->next;
1223:   ynext = mpack->right->next;
1224:   anext = mpack->next;

1226:   while (xnext) {
1227:     if (xnext->type == DMCOMPOSITE_ARRAY) {
1228:       if (rank == ynext->rank) {
1229:         VecGetArray(x,&xarray);
1230:         VecGetArray(y,&yarray);
1231:         PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1232:         VecRestoreArray(x,&xarray);
1233:         VecRestoreArray(y,&yarray);
1234:       }
1235:     } else if (xnext->type == DMCOMPOSITE_DA) {
1236:       VecGetArray(x,&xarray);
1237:       VecGetArray(y,&yarray);
1238:       DAGetGlobalVector(xnext->da,&xglobal);
1239:       DAGetGlobalVector(ynext->da,&yglobal);
1240:       VecPlaceArray(xglobal,xarray+xnext->rstart);
1241:       VecPlaceArray(yglobal,yarray+ynext->rstart);
1242:       MatMultTranspose(anext->A,xglobal,yglobal);
1243:       VecRestoreArray(x,&xarray);
1244:       VecRestoreArray(y,&yarray);
1245:       VecResetArray(xglobal);
1246:       VecResetArray(yglobal);
1247:       DARestoreGlobalVector(xnext->da,&xglobal);
1248:       DARestoreGlobalVector(ynext->da,&yglobal);
1249:       anext = anext->next;
1250:     } else {
1251:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1252:     }
1253:     xnext = xnext->next;
1254:     ynext = ynext->next;
1255:   }
1256:   return(0);
1257: }

1261: PetscErrorCode MatDestroy_Shell_Pack(Mat A)
1262: {
1263:   struct MatPack     *mpack;
1264:   struct MatPackLink *anext,*oldanext;
1265:   PetscErrorCode     ierr;

1268:   MatShellGetContext(A,(void**)&mpack);
1269:   anext = mpack->next;

1271:   while (anext) {
1272:     MatDestroy(anext->A);
1273:     oldanext = anext;
1274:     anext    = anext->next;
1275:     PetscFree(oldanext);
1276:   }
1277:   PetscFree(mpack);
1278:   PetscObjectChangeTypeName((PetscObject)A,0);
1279:   return(0);
1280: }

1284: /*@C
1285:     DMCompositeGetInterpolation - GetInterpolations a DMComposite by refining all of its DAs

1287:     Collective on DMComposite

1289:     Input Parameters:
1290: +    coarse - coarse grid packer
1291: -    fine - fine grid packer

1293:     Output Parameter:
1294: +    A - interpolation matrix
1295: -    v - scaling vector
1296:  
1297:     Level: advanced

1299: .seealso DMCompositeDestroy(), DMCompositeAddArray(), DMCompositeAddDA(), DMCompositeCreateGlobalVector(),
1300:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetGlobalIndices(), DMCompositeGetAccess(),
1301:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),  DMCompositeScatter(),DMCompositeGetEntries()

1303: @*/
1304: PetscErrorCode  DMCompositeGetInterpolation(DMComposite coarse,DMComposite fine,Mat *A,Vec *v)
1305: {
1306:   PetscErrorCode         ierr;
1307:   PetscInt               m,n,M,N;
1308:   struct DMCompositeLink *nextc;
1309:   struct DMCompositeLink *nextf;
1310:   struct MatPackLink     *nextmat,*pnextmat = 0;
1311:   struct MatPack         *mpack;
1312:   Vec                    gcoarse,gfine;

1317:   nextc = coarse->next;
1318:   nextf = fine->next;
1319:   /* use global vectors only for determining matrix layout */
1320:   DMCompositeCreateGlobalVector(coarse,&gcoarse);
1321:   DMCompositeCreateGlobalVector(fine,&gfine);
1322:   VecGetLocalSize(gcoarse,&n);
1323:   VecGetLocalSize(gfine,&m);
1324:   VecGetSize(gcoarse,&N);
1325:   VecGetSize(gfine,&M);
1326:   VecDestroy(gcoarse);
1327:   VecDestroy(gfine);

1329:   PetscNew(struct MatPack,&mpack);
1330:   mpack->right = coarse;
1331:   mpack->left  = fine;
1332:   MatCreate(((PetscObject)fine)->comm,A);
1333:   MatSetSizes(*A,m,n,M,N);
1334:   MatSetType(*A,MATSHELL);
1335:   MatShellSetContext(*A,mpack);
1336:   MatShellSetOperation(*A,MATOP_MULT,(void(*)(void))MatMult_Shell_Pack);
1337:   MatShellSetOperation(*A,MATOP_MULT_TRANSPOSE,(void(*)(void))MatMultTranspose_Shell_Pack);
1338:   MatShellSetOperation(*A,MATOP_MULT_ADD,(void(*)(void))MatMultAdd_Shell_Pack);
1339:   MatShellSetOperation(*A,MATOP_DESTROY,(void(*)(void))MatDestroy_Shell_Pack);

1341:   /* loop over packed objects, handling one at at time */
1342:   while (nextc) {
1343:     if (nextc->type != nextf->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Two DMComposite have different layout");

1345:     if (nextc->type == DMCOMPOSITE_ARRAY) {
1346:       ;
1347:     } else if (nextc->type == DMCOMPOSITE_DA) {
1348:       PetscNew(struct MatPackLink,&nextmat);
1349:       nextmat->next = 0;
1350:       if (pnextmat) {
1351:         pnextmat->next = nextmat;
1352:         pnextmat       = nextmat;
1353:       } else {
1354:         pnextmat    = nextmat;
1355:         mpack->next = nextmat;
1356:       }
1357:       DAGetInterpolation(nextc->da,nextf->da,&nextmat->A,PETSC_NULL);
1358:     } else {
1359:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1360:     }
1361:     nextc = nextc->next;
1362:     nextf = nextf->next;
1363:   }
1364:   return(0);
1365: }

1369: /*@C
1370:     DMCompositeGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for 
1371:       computing the Jacobian on a function defined using the stencils set in the DA's and coupling in the array variables

1373:     Collective on DA

1375:     Input Parameter:
1376: +   da - the distributed array
1377: -   mtype - Supported types are MATSEQAIJ, MATMPIAIJ

1379:     Output Parameters:
1380: .   J  - matrix with the correct nonzero structure
1381:         (obviously without the correct Jacobian values)

1383:     Level: advanced

1385:     Notes: This properly preallocates the number of nonzeros in the sparse matrix so you 
1386:        do not need to do it yourself. 


1389: .seealso DAGetMatrix(), DMCompositeCreate()

1391: @*/
1392: PetscErrorCode  DMCompositeGetMatrix(DMComposite packer, MatType mtype,Mat *J)
1393: {
1394:   PetscErrorCode         ierr;
1395:   struct DMCompositeLink *next = packer->next;
1396:   PetscInt               m,*dnz,*onz,i,j,mA;
1397:   Mat                    Atmp;
1398:   PetscMPIInt            rank;
1399:   PetscScalar            zero = 0.0;

1403:   next = packer->next;

1405:   /* use global vector to determine layout needed for matrix */
1406:   m = packer->n;
1407:   MPI_Comm_rank(((PetscObject)packer)->comm,&rank);
1408:   MatCreate(((PetscObject)packer)->comm,J);
1409:   MatSetSizes(*J,m,m,PETSC_DETERMINE,PETSC_DETERMINE);
1410:   MatSetType(*J,mtype);

1412:   MatPreallocateInitialize(((PetscObject)packer)->comm,m,m,dnz,onz);
1413:   /* loop over packed objects, handling one at at time */
1414:   while (next) {
1415:     if (next->type == DMCOMPOSITE_ARRAY) {
1416:       if (rank == next->rank) {  /* zero the "little" block */
1417:         for (j=packer->rstart+next->rstart; j<packer->rstart+next->rstart+next->n; j++) {
1418:           for (i=packer->rstart+next->rstart; i<packer->rstart+next->rstart+next->n; i++) {
1419:             MatPreallocateSet(j,1,&i,dnz,onz);
1420:           }
1421:         }
1422:       }
1423:     } else if (next->type == DMCOMPOSITE_DA) {
1424:       PetscInt       nc,rstart,*ccols,maxnc;
1425:       const PetscInt *cols,*rstarts;
1426:       PetscMPIInt    proc;

1428:       DAGetMatrix(next->da,mtype,&Atmp);
1429:       MatGetOwnershipRange(Atmp,&rstart,PETSC_NULL);
1430:       MatGetOwnershipRanges(Atmp,&rstarts);
1431:       MatGetLocalSize(Atmp,&mA,PETSC_NULL);

1433:       maxnc = 0;
1434:       for (i=0; i<mA; i++) {
1435:         MatGetRow(Atmp,rstart+i,&nc,PETSC_NULL,PETSC_NULL);
1436:         MatRestoreRow(Atmp,rstart+i,&nc,PETSC_NULL,PETSC_NULL);
1437:         maxnc = PetscMax(nc,maxnc);
1438:       }
1439:       PetscMalloc(maxnc*sizeof(PetscInt),&ccols);
1440:       for (i=0; i<mA; i++) {
1441:         MatGetRow(Atmp,rstart+i,&nc,&cols,PETSC_NULL);
1442:         /* remap the columns taking into how much they are shifted on each process */
1443:         for (j=0; j<nc; j++) {
1444:           proc = 0;
1445:           while (cols[j] >= rstarts[proc+1]) proc++;
1446:           ccols[j] = cols[j] + next->grstarts[proc] - rstarts[proc];
1447:         }
1448:         MatPreallocateSet(packer->rstart+next->rstart+i,nc,ccols,dnz,onz);
1449:         MatRestoreRow(Atmp,rstart+i,&nc,&cols,PETSC_NULL);
1450:       }
1451:       PetscFree(ccols);
1452:       MatDestroy(Atmp);
1453:     } else {
1454:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1455:     }
1456:     next = next->next;
1457:   }
1458:   MatMPIAIJSetPreallocation(*J,0,dnz,0,onz);
1459:   MatSeqAIJSetPreallocation(*J,0,dnz);
1460:   MatPreallocateFinalize(dnz,onz);

1462:   next = packer->next;
1463:   while (next) {
1464:     if (next->type == DMCOMPOSITE_ARRAY) {
1465:       if (rank == next->rank) {
1466:         for (j=packer->rstart+next->rstart; j<packer->rstart+next->rstart+next->n; j++) {
1467:           for (i=packer->rstart+next->rstart; i<packer->rstart+next->rstart+next->n; i++) {
1468:             MatSetValues(*J,1,&j,1,&i,&zero,INSERT_VALUES);
1469:           }
1470:         }
1471:       }
1472:     } else if (next->type == DMCOMPOSITE_DA) {
1473:       PetscInt          nc,rstart,row,maxnc,*ccols;
1474:       const PetscInt    *cols,*rstarts;
1475:       const PetscScalar *values;
1476:       PetscMPIInt       proc;

1478:       DAGetMatrix(next->da,mtype,&Atmp);
1479:       MatGetOwnershipRange(Atmp,&rstart,PETSC_NULL);
1480:       MatGetOwnershipRanges(Atmp,&rstarts);
1481:       MatGetLocalSize(Atmp,&mA,PETSC_NULL);
1482:       maxnc = 0;
1483:       for (i=0; i<mA; i++) {
1484:         MatGetRow(Atmp,rstart+i,&nc,PETSC_NULL,PETSC_NULL);
1485:         MatRestoreRow(Atmp,rstart+i,&nc,PETSC_NULL,PETSC_NULL);
1486:         maxnc = PetscMax(nc,maxnc);
1487:       }
1488:       PetscMalloc(maxnc*sizeof(PetscInt),&ccols);
1489:       for (i=0; i<mA; i++) {
1490:         MatGetRow(Atmp,rstart+i,&nc,(const PetscInt **)&cols,&values);
1491:         for (j=0; j<nc; j++) {
1492:           proc = 0;
1493:           while (cols[j] >= rstarts[proc+1]) proc++;
1494:           ccols[j] = cols[j] + next->grstarts[proc] - rstarts[proc];
1495:         }
1496:         row  = packer->rstart+next->rstart+i;
1497:         MatSetValues(*J,1,&row,nc,ccols,values,INSERT_VALUES);
1498:         MatRestoreRow(Atmp,rstart+i,&nc,(const PetscInt **)&cols,&values);
1499:       }
1500:       PetscFree(ccols);
1501:       MatDestroy(Atmp);
1502:     } else {
1503:       SETERRQ(PETSC_ERR_SUP,"Cannot handle that object type yet");
1504:     }
1505:     next = next->next;
1506:   }
1507:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
1508:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
1509:   return(0);
1510: }

1514: /*@
1515:     DMCompositeGetColoring - Gets the coloring required for computing the Jacobian via
1516:     finite differences on a function defined using a DMComposite "grid"

1518:     Collective on DA

1520:     Input Parameter:
1521: +   dmcomposite - the DMComposite object
1522: -   ctype - IS_COLORING_GLOBAL or IS_COLORING_GHOSTED

1524:     Output Parameters:
1525: .   coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)

1527:     Level: advanced

1529:     Notes: This currentlu uses one color per column so is very slow.

1531:     Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used 
1532:    for efficient (parallel or thread based) triangular solves etc is NOT yet 
1533:    available. 


1536: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring, DAGetColoring()

1538: @*/
1539: PetscErrorCode  DMCompositeGetColoring(DMComposite dmcomposite,ISColoringType ctype,ISColoring *coloring)
1540: {
1541:   PetscErrorCode  ierr;
1542:   PetscInt        n,i;
1543:   ISColoringValue *colors;

1547:   if (ctype == IS_COLORING_GHOSTED) {
1548:     SETERRQ(PETSC_ERR_SUP,"Lazy Barry");
1549:   } else if (ctype == IS_COLORING_GLOBAL) {
1550:     n = dmcomposite->n;
1551:   } else SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Unknown ISColoringType");

1553:   PetscMalloc(n*sizeof(ISColoringValue),&colors); /* freed in ISColoringDestroy() */
1554:   for (i=0; i<n; i++) {
1555:     colors[i] = (ISColoringValue)(dmcomposite->rstart + i);
1556:   }
1557:   ISColoringCreate(((PetscObject)dmcomposite)->comm,dmcomposite->N,n,colors,coloring);
1558:   return(0);
1559: }