Actual source code: dalocal.c

petsc-dev 2014-02-02
Report Typos and Errors
  2: /*
  3:   Code for manipulating distributed regular arrays in parallel.
  4: */

  6: #include <petsc-private/dmdaimpl.h>    /*I   "petscdmda.h"   I*/
  7: #include <petscbt.h>
  8: #include <petscsf.h>
  9: #include <petscfe.h>

 11: /*
 12:    This allows the DMDA vectors to properly tell MATLAB their dimensions
 13: */
 14: #if defined(PETSC_HAVE_MATLAB_ENGINE)
 15: #include <engine.h>   /* MATLAB include file */
 16: #include <mex.h>      /* MATLAB include file */
 19: static PetscErrorCode  VecMatlabEnginePut_DA2d(PetscObject obj,void *mengine)
 20: {
 22:   PetscInt       n,m;
 23:   Vec            vec = (Vec)obj;
 24:   PetscScalar    *array;
 25:   mxArray        *mat;
 26:   DM             da;

 29:   VecGetDM(vec, &da);
 30:   if (!da) SETERRQ(PetscObjectComm((PetscObject)vec),PETSC_ERR_ARG_WRONGSTATE,"Vector not associated with a DMDA");
 31:   DMDAGetGhostCorners(da,0,0,0,&m,&n,0);

 33:   VecGetArray(vec,&array);
 34: #if !defined(PETSC_USE_COMPLEX)
 35:   mat = mxCreateDoubleMatrix(m,n,mxREAL);
 36: #else
 37:   mat = mxCreateDoubleMatrix(m,n,mxCOMPLEX);
 38: #endif
 39:   PetscMemcpy(mxGetPr(mat),array,n*m*sizeof(PetscScalar));
 40:   PetscObjectName(obj);
 41:   engPutVariable((Engine*)mengine,obj->name,mat);

 43:   VecRestoreArray(vec,&array);
 44:   return(0);
 45: }
 46: #endif


 51: PetscErrorCode  DMCreateLocalVector_DA(DM da,Vec *g)
 52: {
 54:   DM_DA          *dd = (DM_DA*)da->data;

 59:   if (da->defaultSection) {
 60:     DMCreateLocalVector_Section_Private(da,g);
 61:   } else {
 62:     VecCreate(PETSC_COMM_SELF,g);
 63:     VecSetSizes(*g,dd->nlocal,PETSC_DETERMINE);
 64:     VecSetBlockSize(*g,dd->w);
 65:     VecSetType(*g,da->vectype);
 66:     VecSetDM(*g, da);
 67: #if defined(PETSC_HAVE_MATLAB_ENGINE)
 68:     if (dd->w == 1  && dd->dim == 2) {
 69:       PetscObjectComposeFunction((PetscObject)*g,"PetscMatlabEnginePut_C",VecMatlabEnginePut_DA2d);
 70:     }
 71: #endif
 72:   }
 73:   return(0);
 74: }

 78: /*@
 79:   DMDAGetNumCells - Get the number of cells in the local piece of the DMDA. This includes ghost cells.

 81:   Input Parameter:
 82: . dm - The DM object

 84:   Output Parameters:
 85: + numCellsX - The number of local cells in the x-direction
 86: . numCellsY - The number of local cells in the y-direction
 87: . numCellsZ - The number of local cells in the z-direction
 88: - numCells - The number of local cells

 90:   Level: developer

 92: .seealso: DMDAGetCellPoint()
 93: @*/
 94: PetscErrorCode DMDAGetNumCells(DM dm, PetscInt *numCellsX, PetscInt *numCellsY, PetscInt *numCellsZ, PetscInt *numCells)
 95: {
 96:   DM_DA         *da  = (DM_DA*) dm->data;
 97:   const PetscInt dim = da->dim;
 98:   const PetscInt mx  = (da->Xe - da->Xs)/da->w, my = da->Ye - da->Ys, mz = da->Ze - da->Zs;
 99:   const PetscInt nC  = (mx)*(dim > 1 ? (my)*(dim > 2 ? (mz) : 1) : 1);

103:   if (numCellsX) {
105:     *numCellsX = mx;
106:   }
107:   if (numCellsY) {
109:     *numCellsY = my;
110:   }
111:   if (numCellsZ) {
113:     *numCellsZ = mz;
114:   }
115:   if (numCells) {
117:     *numCells = nC;
118:   }
119:   return(0);
120: }

124: /*@
125:   DMDAGetCellPoint - Get the DM point corresponding to the tuple (i, j, k) in the DMDA

127:   Input Parameters:
128: + dm - The DM object
129: - i,j,k - The global indices for the cell

131:   Output Parameters:
132: . point - The local DM point

134:   Level: developer

136: .seealso: DMDAGetNumCells()
137: @*/
138: PetscErrorCode DMDAGetCellPoint(DM dm, PetscInt i, PetscInt j, PetscInt k, PetscInt *point)
139: {
140:   DM_DA         *da  = (DM_DA*) dm->data;
141:   const PetscInt dim = da->dim;
142:   DMDALocalInfo  info;

148:   DMDAGetLocalInfo(dm, &info);
149:   if (dim > 0) {if ((i < info.gxs) || (i >= info.gxs+info.gxm)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "X index %d not in [%d, %d)", i, info.gxs, info.gxs+info.gxm);}
150:   if (dim > 1) {if ((i < info.gys) || (i >= info.gys+info.gym)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Y index %d not in [%d, %d)", i, info.gys, info.gys+info.gym);}
151:   if (dim > 2) {if ((i < info.gzs) || (i >= info.gzs+info.gzm)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Z index %d not in [%d, %d)", i, info.gzs, info.gzs+info.gzm);}
152:   *point = i + (dim > 1 ? (j + (dim > 2 ? k*info.gym : 0))*info.gxm : 0);
153:   return(0);
154: }

158: PetscErrorCode DMDAGetNumVertices(DM dm, PetscInt *numVerticesX, PetscInt *numVerticesY, PetscInt *numVerticesZ, PetscInt *numVertices)
159: {
160:   DM_DA          *da = (DM_DA*) dm->data;
161:   const PetscInt dim = da->dim;
162:   const PetscInt mx  = (da->Xe - da->Xs)/da->w, my = da->Ye - da->Ys, mz = da->Ze - da->Zs;
163:   const PetscInt nVx = mx+1;
164:   const PetscInt nVy = dim > 1 ? (my+1) : 1;
165:   const PetscInt nVz = dim > 2 ? (mz+1) : 1;
166:   const PetscInt nV  = nVx*nVy*nVz;

169:   if (numVerticesX) {
171:     *numVerticesX = nVx;
172:   }
173:   if (numVerticesY) {
175:     *numVerticesY = nVy;
176:   }
177:   if (numVerticesZ) {
179:     *numVerticesZ = nVz;
180:   }
181:   if (numVertices) {
183:     *numVertices = nV;
184:   }
185:   return(0);
186: }

190: PetscErrorCode DMDAGetNumFaces(DM dm, PetscInt *numXFacesX, PetscInt *numXFaces, PetscInt *numYFacesY, PetscInt *numYFaces, PetscInt *numZFacesZ, PetscInt *numZFaces)
191: {
192:   DM_DA          *da = (DM_DA*) dm->data;
193:   const PetscInt dim = da->dim;
194:   const PetscInt mx  = (da->Xe - da->Xs)/da->w, my = da->Ye - da->Ys, mz = da->Ze - da->Zs;
195:   const PetscInt nxF = (dim > 1 ? (my)*(dim > 2 ? (mz) : 1) : 1);
196:   const PetscInt nXF = (mx+1)*nxF;
197:   const PetscInt nyF = mx*(dim > 2 ? mz : 1);
198:   const PetscInt nYF = dim > 1 ? (my+1)*nyF : 0;
199:   const PetscInt nzF = mx*(dim > 1 ? my : 0);
200:   const PetscInt nZF = dim > 2 ? (mz+1)*nzF : 0;

203:   if (numXFacesX) {
205:     *numXFacesX = nxF;
206:   }
207:   if (numXFaces) {
209:     *numXFaces = nXF;
210:   }
211:   if (numYFacesY) {
213:     *numYFacesY = nyF;
214:   }
215:   if (numYFaces) {
217:     *numYFaces = nYF;
218:   }
219:   if (numZFacesZ) {
221:     *numZFacesZ = nzF;
222:   }
223:   if (numZFaces) {
225:     *numZFaces = nZF;
226:   }
227:   return(0);
228: }

232: PetscErrorCode DMDAGetHeightStratum(DM dm, PetscInt height, PetscInt *pStart, PetscInt *pEnd)
233: {
234:   DM_DA          *da = (DM_DA*) dm->data;
235:   const PetscInt dim = da->dim;
236:   PetscInt       nC, nV, nXF, nYF, nZF;

242:   DMDAGetNumCells(dm, NULL, NULL, NULL, &nC);
243:   DMDAGetNumVertices(dm, NULL, NULL, NULL, &nV);
244:   DMDAGetNumFaces(dm, NULL, &nXF, NULL, &nYF, NULL, &nZF);
245:   if (height == 0) {
246:     /* Cells */
247:     if (pStart) *pStart = 0;
248:     if (pEnd)   *pEnd   = nC;
249:   } else if (height == 1) {
250:     /* Faces */
251:     if (pStart) *pStart = nC+nV;
252:     if (pEnd)   *pEnd   = nC+nV+nXF+nYF+nZF;
253:   } else if (height == dim) {
254:     /* Vertices */
255:     if (pStart) *pStart = nC;
256:     if (pEnd)   *pEnd   = nC+nV;
257:   } else if (height < 0) {
258:     /* All points */
259:     if (pStart) *pStart = 0;
260:     if (pEnd)   *pEnd   = nC+nV+nXF+nYF+nZF;
261:   } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "No points of height %d in the DA", height);
262:   return(0);
263: }

267: PetscErrorCode DMDAGetDepthStratum(DM dm, PetscInt depth, PetscInt *pStart, PetscInt *pEnd)
268: {
269:   DM_DA         *da  = (DM_DA*) dm->data;
270:   const PetscInt dim = da->dim;
271:   PetscInt       nC, nV, nXF, nYF, nZF;

277:   DMDAGetNumCells(dm, NULL, NULL, NULL, &nC);
278:   DMDAGetNumVertices(dm, NULL, NULL, NULL, &nV);
279:   DMDAGetNumFaces(dm, NULL, &nXF, NULL, &nYF, NULL, &nZF);
280:   if (depth == dim) {
281:     /* Cells */
282:     if (pStart) *pStart = 0;
283:     if (pEnd)   *pEnd   = nC;
284:   } else if (depth == dim-1) {
285:     /* Faces */
286:     if (pStart) *pStart = nC+nV;
287:     if (pEnd)   *pEnd   = nC+nV+nXF+nYF+nZF;
288:   } else if (depth == 0) {
289:     /* Vertices */
290:     if (pStart) *pStart = nC;
291:     if (pEnd)   *pEnd   = nC+nV;
292:   } else if (depth < 0) {
293:     /* All points */
294:     if (pStart) *pStart = 0;
295:     if (pEnd)   *pEnd   = nC+nV+nXF+nYF+nZF;
296:   } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "No points of depth %d in the DA", depth);
297:   return(0);
298: }

302: PetscErrorCode DMDAGetConeSize(DM dm, PetscInt p, PetscInt *coneSize)
303: {
304:   DM_DA         *da  = (DM_DA*) dm->data;
305:   const PetscInt dim = da->dim;
306:   PetscInt       nC, nV, nXF, nYF, nZF;

310:   *coneSize = 0;
311:   DMDAGetNumCells(dm, NULL, NULL, NULL, &nC);
312:   DMDAGetNumVertices(dm, NULL, NULL, NULL, &nV);
313:   DMDAGetNumFaces(dm, NULL, &nXF, NULL, &nYF, NULL, &nZF);
314:   switch (dim) {
315:   case 2:
316:     if (p >= 0) {
317:       if (p < nC) {
318:         *coneSize = 4;
319:       } else if (p < nC+nV) {
320:         *coneSize = 0;
321:       } else if (p < nC+nV+nXF+nYF+nZF) {
322:         *coneSize = 2;
323:       } else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %d should be in [0, %d)", p, nC+nV+nXF+nYF+nZF);
324:     } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Negative point %d is invalid", p);
325:     break;
326:   case 3:
327:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Too lazy to do 3D");
328:     break;
329:   }
330:   return(0);
331: }

335: PetscErrorCode DMDAGetCone(DM dm, PetscInt p, PetscInt *cone[])
336: {
337:   DM_DA         *da  = (DM_DA*) dm->data;
338:   const PetscInt dim = da->dim;
339:   PetscInt       nCx, nCy, nCz, nC, nVx, nVy, nVz, nV, nxF, nyF, nzF, nXF, nYF, nZF;

343:   if (!cone) {DMGetWorkArray(dm, 6, PETSC_INT, cone);}
344:   DMDAGetNumCells(dm, &nCx, &nCy, &nCz, &nC);
345:   DMDAGetNumVertices(dm, &nVx, &nVy, &nVz, &nV);
346:   DMDAGetNumFaces(dm, &nxF, &nXF, &nyF, &nYF, &nzF, &nZF);
347:   switch (dim) {
348:   case 2:
349:     if (p >= 0) {
350:       if (p < nC) {
351:         const PetscInt cy = p / nCx;
352:         const PetscInt cx = p % nCx;

354:         (*cone)[0] = cy*nxF + cx + nC+nV;
355:         (*cone)[1] = cx*nyF + cy + nyF + nC+nV+nXF;
356:         (*cone)[2] = cy*nxF + cx + nxF + nC+nV;
357:         (*cone)[3] = cx*nyF + cy + nC+nV+nXF;
358:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Too lazy to do cell cones");
359:       } else if (p < nC+nV) {
360:       } else if (p < nC+nV+nXF) {
361:         const PetscInt fy = (p - nC+nV) / nxF;
362:         const PetscInt fx = (p - nC+nV) % nxF;

364:         (*cone)[0] = fy*nVx + fx + nC;
365:         (*cone)[1] = fy*nVx + fx + 1 + nC;
366:       } else if (p < nC+nV+nXF+nYF) {
367:         const PetscInt fx = (p - nC+nV+nXF) / nyF;
368:         const PetscInt fy = (p - nC+nV+nXF) % nyF;

370:         (*cone)[0] = fy*nVx + fx + nC;
371:         (*cone)[1] = fy*nVx + fx + nVx + nC;
372:       } else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %d should be in [0, %d)", p, nC+nV+nXF+nYF+nZF);
373:     } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Negative point %d is invalid", p);
374:     break;
375:   case 3:
376:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Too lazy to do 3D");
377:     break;
378:   }
379:   return(0);
380: }

384: PetscErrorCode DMDARestoreCone(DM dm, PetscInt p, PetscInt *cone[])
385: {

389:   DMGetWorkArray(dm, 6, PETSC_INT, cone);
390:   return(0);
391: }

395: /*@C
396:   DMDACreateSection - Create a PetscSection inside the DMDA that describes data layout. This allows multiple fields with
397:   different numbers of dofs on vertices, cells, and faces in each direction.

399:   Input Parameters:
400: + dm- The DMDA
401: . numFields - The number of fields
402: . numComp - The number of components in each field
403: . numDof - The number of dofs per dimension for each field
404: . numFaceDof - The number of dofs per face for each field and direction, or NULL

406:   Level: developer

408:   Note:
409:   The default DMDA numbering is as follows:

411:     - Cells:    [0,             nC)
412:     - Vertices: [nC,            nC+nV)
413:     - X-Faces:  [nC+nV,         nC+nV+nXF)         normal is +- x-dir
414:     - Y-Faces:  [nC+nV+nXF,     nC+nV+nXF+nYF)     normal is +- y-dir
415:     - Z-Faces:  [nC+nV+nXF+nYF, nC+nV+nXF+nYF+nZF) normal is +- z-dir

417:   We interpret the default DMDA partition as a cell partition, and the data assignment as a cell assignment.
418: @*/
419: PetscErrorCode DMDACreateSection(DM dm, PetscInt numComp[], PetscInt numDof[], PetscInt numFaceDof[], PetscSection *s)
420: {
421:   DM_DA            *da  = (DM_DA*) dm->data;
422:   PetscSection      section;
423:   const PetscInt    dim = da->dim;
424:   PetscInt          numFields, numVertexTotDof = 0, numCellTotDof = 0, numFaceTotDof[3] = {0, 0, 0};
425:   PetscBT           isLeaf;
426:   PetscSF           sf;
427:   PetscMPIInt       rank;
428:   const PetscMPIInt *neighbors;
429:   PetscInt          *localPoints;
430:   PetscSFNode       *remotePoints;
431:   PetscInt          nleaves = 0,  nleavesCheck = 0, nL = 0;
432:   PetscInt          nC, nVx, nVy, nVz, nV, nxF, nXF, nyF, nYF, nzF, nZF;
433:   PetscInt          pStart, pEnd, cStart, cEnd, vStart, vEnd, fStart, fEnd, xfStart, xfEnd, yfStart, yfEnd, zfStart, zfEnd;
434:   PetscInt          f, v, c, xf, yf, zf, xn, yn, zn;
435:   PetscErrorCode    ierr;

440:   MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);
441:   DMDAGetNumCells(dm, NULL, NULL, NULL, &nC);
442:   DMDAGetNumVertices(dm, &nVx, &nVy, &nVz, &nV);
443:   DMDAGetNumFaces(dm, &nxF, &nXF, &nyF, &nYF, &nzF, &nZF);
444:   DMDAGetHeightStratum(dm, -1,  &pStart, &pEnd);
445:   DMDAGetHeightStratum(dm, 0,   &cStart, &cEnd);
446:   DMDAGetHeightStratum(dm, 1,   &fStart, &fEnd);
447:   DMDAGetHeightStratum(dm, dim, &vStart, &vEnd);
448:   xfStart = vEnd;  xfEnd = xfStart+nXF;
449:   yfStart = xfEnd; yfEnd = yfStart+nYF;
450:   zfStart = yfEnd; zfEnd = zfStart+nZF;
451:   if (zfEnd != fEnd) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid face end %d, should be %d", zfEnd, fEnd);
452:   /* Create local section */
453:   DMDAGetInfo(dm, 0,0,0,0,0,0,0, &numFields, 0,0,0,0,0);
454:   for (f = 0; f < numFields; ++f) {
455:     numVertexTotDof  += numDof[f*(dim+1)+0];
456:     numCellTotDof    += numDof[f*(dim+1)+dim];
457:     numFaceTotDof[0] += dim > 0 ? (numFaceDof ? numFaceDof[f*dim+0] : numDof[f*(dim+1)+dim-1]) : 0;
458:     numFaceTotDof[1] += dim > 1 ? (numFaceDof ? numFaceDof[f*dim+1] : numDof[f*(dim+1)+dim-1]) : 0;
459:     numFaceTotDof[2] += dim > 2 ? (numFaceDof ? numFaceDof[f*dim+2] : numDof[f*(dim+1)+dim-1]) : 0;
460:   }
461:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);
462:   if (numFields > 0) {
463:     PetscSectionSetNumFields(section, numFields);
464:     for (f = 0; f < numFields; ++f) {
465:       const char *name;

467:       DMDAGetFieldName(dm, f, &name);
468:       PetscSectionSetFieldName(section, f, name ? name : "Field");
469:       if (numComp) {
470:         PetscSectionSetFieldComponents(section, f, numComp[f]);
471:       }
472:     }
473:   }
474:   PetscSectionSetChart(section, pStart, pEnd);
475:   for (v = vStart; v < vEnd; ++v) {
476:     for (f = 0; f < numFields; ++f) {
477:       PetscSectionSetFieldDof(section, v, f, numDof[f*(dim+1)+0]);
478:     }
479:     PetscSectionSetDof(section, v, numVertexTotDof);
480:   }
481:   for (xf = xfStart; xf < xfEnd; ++xf) {
482:     for (f = 0; f < numFields; ++f) {
483:       PetscSectionSetFieldDof(section, xf, f, numFaceDof ? numFaceDof[f*dim+0] : numDof[f*(dim+1)+dim-1]);
484:     }
485:     PetscSectionSetDof(section, xf, numFaceTotDof[0]);
486:   }
487:   for (yf = yfStart; yf < yfEnd; ++yf) {
488:     for (f = 0; f < numFields; ++f) {
489:       PetscSectionSetFieldDof(section, yf, f, numFaceDof ? numFaceDof[f*dim+1] : numDof[f*(dim+1)+dim-1]);
490:     }
491:     PetscSectionSetDof(section, yf, numFaceTotDof[1]);
492:   }
493:   for (zf = zfStart; zf < zfEnd; ++zf) {
494:     for (f = 0; f < numFields; ++f) {
495:       PetscSectionSetFieldDof(section, zf, f, numFaceDof ? numFaceDof[f*dim+2] : numDof[f*(dim+1)+dim-1]);
496:     }
497:     PetscSectionSetDof(section, zf, numFaceTotDof[2]);
498:   }
499:   for (c = cStart; c < cEnd; ++c) {
500:     for (f = 0; f < numFields; ++f) {
501:       PetscSectionSetFieldDof(section, c, f, numDof[f*(dim+1)+dim]);
502:     }
503:     PetscSectionSetDof(section, c, numCellTotDof);
504:   }
505:   PetscSectionSetUp(section);
506:   /* Create mesh point SF */
507:   PetscBTCreate(pEnd-pStart, &isLeaf);
508:   DMDAGetNeighbors(dm, &neighbors);
509:   for (zn = 0; zn < (dim > 2 ? 3 : 1); ++zn) {
510:     for (yn = 0; yn < (dim > 1 ? 3 : 1); ++yn) {
511:       for (xn = 0; xn < 3; ++xn) {
512:         const PetscInt xp       = xn-1, yp = dim > 1 ? yn-1 : 0, zp = dim > 2 ? zn-1 : 0;
513:         const PetscInt neighbor = neighbors[(zn*3+yn)*3+xn];
514:         PetscInt       xv, yv, zv;

516:         if (neighbor >= 0 && neighbor < rank) {
517:           if (xp < 0) { /* left */
518:             if (yp < 0) { /* bottom */
519:               if (zp < 0) { /* back */
520:                 const PetscInt localVertex  = (      0*nVy +     0)*nVx +     0 + nC; /* left bottom back vertex */
521:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
522:               } else if (zp > 0) { /* front */
523:                 const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx +     0 + nC; /* left bottom front vertex */
524:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
525:               } else {
526:                 for (zv = 0; zv < nVz; ++zv) {
527:                   const PetscInt localVertex  = (zv*nVy +     0)*nVx +     0 + nC; /* left bottom vertices */
528:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
529:                 }
530:               }
531:             } else if (yp > 0) { /* top */
532:               if (zp < 0) { /* back */
533:                 const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx +     0 + nC; /* left top back vertex */
534:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
535:               } else if (zp > 0) { /* front */
536:                 const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx +     0 + nC; /* left top front vertex */
537:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
538:               } else {
539:                 for (zv = 0; zv < nVz; ++zv) {
540:                   const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx +     0 + nC; /* left top vertices */
541:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
542:                 }
543:               }
544:             } else {
545:               if (zp < 0) { /* back */
546:                 for (yv = 0; yv < nVy; ++yv) {
547:                   const PetscInt localVertex  = (      0*nVy + yv)*nVx +     0 + nC; /* left back vertices */
548:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
549:                 }
550:               } else if (zp > 0) { /* front */
551:                 for (yv = 0; yv < nVy; ++yv) {
552:                   const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx +     0 + nC; /* left front vertices */
553:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
554:                 }
555:               } else {
556:                 for (zv = 0; zv < nVz; ++zv) {
557:                   for (yv = 0; yv < nVy; ++yv) {
558:                     const PetscInt localVertex  = (zv*nVy + yv)*nVx +     0 + nC; /* left vertices */
559:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
560:                   }
561:                 }
562: #if 0
563:                 for (xf = 0; xf < nxF; ++xf) {
564:                   /* THIS IS WRONG */
565:                   const PetscInt localFace  = 0 + nC+nV; /* left faces */
566:                   if (!PetscBTLookupSet(isLeaf, localFace)) ++nleaves;
567:                 }
568: #endif
569:               }
570:             }
571:           } else if (xp > 0) { /* right */
572:             if (yp < 0) { /* bottom */
573:               if (zp < 0) { /* back */
574:                 const PetscInt localVertex  = (      0*nVy +     0)*nVx + nVx-1 + nC; /* right bottom back vertex */
575:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
576:               } else if (zp > 0) { /* front */
577:                 const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx + nVx-1 + nC; /* right bottom front vertex */
578:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
579:               } else {
580:                 for (zv = 0; zv < nVz; ++zv) {
581:                   const PetscInt localVertex  = (zv*nVy +     0)*nVx + nVx-1 + nC; /* right bottom vertices */
582:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
583:                 }
584:               }
585:             } else if (yp > 0) { /* top */
586:               if (zp < 0) { /* back */
587:                 const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top back vertex */
588:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
589:               } else if (zp > 0) { /* front */
590:                 const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top front vertex */
591:                 if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
592:               } else {
593:                 for (zv = 0; zv < nVz; ++zv) {
594:                   const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top vertices */
595:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
596:                 }
597:               }
598:             } else {
599:               if (zp < 0) { /* back */
600:                 for (yv = 0; yv < nVy; ++yv) {
601:                   const PetscInt localVertex  = (      0*nVy + yv)*nVx + nVx-1 + nC; /* right back vertices */
602:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
603:                 }
604:               } else if (zp > 0) { /* front */
605:                 for (yv = 0; yv < nVy; ++yv) {
606:                   const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx + nVx-1 + nC; /* right front vertices */
607:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
608:                 }
609:               } else {
610:                 for (zv = 0; zv < nVz; ++zv) {
611:                   for (yv = 0; yv < nVy; ++yv) {
612:                     const PetscInt localVertex  = (zv*nVy + yv)*nVx + nVx-1 + nC; /* right vertices */
613:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
614:                   }
615:                 }
616: #if 0
617:                 for (xf = 0; xf < nxF; ++xf) {
618:                   /* THIS IS WRONG */
619:                   const PetscInt localFace  = 0 + nC+nV; /* right faces */
620:                   if (!PetscBTLookupSet(isLeaf, localFace)) ++nleaves;
621:                 }
622: #endif
623:               }
624:             }
625:           } else {
626:             if (yp < 0) { /* bottom */
627:               if (zp < 0) { /* back */
628:                 for (xv = 0; xv < nVx; ++xv) {
629:                   const PetscInt localVertex  = (      0*nVy +     0)*nVx + xv + nC; /* bottom back vertices */
630:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
631:                 }
632:               } else if (zp > 0) { /* front */
633:                 for (xv = 0; xv < nVx; ++xv) {
634:                   const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx + xv + nC; /* bottom front vertices */
635:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
636:                 }
637:               } else {
638:                 for (zv = 0; zv < nVz; ++zv) {
639:                   for (xv = 0; xv < nVx; ++xv) {
640:                     const PetscInt localVertex  = (zv*nVy +     0)*nVx + xv + nC; /* bottom vertices */
641:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
642:                   }
643:                 }
644: #if 0
645:                 for (yf = 0; yf < nyF; ++yf) {
646:                   /* THIS IS WRONG */
647:                   const PetscInt localFace  = 0 + nC+nV; /* bottom faces */
648:                   if (!PetscBTLookupSet(isLeaf, localVFace)) ++nleaves;
649:                 }
650: #endif
651:               }
652:             } else if (yp > 0) { /* top */
653:               if (zp < 0) { /* back */
654:                 for (xv = 0; xv < nVx; ++xv) {
655:                   const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx + xv + nC; /* top back vertices */
656:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
657:                 }
658:               } else if (zp > 0) { /* front */
659:                 for (xv = 0; xv < nVx; ++xv) {
660:                   const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx + xv + nC; /* top front vertices */
661:                   if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
662:                 }
663:               } else {
664:                 for (zv = 0; zv < nVz; ++zv) {
665:                   for (xv = 0; xv < nVx; ++xv) {
666:                     const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx + xv + nC; /* top vertices */
667:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
668:                   }
669:                 }
670: #if 0
671:                 for (yf = 0; yf < nyF; ++yf) {
672:                   /* THIS IS WRONG */
673:                   const PetscInt localFace  = 0 + nC+nV; /* top faces */
674:                   if (!PetscBTLookupSet(isLeaf, localVFace)) ++nleaves;
675:                 }
676: #endif
677:               }
678:             } else {
679:               if (zp < 0) { /* back */
680:                 for (yv = 0; yv < nVy; ++yv) {
681:                   for (xv = 0; xv < nVx; ++xv) {
682:                     const PetscInt localVertex  = (      0*nVy + yv)*nVx + xv + nC; /* back vertices */
683:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
684:                   }
685:                 }
686: #if 0
687:                 for (zf = 0; zf < nzF; ++zf) {
688:                   /* THIS IS WRONG */
689:                   const PetscInt localFace  = 0 + nC+nV; /* back faces */
690:                   if (!PetscBTLookupSet(isLeaf, localFace)) ++nleaves;
691:                 }
692: #endif
693:               } else if (zp > 0) { /* front */
694:                 for (yv = 0; yv < nVy; ++yv) {
695:                   for (xv = 0; xv < nVx; ++xv) {
696:                     const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx + xv + nC; /* front vertices */
697:                     if (!PetscBTLookupSet(isLeaf, localVertex)) ++nleaves;
698:                   }
699:                 }
700: #if 0
701:                 for (zf = 0; zf < nzF; ++zf) {
702:                   /* THIS IS WRONG */
703:                   const PetscInt localFace  = 0 + nC+nV; /* front faces */
704:                   if (!PetscBTLookupSet(isLeaf, localFace)) ++nleaves;
705:                 }
706: #endif
707:               } else {
708:                 /* Nothing is shared from the interior */
709:               }
710:             }
711:           }
712:         }
713:       }
714:     }
715:   }
716:   PetscBTMemzero(pEnd-pStart, isLeaf);
717:   PetscMalloc2(nleaves,&localPoints,nleaves,&remotePoints);
718:   for (zn = 0; zn < (dim > 2 ? 3 : 1); ++zn) {
719:     for (yn = 0; yn < (dim > 1 ? 3 : 1); ++yn) {
720:       for (xn = 0; xn < 3; ++xn) {
721:         const PetscInt xp       = xn-1, yp = dim > 1 ? yn-1 : 0, zp = dim > 2 ? zn-1 : 0;
722:         const PetscInt neighbor = neighbors[(zn*3+yn)*3+xn];
723:         PetscInt       xv, yv, zv;

725:         if (neighbor >= 0 && neighbor < rank) {
726:           if (xp < 0) { /* left */
727:             if (yp < 0) { /* bottom */
728:               if (zp < 0) { /* back */
729:                 const PetscInt localVertex  = (      0*nVy +     0)*nVx +     0 + nC; /* left bottom back vertex */
730:                 const PetscInt remoteVertex = ((nVz-1)*nVy + nVy-1)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

732:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
733:                   localPoints[nL]        = localVertex;
734:                   remotePoints[nL].rank  = neighbor;
735:                   remotePoints[nL].index = remoteVertex;
736:                   ++nL;
737:                 }
738:               } else if (zp > 0) { /* front */
739:                 const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx +     0 + nC; /* left bottom front vertex */
740:                 const PetscInt remoteVertex = (      0*nVy + nVy-1)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

742:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
743:                   localPoints[nL]        = localVertex;
744:                   remotePoints[nL].rank  = neighbor;
745:                   remotePoints[nL].index = remoteVertex;
746:                   ++nL;
747:                 }
748:               } else {
749:                 for (zv = 0; zv < nVz; ++zv) {
750:                   const PetscInt localVertex  = (zv*nVy +     0)*nVx +     0 + nC; /* left bottom vertices */
751:                   const PetscInt remoteVertex = (zv*nVy + nVy-1)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

753:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
754:                     localPoints[nL]        = localVertex;
755:                     remotePoints[nL].rank  = neighbor;
756:                     remotePoints[nL].index = remoteVertex;
757:                     ++nL;
758:                   }
759:                 }
760:               }
761:             } else if (yp > 0) { /* top */
762:               if (zp < 0) { /* back */
763:                 const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx +     0 + nC; /* left top back vertex */
764:                 const PetscInt remoteVertex = ((nVz-1)*nVy +     0)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

766:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
767:                   localPoints[nL]        = localVertex;
768:                   remotePoints[nL].rank  = neighbor;
769:                   remotePoints[nL].index = remoteVertex;
770:                   ++nL;
771:                 }
772:               } else if (zp > 0) { /* front */
773:                 const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx +     0 + nC; /* left top front vertex */
774:                 const PetscInt remoteVertex = (      0*nVy +     0)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

776:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
777:                   localPoints[nL]        = localVertex;
778:                   remotePoints[nL].rank  = neighbor;
779:                   remotePoints[nL].index = remoteVertex;
780:                   ++nL;
781:                 }
782:               } else {
783:                 for (zv = 0; zv < nVz; ++zv) {
784:                   const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx +     0 + nC; /* left top vertices */
785:                   const PetscInt remoteVertex = (zv*nVy +     0)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

787:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
788:                     localPoints[nL]        = localVertex;
789:                     remotePoints[nL].rank  = neighbor;
790:                     remotePoints[nL].index = remoteVertex;
791:                     ++nL;
792:                   }
793:                 }
794:               }
795:             } else {
796:               if (zp < 0) { /* back */
797:                 for (yv = 0; yv < nVy; ++yv) {
798:                   const PetscInt localVertex  = (      0*nVy + yv)*nVx +     0 + nC; /* left back vertices */
799:                   const PetscInt remoteVertex = ((nVz-1)*nVy + yv)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

801:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
802:                     localPoints[nL]        = localVertex;
803:                     remotePoints[nL].rank  = neighbor;
804:                     remotePoints[nL].index = remoteVertex;
805:                     ++nL;
806:                   }
807:                 }
808:               } else if (zp > 0) { /* front */
809:                 for (yv = 0; yv < nVy; ++yv) {
810:                   const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx +     0 + nC; /* left front vertices */
811:                   const PetscInt remoteVertex = (      0*nVy + yv)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

813:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
814:                     localPoints[nL]        = localVertex;
815:                     remotePoints[nL].rank  = neighbor;
816:                     remotePoints[nL].index = remoteVertex;
817:                     ++nL;
818:                   }
819:                 }
820:               } else {
821:                 for (zv = 0; zv < nVz; ++zv) {
822:                   for (yv = 0; yv < nVy; ++yv) {
823:                     const PetscInt localVertex  = (zv*nVy + yv)*nVx +     0 + nC; /* left vertices */
824:                     const PetscInt remoteVertex = (zv*nVy + yv)*nVx + nVx-1 + nC; /* TODO: Correct this for neighbor sizes */

826:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
827:                       localPoints[nL]        = localVertex;
828:                       remotePoints[nL].rank  = neighbor;
829:                       remotePoints[nL].index = remoteVertex;
830:                       ++nL;
831:                     }
832:                   }
833:                 }
834: #if 0
835:                 for (xf = 0; xf < nxF; ++xf) {
836:                   /* THIS IS WRONG */
837:                   const PetscInt localFace  = 0 + nC+nV; /* left faces */
838:                   const PetscInt remoteFace = 0 + nC+nV;

840:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
841:                     localPoints[nL]        = localFace;
842:                     remotePoints[nL].rank  = neighbor;
843:                     remotePoints[nL].index = remoteFace;
844:                   }
845:                 }
846: #endif
847:               }
848:             }
849:           } else if (xp > 0) { /* right */
850:             if (yp < 0) { /* bottom */
851:               if (zp < 0) { /* back */
852:                 const PetscInt localVertex  = (      0*nVy +     0)*nVx + nVx-1 + nC; /* right bottom back vertex */
853:                 const PetscInt remoteVertex = ((nVz-1)*nVy + nVy-1)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

855:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
856:                   localPoints[nL]        = localVertex;
857:                   remotePoints[nL].rank  = neighbor;
858:                   remotePoints[nL].index = remoteVertex;
859:                   ++nL;
860:                 }
861:               } else if (zp > 0) { /* front */
862:                 const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx + nVx-1 + nC; /* right bottom front vertex */
863:                 const PetscInt remoteVertex = (      0*nVy + nVy-1)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

865:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
866:                   localPoints[nL]        = localVertex;
867:                   remotePoints[nL].rank  = neighbor;
868:                   remotePoints[nL].index = remoteVertex;
869:                   ++nL;
870:                 }
871:               } else {
872:                 nleavesCheck += nVz;
873:                 for (zv = 0; zv < nVz; ++zv) {
874:                   const PetscInt localVertex  = (zv*nVy +     0)*nVx + nVx-1 + nC; /* right bottom vertices */
875:                   const PetscInt remoteVertex = (zv*nVy + nVy-1)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

877:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
878:                     localPoints[nL]        = localVertex;
879:                     remotePoints[nL].rank  = neighbor;
880:                     remotePoints[nL].index = remoteVertex;
881:                     ++nL;
882:                   }
883:                 }
884:               }
885:             } else if (yp > 0) { /* top */
886:               if (zp < 0) { /* back */
887:                 const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top back vertex */
888:                 const PetscInt remoteVertex = ((nVz-1)*nVy +     0)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

890:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
891:                   localPoints[nL]        = localVertex;
892:                   remotePoints[nL].rank  = neighbor;
893:                   remotePoints[nL].index = remoteVertex;
894:                   ++nL;
895:                 }
896:               } else if (zp > 0) { /* front */
897:                 const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top front vertex */
898:                 const PetscInt remoteVertex = (      0*nVy +     0)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

900:                 if (!PetscBTLookupSet(isLeaf, localVertex)) {
901:                   localPoints[nL]        = localVertex;
902:                   remotePoints[nL].rank  = neighbor;
903:                   remotePoints[nL].index = remoteVertex;
904:                   ++nL;
905:                 }
906:               } else {
907:                 for (zv = 0; zv < nVz; ++zv) {
908:                   const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx + nVx-1 + nC; /* right top vertices */
909:                   const PetscInt remoteVertex = (zv*nVy +     0)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

911:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
912:                     localPoints[nL]        = localVertex;
913:                     remotePoints[nL].rank  = neighbor;
914:                     remotePoints[nL].index = remoteVertex;
915:                     ++nL;
916:                   }
917:                 }
918:               }
919:             } else {
920:               if (zp < 0) { /* back */
921:                 for (yv = 0; yv < nVy; ++yv) {
922:                   const PetscInt localVertex  = (      0*nVy + yv)*nVx + nVx-1 + nC; /* right back vertices */
923:                   const PetscInt remoteVertex = ((nVz-1)*nVy + yv)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

925:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
926:                     localPoints[nL]        = localVertex;
927:                     remotePoints[nL].rank  = neighbor;
928:                     remotePoints[nL].index = remoteVertex;
929:                     ++nL;
930:                   }
931:                 }
932:               } else if (zp > 0) { /* front */
933:                 for (yv = 0; yv < nVy; ++yv) {
934:                   const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx + nVx-1 + nC; /* right front vertices */
935:                   const PetscInt remoteVertex = (      0*nVy + yv)*nVx +     0 + nC; /* TODO: Correct this for neighbor sizes */

937:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
938:                     localPoints[nL]        = localVertex;
939:                     remotePoints[nL].rank  = neighbor;
940:                     remotePoints[nL].index = remoteVertex;
941:                     ++nL;
942:                   }
943:                 }
944:               } else {
945:                 for (zv = 0; zv < nVz; ++zv) {
946:                   for (yv = 0; yv < nVy; ++yv) {
947:                     const PetscInt localVertex  = (zv*nVy + yv)*nVx + nVx-1 + nC; /* right vertices */
948:                     const PetscInt remoteVertex = (zv*nVy + yv)*nVx + 0     + nC; /* TODO: Correct this for neighbor sizes */

950:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
951:                       localPoints[nL]        = localVertex;
952:                       remotePoints[nL].rank  = neighbor;
953:                       remotePoints[nL].index = remoteVertex;
954:                       ++nL;
955:                     }
956:                   }
957:                 }
958: #if 0
959:                 for (xf = 0; xf < nxF; ++xf) {
960:                   /* THIS IS WRONG */
961:                   const PetscInt localFace  = 0 + nC+nV; /* right faces */
962:                   const PetscInt remoteFace = 0 + nC+nV;

964:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
965:                     localPoints[nL]        = localFace;
966:                     remotePoints[nL].rank  = neighbor;
967:                     remotePoints[nL].index = remoteFace;
968:                     ++nL;
969:                   }
970:                 }
971: #endif
972:               }
973:             }
974:           } else {
975:             if (yp < 0) { /* bottom */
976:               if (zp < 0) { /* back */
977:                 for (xv = 0; xv < nVx; ++xv) {
978:                   const PetscInt localVertex  = (      0*nVy +     0)*nVx + xv + nC; /* bottom back vertices */
979:                   const PetscInt remoteVertex = ((nVz-1)*nVy + nVy-1)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

981:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
982:                     localPoints[nL]        = localVertex;
983:                     remotePoints[nL].rank  = neighbor;
984:                     remotePoints[nL].index = remoteVertex;
985:                     ++nL;
986:                   }
987:                 }
988:               } else if (zp > 0) { /* front */
989:                 for (xv = 0; xv < nVx; ++xv) {
990:                   const PetscInt localVertex  = ((nVz-1)*nVy +     0)*nVx + xv + nC; /* bottom front vertices */
991:                   const PetscInt remoteVertex = (      0*nVy + nVy-1)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

993:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
994:                     localPoints[nL]        = localVertex;
995:                     remotePoints[nL].rank  = neighbor;
996:                     remotePoints[nL].index = remoteVertex;
997:                     ++nL;
998:                   }
999:                 }
1000:               } else {
1001:                 for (zv = 0; zv < nVz; ++zv) {
1002:                   for (xv = 0; xv < nVx; ++xv) {
1003:                     const PetscInt localVertex  = (zv*nVy +     0)*nVx + xv + nC; /* bottom vertices */
1004:                     const PetscInt remoteVertex = (zv*nVy + nVy-1)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1006:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
1007:                       localPoints[nL]        = localVertex;
1008:                       remotePoints[nL].rank  = neighbor;
1009:                       remotePoints[nL].index = remoteVertex;
1010:                       ++nL;
1011:                     }
1012:                   }
1013:                 }
1014: #if 0
1015:                 for (yf = 0; yf < nyF; ++yf) {
1016:                   /* THIS IS WRONG */
1017:                   const PetscInt localFace  = 0 + nC+nV; /* bottom faces */
1018:                   const PetscInt remoteFace = 0 + nC+nV;

1020:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
1021:                     localPoints[nL]        = localFace;
1022:                     remotePoints[nL].rank  = neighbor;
1023:                     remotePoints[nL].index = remoteFace;
1024:                     ++nL;
1025:                   }
1026:                 }
1027: #endif
1028:               }
1029:             } else if (yp > 0) { /* top */
1030:               if (zp < 0) { /* back */
1031:                 for (xv = 0; xv < nVx; ++xv) {
1032:                   const PetscInt localVertex  = (      0*nVy + nVy-1)*nVx + xv + nC; /* top back vertices */
1033:                   const PetscInt remoteVertex = ((nVz-1)*nVy +     0)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1035:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
1036:                     localPoints[nL]        = localVertex;
1037:                     remotePoints[nL].rank  = neighbor;
1038:                     remotePoints[nL].index = remoteVertex;
1039:                     ++nL;
1040:                   }
1041:                 }
1042:               } else if (zp > 0) { /* front */
1043:                 for (xv = 0; xv < nVx; ++xv) {
1044:                   const PetscInt localVertex  = ((nVz-1)*nVy + nVy-1)*nVx + xv + nC; /* top front vertices */
1045:                   const PetscInt remoteVertex = (      0*nVy +     0)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1047:                   if (!PetscBTLookupSet(isLeaf, localVertex)) {
1048:                     localPoints[nL]        = localVertex;
1049:                     remotePoints[nL].rank  = neighbor;
1050:                     remotePoints[nL].index = remoteVertex;
1051:                     ++nL;
1052:                   }
1053:                 }
1054:               } else {
1055:                 for (zv = 0; zv < nVz; ++zv) {
1056:                   for (xv = 0; xv < nVx; ++xv) {
1057:                     const PetscInt localVertex  = (zv*nVy + nVy-1)*nVx + xv + nC; /* top vertices */
1058:                     const PetscInt remoteVertex = (zv*nVy +     0)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1060:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
1061:                       localPoints[nL]        = localVertex;
1062:                       remotePoints[nL].rank  = neighbor;
1063:                       remotePoints[nL].index = remoteVertex;
1064:                       ++nL;
1065:                     }
1066:                   }
1067:                 }
1068: #if 0
1069:                 for (yf = 0; yf < nyF; ++yf) {
1070:                   /* THIS IS WRONG */
1071:                   const PetscInt localFace  = 0 + nC+nV; /* top faces */
1072:                   const PetscInt remoteFace = 0 + nC+nV;

1074:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
1075:                     localPoints[nL]        = localFace;
1076:                     remotePoints[nL].rank  = neighbor;
1077:                     remotePoints[nL].index = remoteFace;
1078:                     ++nL;
1079:                   }
1080:                 }
1081: #endif
1082:               }
1083:             } else {
1084:               if (zp < 0) { /* back */
1085:                 for (yv = 0; yv < nVy; ++yv) {
1086:                   for (xv = 0; xv < nVx; ++xv) {
1087:                     const PetscInt localVertex  = (      0*nVy + yv)*nVx + xv + nC; /* back vertices */
1088:                     const PetscInt remoteVertex = ((nVz-1)*nVy + yv)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1090:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
1091:                       localPoints[nL]        = localVertex;
1092:                       remotePoints[nL].rank  = neighbor;
1093:                       remotePoints[nL].index = remoteVertex;
1094:                       ++nL;
1095:                     }
1096:                   }
1097:                 }
1098: #if 0
1099:                 for (zf = 0; zf < nzF; ++zf) {
1100:                   /* THIS IS WRONG */
1101:                   const PetscInt localFace  = 0 + nC+nV; /* back faces */
1102:                   const PetscInt remoteFace = 0 + nC+nV;

1104:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
1105:                     localPoints[nL]        = localFace;
1106:                     remotePoints[nL].rank  = neighbor;
1107:                     remotePoints[nL].index = remoteFace;
1108:                     ++nL;
1109:                   }
1110:                 }
1111: #endif
1112:               } else if (zp > 0) { /* front */
1113:                 for (yv = 0; yv < nVy; ++yv) {
1114:                   for (xv = 0; xv < nVx; ++xv) {
1115:                     const PetscInt localVertex  = ((nVz-1)*nVy + yv)*nVx + xv + nC; /* front vertices */
1116:                     const PetscInt remoteVertex = (      0*nVy + yv)*nVx + xv + nC; /* TODO: Correct this for neighbor sizes */

1118:                     if (!PetscBTLookupSet(isLeaf, localVertex)) {
1119:                       localPoints[nL]        = localVertex;
1120:                       remotePoints[nL].rank  = neighbor;
1121:                       remotePoints[nL].index = remoteVertex;
1122:                       ++nL;
1123:                     }
1124:                   }
1125:                 }
1126: #if 0
1127:                 for (zf = 0; zf < nzF; ++zf) {
1128:                   /* THIS IS WRONG */
1129:                   const PetscInt localFace  = 0 + nC+nV; /* front faces */
1130:                   const PetscInt remoteFace = 0 + nC+nV;

1132:                   if (!PetscBTLookupSet(isLeaf, localFace)) {
1133:                     localPoints[nL]        = localFace;
1134:                     remotePoints[nL].rank  = neighbor;
1135:                     remotePoints[nL].index = remoteFace;
1136:                     ++nL;
1137:                   }
1138:                 }
1139: #endif
1140:               } else {
1141:                 /* Nothing is shared from the interior */
1142:               }
1143:             }
1144:           }
1145:         }
1146:       }
1147:     }
1148:   }
1149:   PetscBTDestroy(&isLeaf);
1150:   /* Remove duplication in leaf determination */
1151:   if (nleaves != nL) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "The number of leaves %d did not match the number of remote leaves %d", nleaves, nleavesCheck);
1152:   PetscSFCreate(PetscObjectComm((PetscObject)dm), &sf);
1153:   PetscSFSetGraph(sf, pEnd, nleaves, localPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);
1154:   DMSetPointSF(dm, sf);
1155:   PetscSFDestroy(&sf);
1156:   *s = section;
1157:   return(0);
1158: }

1162: PetscErrorCode DMDASetVertexCoordinates(DM dm, PetscReal xl, PetscReal xu, PetscReal yl, PetscReal yu, PetscReal zl, PetscReal zu)
1163: {
1164:   DM_DA         *da = (DM_DA *) dm->data;
1165:   Vec            coordinates;
1166:   PetscSection   section;
1167:   PetscScalar   *coords;
1168:   PetscReal      h[3];
1169:   PetscInt       dim, size, M, N, P, nVx, nVy, nVz, nV, vStart, vEnd, v, i, j, k;

1174:   DMDAGetInfo(dm, &dim, &M, &N, &P, 0,0,0,0,0,0,0,0,0);
1175:   h[0] = (xu - xl)/M;
1176:   h[1] = (yu - yl)/N;
1177:   h[2] = (zu - zl)/P;
1178:   DMDAGetDepthStratum(dm, 0, &vStart, &vEnd);
1179:   DMDAGetNumVertices(dm, &nVx, &nVy, &nVz, &nV);
1180:   PetscSectionCreate(PetscObjectComm((PetscObject) dm), &section);
1181:   PetscSectionSetNumFields(section, 1);
1182:   PetscSectionSetFieldComponents(section, 0, dim);
1183:   PetscSectionSetChart(section, vStart, vEnd);
1184:   for (v = vStart; v < vEnd; ++v) {
1185:     PetscSectionSetDof(section, v, dim);
1186:   }
1187:   PetscSectionSetUp(section);
1188:   PetscSectionGetStorageSize(section, &size);
1189:   VecCreateSeq(PETSC_COMM_SELF, size, &coordinates);
1190:   VecGetArray(coordinates, &coords);
1191:   for (k = 0; k < nVz; ++k) {
1192:     PetscInt ind[3], d, off;

1194:     ind[0] = 0;
1195:     ind[1] = 0;
1196:     ind[2] = k + da->zs;
1197:     for (j = 0; j < nVy; ++j) {
1198:       ind[1] = j + da->ys;
1199:       for (i = 0; i < nVx; ++i) {
1200:         const PetscInt vertex = (k*nVy + j)*nVx + i + vStart;

1202:         PetscSectionGetOffset(section, vertex, &off);
1203:         ind[0] = i + da->xs;
1204:         for (d = 0; d < dim; ++d) {
1205:           coords[off+d] = h[d]*ind[d];
1206:         }
1207:       }
1208:     }
1209:   }
1210:   VecRestoreArray(coordinates, &coords);
1211:   DMSetCoordinateSection(dm, section);
1212:   DMSetCoordinatesLocal(dm, coordinates);
1213:   PetscSectionDestroy(&section);
1214:   VecDestroy(&coordinates);
1215:   return(0);
1216: }

1220: PetscErrorCode DMDAProjectFunctionLocal(DM dm, PetscFE fe[], void (**funcs)(const PetscReal [], PetscScalar *, void *), void **ctxs, InsertMode mode, Vec localX)
1221: {
1222:   PetscDualSpace *sp;
1223:   PetscQuadrature q;
1224:   PetscSection    section;
1225:   PetscScalar    *values;
1226:   PetscReal      *v0, *J, *detJ;
1227:   PetscInt        numFields, numComp, dim, spDim, totDim = 0, numValues, cStart, cEnd, f, c, v, d;
1228:   PetscErrorCode  ierr;

1231:   DMGetDefaultSection(dm, &section);
1232:   PetscFEGetQuadrature(fe[0], &q);
1233:   PetscSectionGetNumFields(section, &numFields);
1234:   PetscMalloc(numFields * sizeof(PetscDualSpace), &sp);
1235:   for (f = 0; f < numFields; ++f) {
1236:     PetscFEGetDualSpace(fe[f], &sp[f]);
1237:     PetscFEGetNumComponents(fe[f], &numComp);
1238:     PetscDualSpaceGetDimension(sp[f], &spDim);
1239:     totDim += spDim*numComp;
1240:   }
1241:   DMDAGetInfo(dm, &dim,0,0,0,0,0,0,0,0,0,0,0,0);
1242:   DMDAGetHeightStratum(dm, 0, &cStart, &cEnd);
1243:   DMDAVecGetClosure(dm, section, localX, cStart, &numValues, NULL);
1244:   if (numValues != totDim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The section cell closure size %d != dual space dimension %d", numValues, totDim);
1245:   DMGetWorkArray(dm, numValues, PETSC_SCALAR, &values);
1246:   PetscMalloc3(dim*q.numPoints,&v0,dim*dim*q.numPoints,&J,q.numPoints,&detJ);
1247:   for (c = cStart; c < cEnd; ++c) {
1248:     PetscCellGeometry geom;

1250:     DMDAComputeCellGeometry(dm, c, &q, v0, J, NULL, detJ);
1251:     geom.v0   = v0;
1252:     geom.J    = J;
1253:     geom.detJ = detJ;
1254:     for (f = 0, v = 0; f < numFields; ++f) {
1255:       void * const ctx = ctxs ? ctxs[f] : NULL;
1256:       PetscFEGetNumComponents(fe[f], &numComp);
1257:       PetscDualSpaceGetDimension(sp[f], &spDim);
1258:       for (d = 0; d < spDim; ++d) {
1259:         PetscDualSpaceApply(sp[f], d, geom, numComp, funcs[f], ctx, &values[v]);
1260:         v += numComp;
1261:       }
1262:     }
1263:     DMDAVecSetClosure(dm, section, localX, c, values, mode);
1264:   }
1265:   DMRestoreWorkArray(dm, numValues, PETSC_SCALAR, &values);
1266:   PetscFree3(v0,J,detJ);
1267:   PetscFree(sp);
1268:   return(0);
1269: }

1273: /*@C
1274:   DMDAProjectFunction - This projects the given function into the function space provided.

1276:   Input Parameters:
1277: + dm      - The DM
1278: . fe      - The PetscFE associated with the field
1279: . funcs   - The coordinate functions to evaluate
1280: . ctxs    - Optional array of contexts to pass to each coordinate function.  ctxs itself may be null.
1281: - mode    - The insertion mode for values

1283:   Output Parameter:
1284: . X - vector

1286:   Level: developer

1288: .seealso: DMDAComputeL2Diff()
1289: @*/
1290: PetscErrorCode DMDAProjectFunction(DM dm, PetscFE fe[], void (**funcs)(const PetscReal [], PetscScalar *, void *), void **ctxs, InsertMode mode, Vec X)
1291: {
1292:   Vec            localX;

1297:   DMGetLocalVector(dm, &localX);
1298:   DMDAProjectFunctionLocal(dm, fe, funcs, ctxs, mode, localX);
1299:   DMLocalToGlobalBegin(dm, localX, mode, X);
1300:   DMLocalToGlobalEnd(dm, localX, mode, X);
1301:   DMRestoreLocalVector(dm, &localX);
1302:   return(0);
1303: }

1307: /*@C
1308:   DMDAComputeL2Diff - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.

1310:   Input Parameters:
1311: + dm    - The DM
1312: . fe    - The PetscFE object for each field
1313: . funcs - The functions to evaluate for each field component
1314: . ctxs  - Optional array of contexts to pass to each coordinate function.  ctxs itself may be null.
1315: - X     - The coefficient vector u_h

1317:   Output Parameter:
1318: . diff - The diff ||u - u_h||_2

1320:   Level: developer

1322: .seealso: DMDAProjectFunction()
1323: @*/
1324: PetscErrorCode DMDAComputeL2Diff(DM dm, PetscFE fe[], void (**funcs)(const PetscReal [], PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1325: {
1326:   const PetscInt  debug = 0;
1327:   PetscSection    section;
1328:   PetscQuadrature quad;
1329:   Vec             localX;
1330:   PetscScalar    *funcVal;
1331:   PetscReal      *coords, *v0, *J, *invJ, detJ;
1332:   PetscReal       localDiff = 0.0;
1333:   PetscInt        dim, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset, comp;
1334:   PetscErrorCode  ierr;

1337:   DMDAGetInfo(dm, &dim,0,0,0,0,0,0,0,0,0,0,0,0);
1338:   DMGetDefaultSection(dm, &section);
1339:   PetscSectionGetNumFields(section, &numFields);
1340:   DMGetLocalVector(dm, &localX);
1341:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);
1342:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);
1343:   for (field = 0; field < numFields; ++field) {
1344:     PetscInt Nc;

1346:     PetscFEGetNumComponents(fe[field], &Nc);
1347:     numComponents += Nc;
1348:   }
1349:   /* There are no BC values in DAs right now: DMDAProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX); */
1350:   PetscMalloc5(numComponents,&funcVal,dim,&coords,dim,&v0,dim*dim,&J,dim*dim,&invJ);
1351:   DMDAGetHeightStratum(dm, 0, &cStart, &cEnd);
1352:   PetscFEGetQuadrature(fe[0], &quad);
1353:   for (c = cStart; c < cEnd; ++c) {
1354:     PetscScalar *x = NULL;
1355:     PetscReal    elemDiff = 0.0;

1357:     DMDAComputeCellGeometry(dm, c, &quad, v0, J, invJ, &detJ);
1358:     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %d", detJ, c);
1359:     DMDAVecGetClosure(dm, NULL, localX, c, NULL, &x);

1361:     for (field = 0, comp = 0, fieldOffset = 0; field < numFields; ++field) {
1362:       void * const ctx = ctxs ? ctxs[field] : NULL;
1363:       const PetscInt   numQuadPoints = quad.numPoints;
1364:       const PetscReal *quadPoints    = quad.points;
1365:       const PetscReal *quadWeights   = quad.weights;
1366:       PetscReal       *basis;
1367:       PetscInt         numBasisFuncs, numBasisComps, q, d, e, fc, f;

1369:       PetscFEGetDimension(fe[field], &numBasisFuncs);
1370:       PetscFEGetNumComponents(fe[field], &numBasisComps);
1371:       PetscFEGetDefaultTabulation(fe[field], &basis, NULL, NULL);
1372:       if (debug) {
1373:         char title[1024];
1374:         PetscSNPrintf(title, 1023, "Solution for Field %d", field);
1375:         DMPrintCellVector(c, title, numBasisFuncs*numBasisComps, &x[fieldOffset]);
1376:       }
1377:       for (q = 0; q < numQuadPoints; ++q) {
1378:         for (d = 0; d < dim; d++) {
1379:           coords[d] = v0[d];
1380:           for (e = 0; e < dim; e++) {
1381:             coords[d] += J[d*dim+e]*(quadPoints[q*dim+e] + 1.0);
1382:           }
1383:         }
1384:         (*funcs[field])(coords, funcVal, ctx);
1385:         for (fc = 0; fc < numBasisComps; ++fc) {
1386:           PetscScalar interpolant = 0.0;

1388:           for (f = 0; f < numBasisFuncs; ++f) {
1389:             const PetscInt fidx = f*numBasisComps+fc;
1390:             interpolant += x[fieldOffset+fidx]*basis[q*numBasisFuncs*numBasisComps+fidx];
1391:           }
1392:           if (debug) {PetscPrintf(PETSC_COMM_SELF, "    elem %d field %d diff %g\n", c, field, PetscSqr(PetscRealPart(interpolant - funcVal[fc]))*quadWeights[q]*detJ);}
1393:           elemDiff += PetscSqr(PetscRealPart(interpolant - funcVal[fc]))*quadWeights[q]*detJ;
1394:         }
1395:       }
1396:       comp        += numBasisComps;
1397:       fieldOffset += numBasisFuncs*numBasisComps;
1398:     }
1399:     DMDAVecRestoreClosure(dm, NULL, localX, c, NULL, &x);
1400:     if (debug) {PetscPrintf(PETSC_COMM_SELF, "  elem %d diff %g\n", c, elemDiff);}
1401:     localDiff += elemDiff;
1402:   }
1403:   PetscFree5(funcVal,coords,v0,J,invJ);
1404:   DMRestoreLocalVector(dm, &localX);
1405:   MPI_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPI_SUM, PetscObjectComm((PetscObject)dm));
1406:   *diff = PetscSqrtReal(*diff);
1407:   return(0);
1408: }

1410: /* ------------------------------------------------------------------- */

1414: /*@C
1415:      DMDAGetArray - Gets a work array for a DMDA

1417:     Input Parameter:
1418: +    da - information about my local patch
1419: -    ghosted - do you want arrays for the ghosted or nonghosted patch

1421:     Output Parameters:
1422: .    vptr - array data structured

1424:     Note:  The vector values are NOT initialized and may have garbage in them, so you may need
1425:            to zero them.

1427:   Level: advanced

1429: .seealso: DMDARestoreArray()

1431: @*/
1432: PetscErrorCode  DMDAGetArray(DM da,PetscBool ghosted,void *vptr)
1433: {
1435:   PetscInt       j,i,xs,ys,xm,ym,zs,zm;
1436:   char           *iarray_start;
1437:   void           **iptr = (void**)vptr;
1438:   DM_DA          *dd    = (DM_DA*)da->data;

1442:   if (ghosted) {
1443:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1444:       if (dd->arrayghostedin[i]) {
1445:         *iptr                 = dd->arrayghostedin[i];
1446:         iarray_start          = (char*)dd->startghostedin[i];
1447:         dd->arrayghostedin[i] = NULL;
1448:         dd->startghostedin[i] = NULL;

1450:         goto done;
1451:       }
1452:     }
1453:     xs = dd->Xs;
1454:     ys = dd->Ys;
1455:     zs = dd->Zs;
1456:     xm = dd->Xe-dd->Xs;
1457:     ym = dd->Ye-dd->Ys;
1458:     zm = dd->Ze-dd->Zs;
1459:   } else {
1460:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1461:       if (dd->arrayin[i]) {
1462:         *iptr          = dd->arrayin[i];
1463:         iarray_start   = (char*)dd->startin[i];
1464:         dd->arrayin[i] = NULL;
1465:         dd->startin[i] = NULL;

1467:         goto done;
1468:       }
1469:     }
1470:     xs = dd->xs;
1471:     ys = dd->ys;
1472:     zs = dd->zs;
1473:     xm = dd->xe-dd->xs;
1474:     ym = dd->ye-dd->ys;
1475:     zm = dd->ze-dd->zs;
1476:   }

1478:   switch (dd->dim) {
1479:   case 1: {
1480:     void *ptr;

1482:     PetscMalloc(xm*sizeof(PetscScalar),&iarray_start);

1484:     ptr   = (void*)(iarray_start - xs*sizeof(PetscScalar));
1485:     *iptr = (void*)ptr;
1486:     break;
1487:   }
1488:   case 2: {
1489:     void **ptr;

1491:     PetscMalloc((ym+1)*sizeof(void*)+xm*ym*sizeof(PetscScalar),&iarray_start);

1493:     ptr = (void**)(iarray_start + xm*ym*sizeof(PetscScalar) - ys*sizeof(void*));
1494:     for (j=ys; j<ys+ym; j++) ptr[j] = iarray_start + sizeof(PetscScalar)*(xm*(j-ys) - xs);
1495:     *iptr = (void*)ptr;
1496:     break;
1497:   }
1498:   case 3: {
1499:     void ***ptr,**bptr;

1501:     PetscMalloc((zm+1)*sizeof(void**)+(ym*zm+1)*sizeof(void*)+xm*ym*zm*sizeof(PetscScalar),&iarray_start);

1503:     ptr  = (void***)(iarray_start + xm*ym*zm*sizeof(PetscScalar) - zs*sizeof(void*));
1504:     bptr = (void**)(iarray_start + xm*ym*zm*sizeof(PetscScalar) + zm*sizeof(void**));
1505:     for (i=zs; i<zs+zm; i++) ptr[i] = bptr + ((i-zs)*ym - ys);
1506:     for (i=zs; i<zs+zm; i++) {
1507:       for (j=ys; j<ys+ym; j++) {
1508:         ptr[i][j] = iarray_start + sizeof(PetscScalar)*(xm*ym*(i-zs) + xm*(j-ys) - xs);
1509:       }
1510:     }
1511:     *iptr = (void*)ptr;
1512:     break;
1513:   }
1514:   default:
1515:     SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Dimension %D not supported",dd->dim);
1516:   }

1518: done:
1519:   /* add arrays to the checked out list */
1520:   if (ghosted) {
1521:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1522:       if (!dd->arrayghostedout[i]) {
1523:         dd->arrayghostedout[i] = *iptr;
1524:         dd->startghostedout[i] = iarray_start;
1525:         break;
1526:       }
1527:     }
1528:   } else {
1529:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1530:       if (!dd->arrayout[i]) {
1531:         dd->arrayout[i] = *iptr;
1532:         dd->startout[i] = iarray_start;
1533:         break;
1534:       }
1535:     }
1536:   }
1537:   return(0);
1538: }

1542: /*@C
1543:      DMDARestoreArray - Restores an array of derivative types for a DMDA

1545:     Input Parameter:
1546: +    da - information about my local patch
1547: .    ghosted - do you want arrays for the ghosted or nonghosted patch
1548: -    vptr - array data structured to be passed to ad_FormFunctionLocal()

1550:      Level: advanced

1552: .seealso: DMDAGetArray()

1554: @*/
1555: PetscErrorCode  DMDARestoreArray(DM da,PetscBool ghosted,void *vptr)
1556: {
1557:   PetscInt i;
1558:   void     **iptr = (void**)vptr,*iarray_start = 0;
1559:   DM_DA    *dd    = (DM_DA*)da->data;

1563:   if (ghosted) {
1564:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1565:       if (dd->arrayghostedout[i] == *iptr) {
1566:         iarray_start           = dd->startghostedout[i];
1567:         dd->arrayghostedout[i] = NULL;
1568:         dd->startghostedout[i] = NULL;
1569:         break;
1570:       }
1571:     }
1572:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1573:       if (!dd->arrayghostedin[i]) {
1574:         dd->arrayghostedin[i] = *iptr;
1575:         dd->startghostedin[i] = iarray_start;
1576:         break;
1577:       }
1578:     }
1579:   } else {
1580:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1581:       if (dd->arrayout[i] == *iptr) {
1582:         iarray_start    = dd->startout[i];
1583:         dd->arrayout[i] = NULL;
1584:         dd->startout[i] = NULL;
1585:         break;
1586:       }
1587:     }
1588:     for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
1589:       if (!dd->arrayin[i]) {
1590:         dd->arrayin[i] = *iptr;
1591:         dd->startin[i] = iarray_start;
1592:         break;
1593:       }
1594:     }
1595:   }
1596:   return(0);
1597: }