Actual source code: da3.c

  1: #define PETSCDM_DLL
  2: /*
  3:    Code for manipulating distributed regular 3d arrays in parallel.
  4:    File created by Peter Mell  7/14/95
  5:  */

 7:  #include src/dm/da/daimpl.h

 11: PetscErrorCode DAView_3d(DA da,PetscViewer viewer)
 12: {
 14:   PetscMPIInt    rank;
 15:   PetscTruth     iascii,isdraw;

 18:   MPI_Comm_rank(((PetscObject)da)->comm,&rank);

 20:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 21:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
 22:   if (iascii) {
 23:     PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D P %D m %D n %D p %D w %D s %D\n",
 24:                rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
 25:     PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D, Z range of indices: %D %D\n",
 26:                da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
 27: #if !defined(PETSC_USE_COMPLEX)
 28:     if (da->coordinates) {
 29:       PetscInt  last;
 30:       PetscReal *coors;
 31:       VecGetArray(da->coordinates,&coors);
 32:       VecGetLocalSize(da->coordinates,&last);
 33:       last = last - 3;
 34:       PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %G %G %G : Upper right %G %G %G\n",
 35:                coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
 36:       VecRestoreArray(da->coordinates,&coors);
 37:     }
 38: #endif
 39:     PetscViewerFlush(viewer);
 40:   } else if (isdraw) {
 41:     PetscDraw       draw;
 42:     PetscReal     ymin = -1.0,ymax = (PetscReal)da->N;
 43:     PetscReal     xmin = -1.0,xmax = (PetscReal)((da->M+2)*da->P),x,y,ycoord,xcoord;
 44:     PetscInt        k,plane,base,*idx;
 45:     char       node[10];
 46:     PetscTruth isnull;

 48:     PetscViewerDrawGetDraw(viewer,0,&draw);
 49:     PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
 50:     PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
 51:     PetscDrawSynchronizedClear(draw);

 53:     /* first processor draw all node lines */
 54:     if (!rank) {
 55:       for (k=0; k<da->P; k++) {
 56:         ymin = 0.0; ymax = (PetscReal)(da->N - 1);
 57:         for (xmin=(PetscReal)(k*(da->M+1)); xmin<(PetscReal)(da->M+(k*(da->M+1))); xmin++) {
 58:           PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
 59:         }
 60: 
 61:         xmin = (PetscReal)(k*(da->M+1)); xmax = xmin + (PetscReal)(da->M - 1);
 62:         for (ymin=0; ymin<(PetscReal)da->N; ymin++) {
 63:           PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
 64:         }
 65:       }
 66:     }
 67:     PetscDrawSynchronizedFlush(draw);
 68:     PetscDrawPause(draw);

 70:     for (k=0; k<da->P; k++) {  /*Go through and draw for each plane*/
 71:       if ((k >= da->zs) && (k < da->ze)) {
 72:         /* draw my box */
 73:         ymin = da->ys;
 74:         ymax = da->ye - 1;
 75:         xmin = da->xs/da->w    + (da->M+1)*k;
 76:         xmax =(da->xe-1)/da->w + (da->M+1)*k;

 78:         PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
 79:         PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
 80:         PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
 81:         PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);

 83:         xmin = da->xs/da->w;
 84:         xmax =(da->xe-1)/da->w;

 86:         /* put in numbers*/
 87:         base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;

 89:         /* Identify which processor owns the box */
 90:         sprintf(node,"%d",rank);
 91:         PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);

 93:         for (y=ymin; y<=ymax; y++) {
 94:           for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
 95:             sprintf(node,"%d",(int)base++);
 96:             PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
 97:           }
 98:         }
 99: 
100:       }
101:     }
102:     PetscDrawSynchronizedFlush(draw);
103:     PetscDrawPause(draw);

105:     for (k=0-da->s; k<da->P+da->s; k++) {
106:       /* Go through and draw for each plane */
107:       if ((k >= da->Zs) && (k < da->Ze)) {
108: 
109:         /* overlay ghost numbers, useful for error checking */
110:         base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
111:         plane=k;
112:         /* Keep z wrap around points on the dradrawg */
113:         if (k<0)    { plane=da->P+k; }
114:         if (k>=da->P) { plane=k-da->P; }
115:         ymin = da->Ys; ymax = da->Ye;
116:         xmin = (da->M+1)*plane*da->w;
117:         xmax = (da->M+1)*plane*da->w+da->M*da->w;
118:         for (y=ymin; y<ymax; y++) {
119:           for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
120:             sprintf(node,"%d",(int)(idx[base]/da->w));
121:             ycoord = y;
122:             /*Keep y wrap around points on drawing */
123:             if (y<0)      { ycoord = da->N+y; }

125:             if (y>=da->N) { ycoord = y-da->N; }
126:             xcoord = x;   /* Keep x wrap points on drawing */

128:             if (x<xmin)  { xcoord = xmax - (xmin-x); }
129:             if (x>=xmax) { xcoord = xmin + (x-xmax); }
130:             PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
131:             base+=da->w;
132:           }
133:         }
134:       }
135:     }
136:     PetscDrawSynchronizedFlush(draw);
137:     PetscDrawPause(draw);
138:   } else {
139:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
140:   }
141:   return(0);
142: }

144: #if 0
145: EXTERN PetscErrorCode DAPublish_Petsc(PetscObject);
146: #endif

150: /*@C
151:    DACreate3d - Creates an object that will manage the communication of three-dimensional 
152:    regular array data that is distributed across some processors.

154:    Collective on MPI_Comm

156:    Input Parameters:
157: +  comm - MPI communicator
158: .  wrap - type of periodicity the array should have, if any.  Use one
159:           of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
160: .  stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
161: .  M,N,P - global dimension in each direction of the array (use -M, -N, and or -P to indicate that it may be set to a different value 
162:             from the command line with -da_grid_x <M> -da_grid_y <N> -da_grid_z <P>)
163: .  m,n,p - corresponding number of processors in each dimension 
164:            (or PETSC_DECIDE to have calculated)
165: .  dof - number of degrees of freedom per node
166: .  lx, ly, lz - arrays containing the number of nodes in each cell along
167:           the x, y, and z coordinates, or PETSC_NULL. If non-null, these
168:           must be of length as m,n,p and the corresponding
169:           m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
170:           the ly[] must N, sum of the lz[] must be P
171: -  s - stencil width

173:    Output Parameter:
174: .  inra - the resulting distributed array object

176:    Options Database Key:
177: +  -da_view - Calls DAView() at the conclusion of DACreate3d()
178: .  -da_grid_x <nx> - number of grid points in x direction, if M < 0
179: .  -da_grid_y <ny> - number of grid points in y direction, if N < 0
180: .  -da_grid_z <nz> - number of grid points in z direction, if P < 0
181: .  -da_processors_x <MX> number of processors in x direction
182: .  -da_processors_y <MY> number of processors in y direction
183: .  -da_processors_z <MZ> number of processors in z direction
184: .  -da_refine_x - refinement ratio in x direction
185: .  -da_refine_y - refinement ratio in y direction
186: -  -da_refine_y - refinement ratio in z direction

188:    Level: beginner

190:    Notes:
191:    The stencil type DA_STENCIL_STAR with width 1 corresponds to the 
192:    standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
193:    the standard 27-pt stencil.

195:    The array data itself is NOT stored in the DA, it is stored in Vec objects;
196:    The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
197:    and DACreateLocalVector() and calls to VecDuplicate() if more are needed.

199: .keywords: distributed array, create, three-dimensional

201: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(), DAGetRefinementFactor(),
202:           DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(), DASetRefinementFactor(),
203:           DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView(), DAGetOwnershipRange()

205: @*/
206: PetscErrorCode  DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
207:                PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,PetscInt *lx,PetscInt *ly,PetscInt *lz,DA *inra)
208: {
210:   PetscMPIInt    rank,size;
211:   PetscInt       xs = 0,xe,ys = 0,ye,zs = 0,ze,x = 0,y = 0,z = 0,Xs,Xe,Ys,Ye,Zs,Ze,start,end,pm;
212:   PetscInt       left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
213:   PetscInt       n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
214:   PetscInt       n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
215:   PetscInt       *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
216:   PetscInt       tM = M,tN = N,tP = P;
217:   PetscInt       sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
218:   PetscInt       sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
219:   PetscInt       sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
220:   DA             da;
221:   Vec            local,global;
222:   VecScatter     ltog,gtol;
223:   IS             to,from;

227:   *inra = 0;
228: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
229:   DMInitializePackage(PETSC_NULL);
230: #endif

232:   if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %D",dof);
233:   if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %D",s);

235:   PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
236:     if (M < 0){
237:       tM   = -M;
238:       PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",tM,&tM,PETSC_NULL);
239:     }
240:     if (N < 0){
241:       tN   = -N;
242:       PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",tN,&tN,PETSC_NULL);
243:     }
244:     if (P < 0){
245:       tP   = -P;
246:       PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",tP,&tP,PETSC_NULL);
247:     }
248:     PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
249:     PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
250:     PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
251:     PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DASetRefinementFactor",refine_x,&refine_x,PETSC_NULL);
252:     PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DASetRefinementFactor",refine_y,&refine_y,PETSC_NULL);
253:     PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DASetRefinementFactor",refine_z,&refine_z,PETSC_NULL);
254:   PetscOptionsEnd();
255:   M = tM; N = tN; P = tP;

257:   PetscHeaderCreate(da,_p_DA,struct _DAOps,DA_COOKIE,0,"DA",comm,DADestroy,DAView);
258:   da->ops->globaltolocalbegin = DAGlobalToLocalBegin;
259:   da->ops->globaltolocalend   = DAGlobalToLocalEnd;
260:   da->ops->localtoglobal      = DALocalToGlobal;
261:   da->ops->createglobalvector = DACreateGlobalVector;
262:   da->ops->getinterpolation   = DAGetInterpolation;
263:   da->ops->getcoloring        = DAGetColoring;
264:   da->ops->getmatrix          = DAGetMatrix;
265:   da->ops->refine             = DARefine;
266:   da->ops->coarsen            = DACoarsen;
267:   da->ops->getaggregates      = DAGetAggregates;

269:   da->dim        = 3;
270:   da->interptype = DA_Q1;
271:   da->refine_x   = refine_x;
272:   da->refine_y   = refine_y;
273:   da->refine_z   = refine_z;
274:   PetscMalloc(dof*sizeof(char*),&da->fieldname);
275:   PetscMemzero(da->fieldname,dof*sizeof(char*));

277:   MPI_Comm_size(comm,&size);
278:   MPI_Comm_rank(comm,&rank);

280:   if (m != PETSC_DECIDE) {
281:     if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %D",m);}
282:     else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %D %d",m,size);}
283:   }
284:   if (n != PETSC_DECIDE) {
285:     if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %D",n);}
286:     else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %D %d",n,size);}
287:   }
288:   if (p != PETSC_DECIDE) {
289:     if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %D",p);}
290:     else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %D %d",p,size);}
291:   }

293:   /* Partition the array among the processors */
294:   if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
295:     m = size/(n*p);
296:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
297:     n = size/(m*p);
298:   } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
299:     p = size/(m*n);
300:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
301:     /* try for squarish distribution */
302:     m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)N*p)));
303:     if (!m) m = 1;
304:     while (m > 0) {
305:       n = size/(m*p);
306:       if (m*n*p == size) break;
307:       m--;
308:     }
309:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %D",p);
310:     if (M > N && m < n) {PetscInt _m = m; m = n; n = _m;}
311:   } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
312:     /* try for squarish distribution */
313:     m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
314:     if (!m) m = 1;
315:     while (m > 0) {
316:       p = size/(m*n);
317:       if (m*n*p == size) break;
318:       m--;
319:     }
320:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %D",n);
321:     if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
322:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
323:     /* try for squarish distribution */
324:     n = (int)(0.5 + sqrt(((PetscReal)N)*((PetscReal)size)/((PetscReal)P*m)));
325:     if (!n) n = 1;
326:     while (n > 0) {
327:       p = size/(m*n);
328:       if (m*n*p == size) break;
329:       n--;
330:     }
331:     if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %D",n);
332:     if (N > P && n < p) {PetscInt _n = n; n = p; p = _n;}
333:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
334:     /* try for squarish distribution */
335:     n = (PetscInt)(0.5 + pow(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));
336:     if (!n) n = 1;
337:     while (n > 0) {
338:       pm = size/n;
339:       if (n*pm == size) break;
340:       n--;
341:     }
342:     if (!n) n = 1;
343:     m = (PetscInt)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
344:     if (!m) m = 1;
345:     while (m > 0) {
346:       p = size/(m*n);
347:       if (m*n*p == size) break;
348:       m--;
349:     }
350:     if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
351:   } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");

353:   if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
354:   if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %D %D",M,m);
355:   if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %D %D",N,n);
356:   if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %D %D",P,p);

358:   /* 
359:      Determine locally owned region 
360:      [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes 
361:   */
362:   if (!lx) { /* user decided distribution */
363:     PetscMalloc(m*sizeof(PetscInt),&lx);
364:     flx = lx;
365:     for (i=0; i<m; i++) {
366:       lx[i] = M/m + ((M % m) > (i % m));
367:     }
368:   }
369:   x  = lx[rank % m];
370:   xs = 0;
371:   for (i=0; i<(rank%m); i++) { xs += lx[i];}
372:   if (m > 1 && x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %D %D",x,s);

374:   if (!ly) { /* user decided distribution */
375:     PetscMalloc(n*sizeof(PetscInt),&ly);
376:     fly = ly;
377:     for (i=0; i<n; i++) {
378:       ly[i] = N/n + ((N % n) > (i % n));
379:     }
380:   }
381:   y  = ly[(rank % (m*n))/m];
382:   if (n > 1 && y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %D %D",y,s);
383:   ys = 0;
384:   for (i=0; i<(rank % (m*n))/m; i++) { ys += ly[i];}

386:   if (!lz) { /* user decided distribution */
387:     PetscMalloc(p*sizeof(PetscInt),&lz);
388:     flz = lz;
389:     for (i=0; i<p; i++) {
390:       lz[i] = P/p + ((P % p) > (i % p));
391:     }
392:   }
393:   z  = lz[rank/(m*n)];
394:   if (p > 1 && z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %D %D",z,s);
395:   zs = 0;
396:   for (i=0; i<(rank/(m*n)); i++) { zs += lz[i];}
397:   ye = ys + y;
398:   xe = xs + x;
399:   ze = zs + z;

401:   /* determine ghost region */
402:   /* Assume No Periodicity */
403:   if (xs-s > 0) Xs = xs - s; else Xs = 0;
404:   if (ys-s > 0) Ys = ys - s; else Ys = 0;
405:   if (zs-s > 0) Zs = zs - s; else Zs = 0;
406:   if (xe+s <= M) Xe = xe + s; else Xe = M;
407:   if (ye+s <= N) Ye = ye + s; else Ye = N;
408:   if (ze+s <= P) Ze = ze + s; else Ze = P;

410:   /* X Periodic */
411:   if (DAXPeriodic(wrap)){
412:     Xs = xs - s;
413:     Xe = xe + s;
414:   }

416:   /* Y Periodic */
417:   if (DAYPeriodic(wrap)){
418:     Ys = ys - s;
419:     Ye = ye + s;
420:   }

422:   /* Z Periodic */
423:   if (DAZPeriodic(wrap)){
424:     Zs = zs - s;
425:     Ze = ze + s;
426:   }

428:   /* Resize all X parameters to reflect w */
429:   x   *= dof;
430:   xs  *= dof;
431:   xe  *= dof;
432:   Xs  *= dof;
433:   Xe  *= dof;
434:   s_x  = s*dof;
435:   s_y  = s;
436:   s_z  = s;

438:   /* determine starting point of each processor */
439:   nn       = x*y*z;
440:   PetscMalloc((2*size+1)*sizeof(PetscInt),&bases);
441:   ldims    = (PetscInt*)(bases+size+1);
442:   MPI_Allgather(&nn,1,MPIU_INT,ldims,1,MPIU_INT,comm);
443:   bases[0] = 0;
444:   for (i=1; i<=size; i++) {
445:     bases[i] = ldims[i-1];
446:   }
447:   for (i=1; i<=size; i++) {
448:     bases[i] += bases[i-1];
449:   }

451:   /* allocate the base parallel and sequential vectors */
452:   da->Nlocal = x*y*z;
453:   VecCreateMPIWithArray(comm,da->Nlocal,PETSC_DECIDE,0,&global);
454:   VecSetBlockSize(global,dof);
455:   da->nlocal = (Xe-Xs)*(Ye-Ys)*(Ze-Zs);
456:   VecCreateSeqWithArray(MPI_COMM_SELF,da->nlocal,0,&local);
457:   VecSetBlockSize(local,dof);

459:   /* generate appropriate vector scatters */
460:   /* local to global inserts non-ghost point region into global */
461:   VecGetOwnershipRange(global,&start,&end);
462:   ISCreateStride(comm,x*y*z,start,1,&to);

464:   left   = xs - Xs;
465:   bottom = ys - Ys; top = bottom + y;
466:   down   = zs - Zs; up  = down + z;
467:   count  = x*(top-bottom)*(up-down);
468:   PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
469:   count  = 0;
470:   for (i=down; i<up; i++) {
471:     for (j=bottom; j<top; j++) {
472:       for (k=0; k<x; k += dof) {
473:         idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
474:       }
475:     }
476:   }
477:   ISCreateBlock(comm,dof,count,idx,&from);
478:   PetscFree(idx);

480:   VecScatterCreate(local,from,global,to,&ltog);
481:   PetscLogObjectParent(da,to);
482:   PetscLogObjectParent(da,from);
483:   PetscLogObjectParent(da,ltog);
484:   ISDestroy(from);
485:   ISDestroy(to);

487:   /* global to local must include ghost points */
488:   if (stencil_type == DA_STENCIL_BOX) {
489:     ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
490:   } else {
491:     /* This is way ugly! We need to list the funny cross type region */
492:     /* the bottom chunck */
493:     left   = xs - Xs;
494:     bottom = ys - Ys; top = bottom + y;
495:     down   = zs - Zs;   up  = down + z;
496:     count  = down*(top-bottom)*x + (up-down)*(bottom*x  + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) + (Ze-Zs-up)*(top-bottom)*x;
497:     PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
498:     count  = 0;
499:     for (i=0; i<down; i++) {
500:       for (j=bottom; j<top; j++) {
501:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
502:       }
503:     }
504:     /* the middle piece */
505:     for (i=down; i<up; i++) {
506:       /* front */
507:       for (j=0; j<bottom; j++) {
508:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
509:       }
510:       /* middle */
511:       for (j=bottom; j<top; j++) {
512:         for (k=0; k<Xe-Xs; k += dof) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
513:       }
514:       /* back */
515:       for (j=top; j<Ye-Ys; j++) {
516:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
517:       }
518:     }
519:     /* the top piece */
520:     for (i=up; i<Ze-Zs; i++) {
521:       for (j=bottom; j<top; j++) {
522:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
523:       }
524:     }
525:     ISCreateBlock(comm,dof,count,idx,&to);
526:     PetscFree(idx);
527:   }

529:   /* determine who lies on each side of use stored in    n24 n25 n26
530:                                                          n21 n22 n23
531:                                                          n18 n19 n20

533:                                                          n15 n16 n17
534:                                                          n12     n14
535:                                                          n9  n10 n11

537:                                                          n6  n7  n8
538:                                                          n3  n4  n5
539:                                                          n0  n1  n2
540:   */
541: 
542:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
543: 
544:   /* Assume Nodes are Internal to the Cube */
545: 
546:   n0  = rank - m*n - m - 1;
547:   n1  = rank - m*n - m;
548:   n2  = rank - m*n - m + 1;
549:   n3  = rank - m*n -1;
550:   n4  = rank - m*n;
551:   n5  = rank - m*n + 1;
552:   n6  = rank - m*n + m - 1;
553:   n7  = rank - m*n + m;
554:   n8  = rank - m*n + m + 1;

556:   n9  = rank - m - 1;
557:   n10 = rank - m;
558:   n11 = rank - m + 1;
559:   n12 = rank - 1;
560:   n14 = rank + 1;
561:   n15 = rank + m - 1;
562:   n16 = rank + m;
563:   n17 = rank + m + 1;

565:   n18 = rank + m*n - m - 1;
566:   n19 = rank + m*n - m;
567:   n20 = rank + m*n - m + 1;
568:   n21 = rank + m*n - 1;
569:   n22 = rank + m*n;
570:   n23 = rank + m*n + 1;
571:   n24 = rank + m*n + m - 1;
572:   n25 = rank + m*n + m;
573:   n26 = rank + m*n + m + 1;

575:   /* Assume Pieces are on Faces of Cube */

577:   if (xs == 0) { /* First assume not corner or edge */
578:     n0  = rank       -1 - (m*n);
579:     n3  = rank + m   -1 - (m*n);
580:     n6  = rank + 2*m -1 - (m*n);
581:     n9  = rank       -1;
582:     n12 = rank + m   -1;
583:     n15 = rank + 2*m -1;
584:     n18 = rank       -1 + (m*n);
585:     n21 = rank + m   -1 + (m*n);
586:     n24 = rank + 2*m -1 + (m*n);
587:    }

589:   if (xe == M*dof) { /* First assume not corner or edge */
590:     n2  = rank -2*m +1 - (m*n);
591:     n5  = rank - m  +1 - (m*n);
592:     n8  = rank      +1 - (m*n);
593:     n11 = rank -2*m +1;
594:     n14 = rank - m  +1;
595:     n17 = rank      +1;
596:     n20 = rank -2*m +1 + (m*n);
597:     n23 = rank - m  +1 + (m*n);
598:     n26 = rank      +1 + (m*n);
599:   }

601:   if (ys==0) { /* First assume not corner or edge */
602:     n0  = rank + m * (n-1) -1 - (m*n);
603:     n1  = rank + m * (n-1)    - (m*n);
604:     n2  = rank + m * (n-1) +1 - (m*n);
605:     n9  = rank + m * (n-1) -1;
606:     n10 = rank + m * (n-1);
607:     n11 = rank + m * (n-1) +1;
608:     n18 = rank + m * (n-1) -1 + (m*n);
609:     n19 = rank + m * (n-1)    + (m*n);
610:     n20 = rank + m * (n-1) +1 + (m*n);
611:   }

613:   if (ye == N) { /* First assume not corner or edge */
614:     n6  = rank - m * (n-1) -1 - (m*n);
615:     n7  = rank - m * (n-1)    - (m*n);
616:     n8  = rank - m * (n-1) +1 - (m*n);
617:     n15 = rank - m * (n-1) -1;
618:     n16 = rank - m * (n-1);
619:     n17 = rank - m * (n-1) +1;
620:     n24 = rank - m * (n-1) -1 + (m*n);
621:     n25 = rank - m * (n-1)    + (m*n);
622:     n26 = rank - m * (n-1) +1 + (m*n);
623:   }
624: 
625:   if (zs == 0) { /* First assume not corner or edge */
626:     n0 = size - (m*n) + rank - m - 1;
627:     n1 = size - (m*n) + rank - m;
628:     n2 = size - (m*n) + rank - m + 1;
629:     n3 = size - (m*n) + rank - 1;
630:     n4 = size - (m*n) + rank;
631:     n5 = size - (m*n) + rank + 1;
632:     n6 = size - (m*n) + rank + m - 1;
633:     n7 = size - (m*n) + rank + m ;
634:     n8 = size - (m*n) + rank + m + 1;
635:   }

637:   if (ze == P) { /* First assume not corner or edge */
638:     n18 = (m*n) - (size-rank) - m - 1;
639:     n19 = (m*n) - (size-rank) - m;
640:     n20 = (m*n) - (size-rank) - m + 1;
641:     n21 = (m*n) - (size-rank) - 1;
642:     n22 = (m*n) - (size-rank);
643:     n23 = (m*n) - (size-rank) + 1;
644:     n24 = (m*n) - (size-rank) + m - 1;
645:     n25 = (m*n) - (size-rank) + m;
646:     n26 = (m*n) - (size-rank) + m + 1;
647:   }

649:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
650:     n0 = size - m*n + rank + m-1 - m;
651:     n3 = size - m*n + rank + m-1;
652:     n6 = size - m*n + rank + m-1 + m;
653:   }
654: 
655:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
656:     n18 = m*n - (size - rank) + m-1 - m;
657:     n21 = m*n - (size - rank) + m-1;
658:     n24 = m*n - (size - rank) + m-1 + m;
659:   }

661:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
662:     n0  = rank + m*n -1 - m*n;
663:     n9  = rank + m*n -1;
664:     n18 = rank + m*n -1 + m*n;
665:   }

667:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
668:     n6  = rank - m*(n-1) + m-1 - m*n;
669:     n15 = rank - m*(n-1) + m-1;
670:     n24 = rank - m*(n-1) + m-1 + m*n;
671:   }

673:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
674:     n2 = size - (m*n-rank) - (m-1) - m;
675:     n5 = size - (m*n-rank) - (m-1);
676:     n8 = size - (m*n-rank) - (m-1) + m;
677:   }

679:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
680:     n20 = m*n - (size - rank) - (m-1) - m;
681:     n23 = m*n - (size - rank) - (m-1);
682:     n26 = m*n - (size - rank) - (m-1) + m;
683:   }

685:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
686:     n2  = rank + m*(n-1) - (m-1) - m*n;
687:     n11 = rank + m*(n-1) - (m-1);
688:     n20 = rank + m*(n-1) - (m-1) + m*n;
689:   }

691:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
692:     n8  = rank - m*n +1 - m*n;
693:     n17 = rank - m*n +1;
694:     n26 = rank - m*n +1 + m*n;
695:   }

697:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
698:     n0 = size - m + rank -1;
699:     n1 = size - m + rank;
700:     n2 = size - m + rank +1;
701:   }

703:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
704:     n18 = m*n - (size - rank) + m*(n-1) -1;
705:     n19 = m*n - (size - rank) + m*(n-1);
706:     n20 = m*n - (size - rank) + m*(n-1) +1;
707:   }

709:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
710:     n6 = size - (m*n-rank) - m * (n-1) -1;
711:     n7 = size - (m*n-rank) - m * (n-1);
712:     n8 = size - (m*n-rank) - m * (n-1) +1;
713:   }

715:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
716:     n24 = rank - (size-m) -1;
717:     n25 = rank - (size-m);
718:     n26 = rank - (size-m) +1;
719:   }

721:   /* Check for Corners */
722:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
723:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
724:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
725:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
726:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
727:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
728:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
729:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

731:   /* Check for when not X,Y, and Z Periodic */

733:   /* If not X periodic */
734:   if ((wrap != DA_XPERIODIC)  && (wrap != DA_XYPERIODIC) &&
735:      (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
736:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
737:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
738:   }

740:   /* If not Y periodic */
741:   if ((wrap != DA_YPERIODIC)  && (wrap != DA_XYPERIODIC) &&
742:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
743:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
744:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
745:   }

747:   /* If not Z periodic */
748:   if ((wrap != DA_ZPERIODIC)  && (wrap != DA_XZPERIODIC) &&
749:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
750:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
751:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
752:   }

754:   /* If star stencil then delete the corner neighbors */
755:   if (stencil_type == DA_STENCIL_STAR) {
756:      /* save information about corner neighbors */
757:      sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
758:      sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
759:      sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
760:      sn26 = n26;
761:      n0  = n1  = n2  = n3  = n5  = n6  = n7  = n8  = n9  = n11 =
762:      n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
763:   }


766:   PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt),&idx);
767:   PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt));

769:   nn = 0;

771:   /* Bottom Level */
772:   for (k=0; k<s_z; k++) {
773:     for (i=1; i<=s_y; i++) {
774:       if (n0 >= 0) { /* left below */
775:         x_t = lx[n0 % m]*dof;
776:         y_t = ly[(n0 % (m*n))/m];
777:         z_t = lz[n0 / (m*n)];
778:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
779:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
780:       }
781:       if (n1 >= 0) { /* directly below */
782:         x_t = x;
783:         y_t = ly[(n1 % (m*n))/m];
784:         z_t = lz[n1 / (m*n)];
785:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
786:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
787:       }
788:       if (n2 >= 0) { /* right below */
789:         x_t = lx[n2 % m]*dof;
790:         y_t = ly[(n2 % (m*n))/m];
791:         z_t = lz[n2 / (m*n)];
792:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
793:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
794:       }
795:     }

797:     for (i=0; i<y; i++) {
798:       if (n3 >= 0) { /* directly left */
799:         x_t = lx[n3 % m]*dof;
800:         y_t = y;
801:         z_t = lz[n3 / (m*n)];
802:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
803:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
804:       }

806:       if (n4 >= 0) { /* middle */
807:         x_t = x;
808:         y_t = y;
809:         z_t = lz[n4 / (m*n)];
810:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
811:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
812:       }

814:       if (n5 >= 0) { /* directly right */
815:         x_t = lx[n5 % m]*dof;
816:         y_t = y;
817:         z_t = lz[n5 / (m*n)];
818:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
819:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
820:       }
821:     }

823:     for (i=1; i<=s_y; i++) {
824:       if (n6 >= 0) { /* left above */
825:         x_t = lx[n6 % m]*dof;
826:         y_t = ly[(n6 % (m*n))/m];
827:         z_t = lz[n6 / (m*n)];
828:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
829:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
830:       }
831:       if (n7 >= 0) { /* directly above */
832:         x_t = x;
833:         y_t = ly[(n7 % (m*n))/m];
834:         z_t = lz[n7 / (m*n)];
835:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
836:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
837:       }
838:       if (n8 >= 0) { /* right above */
839:         x_t = lx[n8 % m]*dof;
840:         y_t = ly[(n8 % (m*n))/m];
841:         z_t = lz[n8 / (m*n)];
842:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
843:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
844:       }
845:     }
846:   }

848:   /* Middle Level */
849:   for (k=0; k<z; k++) {
850:     for (i=1; i<=s_y; i++) {
851:       if (n9 >= 0) { /* left below */
852:         x_t = lx[n9 % m]*dof;
853:         y_t = ly[(n9 % (m*n))/m];
854:         /* z_t = z; */
855:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
856:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
857:       }
858:       if (n10 >= 0) { /* directly below */
859:         x_t = x;
860:         y_t = ly[(n10 % (m*n))/m];
861:         /* z_t = z; */
862:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
863:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
864:       }
865:       if (n11 >= 0) { /* right below */
866:         x_t = lx[n11 % m]*dof;
867:         y_t = ly[(n11 % (m*n))/m];
868:         /* z_t = z; */
869:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
870:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
871:       }
872:     }

874:     for (i=0; i<y; i++) {
875:       if (n12 >= 0) { /* directly left */
876:         x_t = lx[n12 % m]*dof;
877:         y_t = y;
878:         /* z_t = z; */
879:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
880:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
881:       }

883:       /* Interior */
884:       s_t = bases[rank] + i*x + k*x*y;
885:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

887:       if (n14 >= 0) { /* directly right */
888:         x_t = lx[n14 % m]*dof;
889:         y_t = y;
890:         /* z_t = z; */
891:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
892:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
893:       }
894:     }

896:     for (i=1; i<=s_y; i++) {
897:       if (n15 >= 0) { /* left above */
898:         x_t = lx[n15 % m]*dof;
899:         y_t = ly[(n15 % (m*n))/m];
900:         /* z_t = z; */
901:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
902:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
903:       }
904:       if (n16 >= 0) { /* directly above */
905:         x_t = x;
906:         y_t = ly[(n16 % (m*n))/m];
907:         /* z_t = z; */
908:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
909:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
910:       }
911:       if (n17 >= 0) { /* right above */
912:         x_t = lx[n17 % m]*dof;
913:         y_t = ly[(n17 % (m*n))/m];
914:         /* z_t = z; */
915:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
916:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
917:       }
918:     }
919:   }
920: 
921:   /* Upper Level */
922:   for (k=0; k<s_z; k++) {
923:     for (i=1; i<=s_y; i++) {
924:       if (n18 >= 0) { /* left below */
925:         x_t = lx[n18 % m]*dof;
926:         y_t = ly[(n18 % (m*n))/m];
927:         /* z_t = lz[n18 / (m*n)]; */
928:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
929:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
930:       }
931:       if (n19 >= 0) { /* directly below */
932:         x_t = x;
933:         y_t = ly[(n19 % (m*n))/m];
934:         /* z_t = lz[n19 / (m*n)]; */
935:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
936:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
937:       }
938:       if (n20 >= 0) { /* right below */
939:         x_t = lx[n20 % m]*dof;
940:         y_t = ly[(n20 % (m*n))/m];
941:         /* z_t = lz[n20 / (m*n)]; */
942:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
943:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
944:       }
945:     }

947:     for (i=0; i<y; i++) {
948:       if (n21 >= 0) { /* directly left */
949:         x_t = lx[n21 % m]*dof;
950:         y_t = y;
951:         /* z_t = lz[n21 / (m*n)]; */
952:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
953:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
954:       }

956:       if (n22 >= 0) { /* middle */
957:         x_t = x;
958:         y_t = y;
959:         /* z_t = lz[n22 / (m*n)]; */
960:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
961:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
962:       }

964:       if (n23 >= 0) { /* directly right */
965:         x_t = lx[n23 % m]*dof;
966:         y_t = y;
967:         /* z_t = lz[n23 / (m*n)]; */
968:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
969:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
970:       }
971:     }

973:     for (i=1; i<=s_y; i++) {
974:       if (n24 >= 0) { /* left above */
975:         x_t = lx[n24 % m]*dof;
976:         y_t = ly[(n24 % (m*n))/m];
977:         /* z_t = lz[n24 / (m*n)]; */
978:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
979:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
980:       }
981:       if (n25 >= 0) { /* directly above */
982:         x_t = x;
983:         y_t = ly[(n25 % (m*n))/m];
984:         /* z_t = lz[n25 / (m*n)]; */
985:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
986:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
987:       }
988:       if (n26 >= 0) { /* right above */
989:         x_t = lx[n26 % m]*dof;
990:         y_t = ly[(n26 % (m*n))/m];
991:         /* z_t = lz[n26 / (m*n)]; */
992:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
993:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
994:       }
995:     }
996:   }
997:   base = bases[rank];
998:   {
999:     PetscInt nnn = nn/dof,*iidx;
1000:     PetscMalloc(nnn*sizeof(PetscInt),&iidx);
1001:     for (i=0; i<nnn; i++) {
1002:       iidx[i] = idx[dof*i];
1003:     }
1004:     ISCreateBlock(comm,dof,nnn,iidx,&from);
1005:     PetscFree(iidx);
1006:   }
1007:   VecScatterCreate(global,from,local,to,&gtol);
1008:   PetscLogObjectParent(da,gtol);
1009:   PetscLogObjectParent(da,to);
1010:   PetscLogObjectParent(da,from);
1011:   ISDestroy(to);
1012:   ISDestroy(from);
1013:   da->stencil_type = stencil_type;
1014:   da->M  = M;  da->N  = N; da->P = P;
1015:   da->m  = m;  da->n  = n; da->p = p;
1016:   da->w  = dof;  da->s  = s;
1017:   da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1018:   da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;

1020:   VecDestroy(local);
1021:   VecDestroy(global);

1023:   if (stencil_type == DA_STENCIL_STAR) {
1024:     /*
1025:         Recompute the local to global mappings, this time keeping the 
1026:       information about the cross corner processor numbers.
1027:     */
1028:     n0  = sn0;  n1  = sn1;  n2  = sn2;  n3  = sn3;  n5  = sn5;  n6  = sn6; n7 = sn7;
1029:     n8  = sn8;  n9  = sn9;  n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1030:     n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1031:     n26 = sn26;

1033:     nn = 0;

1035:     /* Bottom Level */
1036:     for (k=0; k<s_z; k++) {
1037:       for (i=1; i<=s_y; i++) {
1038:         if (n0 >= 0) { /* left below */
1039:           x_t = lx[n0 % m]*dof;
1040:           y_t = ly[(n0 % (m*n))/m];
1041:           z_t = lz[n0 / (m*n)];
1042:           s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1043:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1044:         }
1045:         if (n1 >= 0) { /* directly below */
1046:           x_t = x;
1047:           y_t = ly[(n1 % (m*n))/m];
1048:           z_t = lz[n1 / (m*n)];
1049:           s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1050:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1051:         }
1052:         if (n2 >= 0) { /* right below */
1053:           x_t = lx[n2 % m]*dof;
1054:           y_t = ly[(n2 % (m*n))/m];
1055:           z_t = lz[n2 / (m*n)];
1056:           s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1057:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1058:         }
1059:       }

1061:       for (i=0; i<y; i++) {
1062:         if (n3 >= 0) { /* directly left */
1063:           x_t = lx[n3 % m]*dof;
1064:           y_t = y;
1065:           z_t = lz[n3 / (m*n)];
1066:           s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1067:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1068:         }

1070:         if (n4 >= 0) { /* middle */
1071:           x_t = x;
1072:           y_t = y;
1073:           z_t = lz[n4 / (m*n)];
1074:           s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1075:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1076:         }

1078:         if (n5 >= 0) { /* directly right */
1079:           x_t = lx[n5 % m]*dof;
1080:           y_t = y;
1081:           z_t = lz[n5 / (m*n)];
1082:           s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1083:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1084:         }
1085:       }

1087:       for (i=1; i<=s_y; i++) {
1088:         if (n6 >= 0) { /* left above */
1089:           x_t = lx[n6 % m]*dof;
1090:           y_t = ly[(n6 % (m*n))/m];
1091:           z_t = lz[n6 / (m*n)];
1092:           s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1093:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1094:         }
1095:         if (n7 >= 0) { /* directly above */
1096:           x_t = x;
1097:           y_t = ly[(n7 % (m*n))/m];
1098:           z_t = lz[n7 / (m*n)];
1099:           s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1100:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1101:         }
1102:         if (n8 >= 0) { /* right above */
1103:           x_t = lx[n8 % m]*dof;
1104:           y_t = ly[(n8 % (m*n))/m];
1105:           z_t = lz[n8 / (m*n)];
1106:           s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1107:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1108:         }
1109:       }
1110:     }

1112:     /* Middle Level */
1113:     for (k=0; k<z; k++) {
1114:       for (i=1; i<=s_y; i++) {
1115:         if (n9 >= 0) { /* left below */
1116:           x_t = lx[n9 % m]*dof;
1117:           y_t = ly[(n9 % (m*n))/m];
1118:           /* z_t = z; */
1119:           s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1120:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1121:         }
1122:         if (n10 >= 0) { /* directly below */
1123:           x_t = x;
1124:           y_t = ly[(n10 % (m*n))/m];
1125:           /* z_t = z; */
1126:           s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1127:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1128:         }
1129:         if (n11 >= 0) { /* right below */
1130:           x_t = lx[n11 % m]*dof;
1131:           y_t = ly[(n11 % (m*n))/m];
1132:           /* z_t = z; */
1133:           s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1134:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1135:         }
1136:       }

1138:       for (i=0; i<y; i++) {
1139:         if (n12 >= 0) { /* directly left */
1140:           x_t = lx[n12 % m]*dof;
1141:           y_t = y;
1142:           /* z_t = z; */
1143:           s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1144:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1145:         }

1147:         /* Interior */
1148:         s_t = bases[rank] + i*x + k*x*y;
1149:         for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1151:         if (n14 >= 0) { /* directly right */
1152:           x_t = lx[n14 % m]*dof;
1153:           y_t = y;
1154:           /* z_t = z; */
1155:           s_t = bases[n14] + i*x_t + k*x_t*y_t;
1156:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1157:         }
1158:       }

1160:       for (i=1; i<=s_y; i++) {
1161:         if (n15 >= 0) { /* left above */
1162:           x_t = lx[n15 % m]*dof;
1163:           y_t = ly[(n15 % (m*n))/m];
1164:           /* z_t = z; */
1165:           s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1166:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1167:         }
1168:         if (n16 >= 0) { /* directly above */
1169:           x_t = x;
1170:           y_t = ly[(n16 % (m*n))/m];
1171:           /* z_t = z; */
1172:           s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1173:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1174:         }
1175:         if (n17 >= 0) { /* right above */
1176:           x_t = lx[n17 % m]*dof;
1177:           y_t = ly[(n17 % (m*n))/m];
1178:           /* z_t = z; */
1179:           s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1180:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1181:         }
1182:       }
1183:     }
1184: 
1185:     /* Upper Level */
1186:     for (k=0; k<s_z; k++) {
1187:       for (i=1; i<=s_y; i++) {
1188:         if (n18 >= 0) { /* left below */
1189:           x_t = lx[n18 % m]*dof;
1190:           y_t = ly[(n18 % (m*n))/m];
1191:           /* z_t = lz[n18 / (m*n)]; */
1192:           s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1193:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1194:         }
1195:         if (n19 >= 0) { /* directly below */
1196:           x_t = x;
1197:           y_t = ly[(n19 % (m*n))/m];
1198:           /* z_t = lz[n19 / (m*n)]; */
1199:           s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1200:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1201:         }
1202:         if (n20 >= 0) { /* right below */
1203:           x_t = lx[n20 % m]*dof;
1204:           y_t = ly[(n20 % (m*n))/m];
1205:           /* z_t = lz[n20 / (m*n)]; */
1206:           s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1207:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1208:         }
1209:       }

1211:       for (i=0; i<y; i++) {
1212:         if (n21 >= 0) { /* directly left */
1213:           x_t = lx[n21 % m]*dof;
1214:           y_t = y;
1215:           /* z_t = lz[n21 / (m*n)]; */
1216:           s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1217:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1218:         }

1220:         if (n22 >= 0) { /* middle */
1221:           x_t = x;
1222:           y_t = y;
1223:           /* z_t = lz[n22 / (m*n)]; */
1224:           s_t = bases[n22] + i*x_t + k*x_t*y_t;
1225:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1226:         }

1228:         if (n23 >= 0) { /* directly right */
1229:           x_t = lx[n23 % m]*dof;
1230:           y_t = y;
1231:           /* z_t = lz[n23 / (m*n)]; */
1232:           s_t = bases[n23] + i*x_t + k*x_t*y_t;
1233:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1234:         }
1235:       }

1237:       for (i=1; i<=s_y; i++) {
1238:         if (n24 >= 0) { /* left above */
1239:           x_t = lx[n24 % m]*dof;
1240:           y_t = ly[(n24 % (m*n))/m];
1241:           /* z_t = lz[n24 / (m*n)]; */
1242:           s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1243:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1244:         }
1245:         if (n25 >= 0) { /* directly above */
1246:           x_t = x;
1247:           y_t = ly[(n25 % (m*n))/m];
1248:           /* z_t = lz[n25 / (m*n)]; */
1249:           s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1250:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1251:         }
1252:         if (n26 >= 0) { /* right above */
1253:           x_t = lx[n26 % m]*dof;
1254:           y_t = ly[(n26 % (m*n))/m];
1255:           /* z_t = lz[n26 / (m*n)]; */
1256:           s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1257:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1258:         }
1259:       }
1260:     }
1261:   }
1262:   /* redo idx to include "missing" ghost points */
1263:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1264: 
1265:   /* Assume Nodes are Internal to the Cube */
1266: 
1267:   n0  = rank - m*n - m - 1;
1268:   n1  = rank - m*n - m;
1269:   n2  = rank - m*n - m + 1;
1270:   n3  = rank - m*n -1;
1271:   n4  = rank - m*n;
1272:   n5  = rank - m*n + 1;
1273:   n6  = rank - m*n + m - 1;
1274:   n7  = rank - m*n + m;
1275:   n8  = rank - m*n + m + 1;

1277:   n9  = rank - m - 1;
1278:   n10 = rank - m;
1279:   n11 = rank - m + 1;
1280:   n12 = rank - 1;
1281:   n14 = rank + 1;
1282:   n15 = rank + m - 1;
1283:   n16 = rank + m;
1284:   n17 = rank + m + 1;

1286:   n18 = rank + m*n - m - 1;
1287:   n19 = rank + m*n - m;
1288:   n20 = rank + m*n - m + 1;
1289:   n21 = rank + m*n - 1;
1290:   n22 = rank + m*n;
1291:   n23 = rank + m*n + 1;
1292:   n24 = rank + m*n + m - 1;
1293:   n25 = rank + m*n + m;
1294:   n26 = rank + m*n + m + 1;

1296:   /* Assume Pieces are on Faces of Cube */

1298:   if (xs == 0) { /* First assume not corner or edge */
1299:     n0  = rank       -1 - (m*n);
1300:     n3  = rank + m   -1 - (m*n);
1301:     n6  = rank + 2*m -1 - (m*n);
1302:     n9  = rank       -1;
1303:     n12 = rank + m   -1;
1304:     n15 = rank + 2*m -1;
1305:     n18 = rank       -1 + (m*n);
1306:     n21 = rank + m   -1 + (m*n);
1307:     n24 = rank + 2*m -1 + (m*n);
1308:    }

1310:   if (xe == M*dof) { /* First assume not corner or edge */
1311:     n2  = rank -2*m +1 - (m*n);
1312:     n5  = rank - m  +1 - (m*n);
1313:     n8  = rank      +1 - (m*n);
1314:     n11 = rank -2*m +1;
1315:     n14 = rank - m  +1;
1316:     n17 = rank      +1;
1317:     n20 = rank -2*m +1 + (m*n);
1318:     n23 = rank - m  +1 + (m*n);
1319:     n26 = rank      +1 + (m*n);
1320:   }

1322:   if (ys==0) { /* First assume not corner or edge */
1323:     n0  = rank + m * (n-1) -1 - (m*n);
1324:     n1  = rank + m * (n-1)    - (m*n);
1325:     n2  = rank + m * (n-1) +1 - (m*n);
1326:     n9  = rank + m * (n-1) -1;
1327:     n10 = rank + m * (n-1);
1328:     n11 = rank + m * (n-1) +1;
1329:     n18 = rank + m * (n-1) -1 + (m*n);
1330:     n19 = rank + m * (n-1)    + (m*n);
1331:     n20 = rank + m * (n-1) +1 + (m*n);
1332:   }

1334:   if (ye == N) { /* First assume not corner or edge */
1335:     n6  = rank - m * (n-1) -1 - (m*n);
1336:     n7  = rank - m * (n-1)    - (m*n);
1337:     n8  = rank - m * (n-1) +1 - (m*n);
1338:     n15 = rank - m * (n-1) -1;
1339:     n16 = rank - m * (n-1);
1340:     n17 = rank - m * (n-1) +1;
1341:     n24 = rank - m * (n-1) -1 + (m*n);
1342:     n25 = rank - m * (n-1)    + (m*n);
1343:     n26 = rank - m * (n-1) +1 + (m*n);
1344:   }
1345: 
1346:   if (zs == 0) { /* First assume not corner or edge */
1347:     n0 = size - (m*n) + rank - m - 1;
1348:     n1 = size - (m*n) + rank - m;
1349:     n2 = size - (m*n) + rank - m + 1;
1350:     n3 = size - (m*n) + rank - 1;
1351:     n4 = size - (m*n) + rank;
1352:     n5 = size - (m*n) + rank + 1;
1353:     n6 = size - (m*n) + rank + m - 1;
1354:     n7 = size - (m*n) + rank + m ;
1355:     n8 = size - (m*n) + rank + m + 1;
1356:   }

1358:   if (ze == P) { /* First assume not corner or edge */
1359:     n18 = (m*n) - (size-rank) - m - 1;
1360:     n19 = (m*n) - (size-rank) - m;
1361:     n20 = (m*n) - (size-rank) - m + 1;
1362:     n21 = (m*n) - (size-rank) - 1;
1363:     n22 = (m*n) - (size-rank);
1364:     n23 = (m*n) - (size-rank) + 1;
1365:     n24 = (m*n) - (size-rank) + m - 1;
1366:     n25 = (m*n) - (size-rank) + m;
1367:     n26 = (m*n) - (size-rank) + m + 1;
1368:   }

1370:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1371:     n0 = size - m*n + rank + m-1 - m;
1372:     n3 = size - m*n + rank + m-1;
1373:     n6 = size - m*n + rank + m-1 + m;
1374:   }
1375: 
1376:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1377:     n18 = m*n - (size - rank) + m-1 - m;
1378:     n21 = m*n - (size - rank) + m-1;
1379:     n24 = m*n - (size - rank) + m-1 + m;
1380:   }

1382:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1383:     n0  = rank + m*n -1 - m*n;
1384:     n9  = rank + m*n -1;
1385:     n18 = rank + m*n -1 + m*n;
1386:   }

1388:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1389:     n6  = rank - m*(n-1) + m-1 - m*n;
1390:     n15 = rank - m*(n-1) + m-1;
1391:     n24 = rank - m*(n-1) + m-1 + m*n;
1392:   }

1394:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1395:     n2 = size - (m*n-rank) - (m-1) - m;
1396:     n5 = size - (m*n-rank) - (m-1);
1397:     n8 = size - (m*n-rank) - (m-1) + m;
1398:   }

1400:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1401:     n20 = m*n - (size - rank) - (m-1) - m;
1402:     n23 = m*n - (size - rank) - (m-1);
1403:     n26 = m*n - (size - rank) - (m-1) + m;
1404:   }

1406:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1407:     n2  = rank + m*(n-1) - (m-1) - m*n;
1408:     n11 = rank + m*(n-1) - (m-1);
1409:     n20 = rank + m*(n-1) - (m-1) + m*n;
1410:   }

1412:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1413:     n8  = rank - m*n +1 - m*n;
1414:     n17 = rank - m*n +1;
1415:     n26 = rank - m*n +1 + m*n;
1416:   }

1418:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1419:     n0 = size - m + rank -1;
1420:     n1 = size - m + rank;
1421:     n2 = size - m + rank +1;
1422:   }

1424:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1425:     n18 = m*n - (size - rank) + m*(n-1) -1;
1426:     n19 = m*n - (size - rank) + m*(n-1);
1427:     n20 = m*n - (size - rank) + m*(n-1) +1;
1428:   }

1430:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1431:     n6 = size - (m*n-rank) - m * (n-1) -1;
1432:     n7 = size - (m*n-rank) - m * (n-1);
1433:     n8 = size - (m*n-rank) - m * (n-1) +1;
1434:   }

1436:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1437:     n24 = rank - (size-m) -1;
1438:     n25 = rank - (size-m);
1439:     n26 = rank - (size-m) +1;
1440:   }

1442:   /* Check for Corners */
1443:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
1444:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
1445:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
1446:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
1447:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
1448:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1449:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
1450:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

1452:   /* Check for when not X,Y, and Z Periodic */

1454:   /* If not X periodic */
1455:   if (!DAXPeriodic(wrap)){
1456:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
1457:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1458:   }

1460:   /* If not Y periodic */
1461:   if (!DAYPeriodic(wrap)){
1462:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
1463:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1464:   }

1466:   /* If not Z periodic */
1467:   if (!DAZPeriodic(wrap)){
1468:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
1469:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1470:   }

1472:   nn = 0;

1474:   /* Bottom Level */
1475:   for (k=0; k<s_z; k++) {
1476:     for (i=1; i<=s_y; i++) {
1477:       if (n0 >= 0) { /* left below */
1478:         x_t = lx[n0 % m]*dof;
1479:         y_t = ly[(n0 % (m*n))/m];
1480:         z_t = lz[n0 / (m*n)];
1481:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1482:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1483:       }
1484:       if (n1 >= 0) { /* directly below */
1485:         x_t = x;
1486:         y_t = ly[(n1 % (m*n))/m];
1487:         z_t = lz[n1 / (m*n)];
1488:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1489:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1490:       }
1491:       if (n2 >= 0) { /* right below */
1492:         x_t = lx[n2 % m]*dof;
1493:         y_t = ly[(n2 % (m*n))/m];
1494:         z_t = lz[n2 / (m*n)];
1495:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1496:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1497:       }
1498:     }

1500:     for (i=0; i<y; i++) {
1501:       if (n3 >= 0) { /* directly left */
1502:         x_t = lx[n3 % m]*dof;
1503:         y_t = y;
1504:         z_t = lz[n3 / (m*n)];
1505:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1506:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1507:       }

1509:       if (n4 >= 0) { /* middle */
1510:         x_t = x;
1511:         y_t = y;
1512:         z_t = lz[n4 / (m*n)];
1513:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1514:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1515:       }

1517:       if (n5 >= 0) { /* directly right */
1518:         x_t = lx[n5 % m]*dof;
1519:         y_t = y;
1520:         z_t = lz[n5 / (m*n)];
1521:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1522:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1523:       }
1524:     }

1526:     for (i=1; i<=s_y; i++) {
1527:       if (n6 >= 0) { /* left above */
1528:         x_t = lx[n6 % m]*dof;
1529:         y_t = ly[(n6 % (m*n))/m];
1530:         z_t = lz[n6 / (m*n)];
1531:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1532:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1533:       }
1534:       if (n7 >= 0) { /* directly above */
1535:         x_t = x;
1536:         y_t = ly[(n7 % (m*n))/m];
1537:         z_t = lz[n7 / (m*n)];
1538:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1539:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1540:       }
1541:       if (n8 >= 0) { /* right above */
1542:         x_t = lx[n8 % m]*dof;
1543:         y_t = ly[(n8 % (m*n))/m];
1544:         z_t = lz[n8 / (m*n)];
1545:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1546:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1547:       }
1548:     }
1549:   }

1551:   /* Middle Level */
1552:   for (k=0; k<z; k++) {
1553:     for (i=1; i<=s_y; i++) {
1554:       if (n9 >= 0) { /* left below */
1555:         x_t = lx[n9 % m]*dof;
1556:         y_t = ly[(n9 % (m*n))/m];
1557:         /* z_t = z; */
1558:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1559:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1560:       }
1561:       if (n10 >= 0) { /* directly below */
1562:         x_t = x;
1563:         y_t = ly[(n10 % (m*n))/m];
1564:         /* z_t = z; */
1565:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1566:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1567:       }
1568:       if (n11 >= 0) { /* right below */
1569:         x_t = lx[n11 % m]*dof;
1570:         y_t = ly[(n11 % (m*n))/m];
1571:         /* z_t = z; */
1572:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1573:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1574:       }
1575:     }

1577:     for (i=0; i<y; i++) {
1578:       if (n12 >= 0) { /* directly left */
1579:         x_t = lx[n12 % m]*dof;
1580:         y_t = y;
1581:         /* z_t = z; */
1582:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1583:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1584:       }

1586:       /* Interior */
1587:       s_t = bases[rank] + i*x + k*x*y;
1588:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1590:       if (n14 >= 0) { /* directly right */
1591:         x_t = lx[n14 % m]*dof;
1592:         y_t = y;
1593:         /* z_t = z; */
1594:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
1595:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1596:       }
1597:     }

1599:     for (i=1; i<=s_y; i++) {
1600:       if (n15 >= 0) { /* left above */
1601:         x_t = lx[n15 % m]*dof;
1602:         y_t = ly[(n15 % (m*n))/m];
1603:         /* z_t = z; */
1604:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1605:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1606:       }
1607:       if (n16 >= 0) { /* directly above */
1608:         x_t = x;
1609:         y_t = ly[(n16 % (m*n))/m];
1610:         /* z_t = z; */
1611:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1612:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1613:       }
1614:       if (n17 >= 0) { /* right above */
1615:         x_t = lx[n17 % m]*dof;
1616:         y_t = ly[(n17 % (m*n))/m];
1617:         /* z_t = z; */
1618:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1619:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1620:       }
1621:     }
1622:   }
1623: 
1624:   /* Upper Level */
1625:   for (k=0; k<s_z; k++) {
1626:     for (i=1; i<=s_y; i++) {
1627:       if (n18 >= 0) { /* left below */
1628:         x_t = lx[n18 % m]*dof;
1629:         y_t = ly[(n18 % (m*n))/m];
1630:         /* z_t = lz[n18 / (m*n)]; */
1631:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1632:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1633:       }
1634:       if (n19 >= 0) { /* directly below */
1635:         x_t = x;
1636:         y_t = ly[(n19 % (m*n))/m];
1637:         /* z_t = lz[n19 / (m*n)]; */
1638:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1639:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1640:       }
1641:       if (n20 >= 0) { /* right belodof */
1642:         x_t = lx[n20 % m]*dof;
1643:         y_t = ly[(n20 % (m*n))/m];
1644:         /* z_t = lz[n20 / (m*n)]; */
1645:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1646:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1647:       }
1648:     }

1650:     for (i=0; i<y; i++) {
1651:       if (n21 >= 0) { /* directly left */
1652:         x_t = lx[n21 % m]*dof;
1653:         y_t = y;
1654:         /* z_t = lz[n21 / (m*n)]; */
1655:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1656:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1657:       }

1659:       if (n22 >= 0) { /* middle */
1660:         x_t = x;
1661:         y_t = y;
1662:         /* z_t = lz[n22 / (m*n)]; */
1663:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
1664:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1665:       }

1667:       if (n23 >= 0) { /* directly right */
1668:         x_t = lx[n23 % m]*dof;
1669:         y_t = y;
1670:         /* z_t = lz[n23 / (m*n)]; */
1671:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
1672:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1673:       }
1674:     }

1676:     for (i=1; i<=s_y; i++) {
1677:       if (n24 >= 0) { /* left above */
1678:         x_t = lx[n24 % m]*dof;
1679:         y_t = ly[(n24 % (m*n))/m];
1680:         /* z_t = lz[n24 / (m*n)]; */
1681:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1682:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1683:       }
1684:       if (n25 >= 0) { /* directly above */
1685:         x_t = x;
1686:         y_t = ly[(n25 % (m*n))/m];
1687:         /* z_t = lz[n25 / (m*n)]; */
1688:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1689:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1690:       }
1691:       if (n26 >= 0) { /* right above */
1692:         x_t = lx[n26 % m]*dof;
1693:         y_t = ly[(n26 % (m*n))/m];
1694:         /* z_t = lz[n26 / (m*n)]; */
1695:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1696:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1697:       }
1698:     }
1699:   }
1700:   PetscFree(bases);
1701:   da->gtol      = gtol;
1702:   da->ltog      = ltog;
1703:   da->idx       = idx;
1704:   da->Nl        = nn;
1705:   da->base      = base;
1706:   da->ops->view = DAView_3d;
1707:   da->wrap      = wrap;
1708:   *inra = da;

1710:   /* 
1711:      Set the local to global ordering in the global vector, this allows use
1712:      of VecSetValuesLocal().
1713:   */
1714:   ISLocalToGlobalMappingCreateNC(comm,nn,idx,&da->ltogmap);
1715:   ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1716:   PetscLogObjectParent(da,da->ltogmap);

1718:   da->ltol = PETSC_NULL;
1719:   da->ao   = PETSC_NULL;

1721:   if (!flx) {
1722:     PetscMalloc(m*sizeof(PetscInt),&flx);
1723:     PetscMemcpy(flx,lx,m*sizeof(PetscInt));
1724:   }
1725:   if (!fly) {
1726:     PetscMalloc(n*sizeof(PetscInt),&fly);
1727:     PetscMemcpy(fly,ly,n*sizeof(PetscInt));
1728:   }
1729:   if (!flz) {
1730:     PetscMalloc(p*sizeof(PetscInt),&flz);
1731:     PetscMemcpy(flz,lz,p*sizeof(PetscInt));
1732:   }
1733:   da->lx = flx;
1734:   da->ly = fly;
1735:   da->lz = flz;

1737:   DAView_Private(da);
1738:   return(0);
1739: }

1743: /*@C
1744:    DACreate - Creates an object that will manage the communication of regular array data that is distributed across some processors
1745:        in 1, 2 or 3 dimensions

1747:    Collective on MPI_Comm

1749:    See the manual pages for the routines for each dimension.

1751:    Level: beginner

1753:    
1754: .keywords: distributed array, create, three-dimensional

1756: .seealso: DACreate1d(), DACreate2d(), DACreate3d(), DAGetOwnershipRange()

1758: @*/
1759: PetscErrorCode  DACreate(MPI_Comm comm,PetscInt dim,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
1760:          PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,PetscInt *lx,PetscInt *ly,PetscInt *lz,DA *inra)
1761: {

1764:   if (dim == 3) {
1765:     DACreate3d(comm,wrap,stencil_type,M,N,P,m,n,p,dof,s,lx,ly,lz,inra);
1766:   } else if (dim == 2) {
1767:     DACreate2d(comm,wrap,stencil_type,M,N,m,n,dof,s,lx,ly,inra);
1768:   } else if (dim == 1) {
1769:     DACreate1d(comm,wrap,M,dof,s,lx,inra);
1770:   }
1771:   return(0);
1772: }