Actual source code: da3.c

  1: /*$Id: da3.c,v 1.132 2001/03/28 19:42:42 balay Exp bsmith $*/

  3: /*
  4:    Code for manipulating distributed regular 3d arrays in parallel.
  5:    File created by Peter Mell  7/14/95
  6:  */

 8:  #include src/dm/da/daimpl.h

 10: #if defined (PETSC_HAVE_AMS)
 11: EXTERN_C_BEGIN
 12: EXTERN int AMSSetFieldBlock_DA(AMS_Memory,char *,Vec);
 13: EXTERN_C_END
 14: #endif

 16: int DAView_3d(DA da,PetscViewer viewer)
 17: {
 18:   int        rank,ierr;
 19:   PetscTruth isascii,isdraw,isbinary;

 22:   MPI_Comm_rank(da->comm,&rank);

 24:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
 25:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
 26:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_BINARY,&isbinary);
 27:   if (isascii) {
 28:     PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %d N %d P %d m %d n %d p %d w %d s %dn",
 29:                rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
 30:     PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %d %d, Y range of indices: %d %d, Z range of indices: %d %dn",
 31:                da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
 32: #if !defined(PETSC_USE_COMPLEX)
 33:     if (da->coordinates) {
 34:       int    last;
 35:       double *coors;
 36:       VecGetArray(da->coordinates,&coors);
 37:       VecGetLocalSize(da->coordinates,&last);
 38:       last = last - 3;
 39:       PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %g %g %g : Upper right %g %g %gn",
 40:                coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
 41:       VecRestoreArray(da->coordinates,&coors);
 42:     }
 43: #endif
 44:     PetscViewerFlush(viewer);
 45:   } else if (isdraw) {
 46:     PetscDraw       draw;
 47:     double     ymin = -1.0,ymax = (double)da->N;
 48:     double     xmin = -1.0,xmax = (double)((da->M+2)*da->P),x,y,ycoord,xcoord;
 49:     int        k,plane,base,*idx;
 50:     char       node[10];
 51:     PetscTruth isnull;

 53:     PetscViewerDrawGetDraw(viewer,0,&draw);
 54:     PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
 55:     PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
 56:     PetscDrawSynchronizedClear(draw);

 58:     /* first processor draw all node lines */
 59:     if (!rank) {
 60:       for (k=0; k<da->P; k++) {
 61:         ymin = 0.0; ymax = (double)(da->N - 1);
 62:         for (xmin=(double)(k*(da->M+1)); xmin<(double)(da->M+(k*(da->M+1))); xmin++) {
 63:           PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
 64:         }
 65: 
 66:         xmin = (double)(k*(da->M+1)); xmax = xmin + (double)(da->M - 1);
 67:         for (ymin=0; ymin<(double)da->N; ymin++) {
 68:           PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
 69:         }
 70:       }
 71:     }
 72:     PetscDrawSynchronizedFlush(draw);
 73:     PetscDrawPause(draw);

 75:     for (k=0; k<da->P; k++) {  /*Go through and draw for each plane*/
 76:       if ((k >= da->zs) && (k < da->ze)) {
 77:         /* draw my box */
 78:         ymin = da->ys;
 79:         ymax = da->ye - 1;
 80:         xmin = da->xs/da->w    + (da->M+1)*k;
 81:         xmax =(da->xe-1)/da->w + (da->M+1)*k;

 83:         PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
 84:         PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
 85:         PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
 86:         PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);

 88:         xmin = da->xs/da->w;
 89:         xmax =(da->xe-1)/da->w;

 91:         /* put in numbers*/
 92:         base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;

 94:         /* Identify which processor owns the box */
 95:         sprintf(node,"%d",rank);
 96:         PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);

 98:         for (y=ymin; y<=ymax; y++) {
 99:           for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
100:             sprintf(node,"%d",base++);
101:             PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
102:           }
103:         }
104: 
105:       }
106:     }
107:     PetscDrawSynchronizedFlush(draw);
108:     PetscDrawPause(draw);

110:     for (k=0-da->s; k<da->P+da->s; k++) {
111:       /* Go through and draw for each plane */
112:       if ((k >= da->Zs) && (k < da->Ze)) {
113: 
114:         /* overlay ghost numbers, useful for error checking */
115:         base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
116:         plane=k;
117:         /* Keep z wrap around points on the dradrawg */
118:         if (k<0)    { plane=da->P+k; }
119:         if (k>=da->P) { plane=k-da->P; }
120:         ymin = da->Ys; ymax = da->Ye;
121:         xmin = (da->M+1)*plane*da->w;
122:         xmax = (da->M+1)*plane*da->w+da->M*da->w;
123:         for (y=ymin; y<ymax; y++) {
124:           for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
125:             sprintf(node,"%d",idx[base]/da->w);
126:             ycoord = y;
127:             /*Keep y wrap around points on drawing */
128:             if (y<0)      { ycoord = da->N+y; }

130:             if (y>=da->N) { ycoord = y-da->N; }
131:             xcoord = x;   /* Keep x wrap points on drawing */

133:             if (x<xmin)  { xcoord = xmax - (xmin-x); }
134:             if (x>=xmax) { xcoord = xmin + (x-xmax); }
135:             PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
136:             base+=da->w;
137:           }
138:         }
139:       }
140:     }
141:     PetscDrawSynchronizedFlush(draw);
142:     PetscDrawPause(draw);
143:   } else if (isbinary) {
144:     DAView_Binary(da,viewer);
145:   } else {
146:     SETERRQ1(1,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
147:   }
148:   return(0);
149: }

151: EXTERN int DAPublish_Petsc(PetscObject);

153: /*@C
154:    DACreate3d - Creates an object that will manage the communication of three-dimensional 
155:    regular array data that is distributed across some processors.

157:    Collective on MPI_Comm

159:    Input Parameters:
160: +  comm - MPI communicator
161: .  wrap - type of periodicity the array should have, if any.  Use one
162:           of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
163: .  stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
164: .  M,N,P - global dimension in each direction of the array
165: .  m,n,p - corresponding number of processors in each dimension 
166:            (or PETSC_DECIDE to have calculated)
167: .  dof - number of degrees of freedom per node
168: .  lx, ly, lz - arrays containing the number of nodes in each cell along
169:           the x, y, and z coordinates, or PETSC_NULL. If non-null, these
170:           must be of length as m,n,p and the corresponding
171:           m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
172:           the ly[] must n, sum of the lz[] must be P
173: -  s - stencil width

175:    Output Parameter:
176: .  inra - the resulting distributed array object

178:    Options Database Key:
179: +  -da_view - Calls DAView() at the conclusion of DACreate3d()
180: .  -da_grid_x <nx> - number of grid points in x direction
181: .  -da_grid_y <ny> - number of grid points in y direction
182: .  -da_grid_z <nz> - number of grid points in z direction
183: -  -da_noao - do not compute natural to PETSc ordering object

185:    Level: beginner

187:    Notes:
188:    The stencil type DA_STENCIL_STAR with width 1 corresponds to the 
189:    standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
190:    the standard 27-pt stencil.

192:    The array data itself is NOT stored in the DA, it is stored in Vec objects;
193:    The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
194:    and DACreateLocalVector() and calls to VecDuplicate() if more are needed.

196: .keywords: distributed array, create, three-dimensional

198: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(),
199:           DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(),
200:           DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView()

202: @*/
203: int DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,int M,
204:                int N,int P,int m,int n,int p,int dof,int s,int *lx,int *ly,int *lz,DA *inra)
205: {
206:   int           rank,size,ierr,start,end,pm;
207:   int           xs,xe,ys,ye,zs,ze,x,y,z,Xs,Xe,Ys,Ye,Zs,Ze;
208:   int           left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
209:   int           n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
210:   int           n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
211:   int           *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
212:   int           *gA,*gB,*gAall,*gBall,ict,ldim,gdim;
213:   int           sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
214:   int           sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
215:   int           sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
216:   PetscTruth    flg1,flg2;
217:   DA            da;
218:   Vec           local,global;
219:   VecScatter    ltog,gtol;
220:   IS            to,from;

223:   *inra = 0;

225:   if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %d",dof);
226:   if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %d",s);
227:   if (M < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have M positive");
228:   if (N < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have N positive");
229:   if (P < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have P positive");

231:   PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
232:     PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",M,&M,PETSC_NULL);
233:     PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",N,&N,PETSC_NULL);
234:     PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",P,&P,PETSC_NULL);
235:     PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
236:     PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
237:     PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
238:     PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DACreate3d",refine_x,&refine_x,PETSC_NULL);
239:     PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DACreate3d",refine_y,&refine_y,PETSC_NULL);
240:     PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DACreate3d",refine_z,&refine_z,PETSC_NULL);
241:   PetscOptionsEnd();

243:   PetscHeaderCreate(da,_p_DA,struct _DAOps,DA_COOKIE,0,"DA",comm,DADestroy,DAView);
244:   da->bops->publish           = DAPublish_Petsc;
245:   da->ops->createglobalvector = DACreateGlobalVector;
246:   da->ops->getinterpolation   = DAGetInterpolation;
247:   da->ops->getcoloring        = DAGetColoring;
248:   da->ops->refine             = DARefine;

250:   PetscLogObjectCreate(da);
251:   PetscLogObjectMemory(da,sizeof(struct _p_DA));
252:   da->dim        = 3;
253:   da->gtog1      = 0;
254:   da->refine_x   = refine_x;
255:   da->refine_y   = refine_y;
256:   da->refine_z   = refine_z;
257:   PetscMalloc(dof*sizeof(char*),&da->fieldname);
258:   PetscMemzero(da->fieldname,dof*sizeof(char*));

260:   MPI_Comm_size(comm,&size);
261:   MPI_Comm_rank(comm,&rank);

263:   if (m != PETSC_DECIDE) {
264:     if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %d",m);}
265:     else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %d %d",m,size);}
266:   }
267:   if (n != PETSC_DECIDE) {
268:     if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %d",n);}
269:     else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %d %d",n,size);}
270:   }
271:   if (p != PETSC_DECIDE) {
272:     if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %d",p);}
273:     else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %d %d",p,size);}
274:   }

276:   /* Partition the array among the processors */
277:   if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
278:     m = size/(n*p);
279:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
280:     n = size/(m*p);
281:   } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
282:     p = size/(m*n);
283:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
284:     /* try for squarish distribution */
285:     m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)N*p)));
286:     if (!m) m = 1;
287:     while (m > 0) {
288:       n = size/(m*p);
289:       if (m*n*p == size) break;
290:       m--;
291:     }
292:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %d",p);
293:     if (M > N && m < n) {int _m = m; m = n; n = _m;}
294:   } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
295:     /* try for squarish distribution */
296:     m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)P*n)));
297:     if (!m) m = 1;
298:     while (m > 0) {
299:       p = size/(m*n);
300:       if (m*n*p == size) break;
301:       m--;
302:     }
303:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %d",n);
304:     if (M > P && m < p) {int _m = m; m = p; p = _m;}
305:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
306:     /* try for squarish distribution */
307:     n = (int)(0.5 + sqrt(((double)N)*((double)size)/((double)P*m)));
308:     if (!n) n = 1;
309:     while (n > 0) {
310:       p = size/(m*n);
311:       if (m*n*p == size) break;
312:       n--;
313:     }
314:     if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %d",n);
315:     if (N > P && n < p) {int _n = n; n = p; p = _n;}
316:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
317:     /* try for squarish distribution */
318:     n = (int)(0.5 + pow(((double)N*N)*((double)size)/((double)P*M),1./3.));
319:     if (!n) n = 1;
320:     while (n > 0) {
321:       pm = size/n;
322:       if (n*pm == size) break;
323:       n--;
324:     }
325:     if (!n) n = 1;
326:     m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)P*n)));
327:     if (!m) m = 1;
328:     while (m > 0) {
329:       p = size/(m*n);
330:       if (m*n*p == size) break;
331:       m--;
332:     }
333:     if (M > P && m < p) {int _m = m; m = p; p = _m;}
334:   } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");

336:   if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
337:   if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %d %d",M,m);
338:   if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %d %d",N,n);
339:   if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %d %d",P,p);

341:   PetscOptionsHasName(PETSC_NULL,"-da_partition_nodes_at_end",&flg2);
342:   /* 
343:      Determine locally owned region 
344:      [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes 
345:   */
346:   if (lx) { /* user decided distribution */
347:     x  = lx[rank % m];
348:     xs = 0;
349:     for (i=0; i<(rank%m); i++) { xs += lx[i];}
350:     if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
351:   } else if (flg2) {
352:     SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
353:   } else { /* Normal PETSc distribution */
354:     x = M/m + ((M % m) > (rank % m));
355:     if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
356:     if ((M % m) > (rank % m)) { xs = (rank % m)*x; }
357:     else                      { xs = (M % m)*(x+1) + ((rank % m)-(M % m))*x; }
358:     PetscMalloc(m*sizeof(int),&lx);
359:     flx = lx;
360:     for (i=0; i<m; i++) {
361:       lx[i] = M/m + ((M % m) > (i % m));
362:     }
363:   }
364:   if (ly) { /* user decided distribution */
365:     y  = ly[(rank % (m*n))/m];
366:     if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
367:     ys = 0;
368:     for (i=0; i<(rank % (m*n))/m; i++) { ys += ly[i];}
369:   } else if (flg2) {
370:     SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
371:   } else { /* Normal PETSc distribution */
372:     y = N/n + ((N % n) > ((rank % (m*n)) /m));
373:     if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
374:     if ((N % n) > ((rank % (m*n)) /m)) {ys = ((rank % (m*n))/m)*y;}
375:     else                               {ys = (N % n)*(y+1) + (((rank % (m*n))/m)-(N % n))*y;}
376:     PetscMalloc(n*sizeof(int),&ly);
377:     fly = ly;
378:     for (i=0; i<n; i++) {
379:       ly[i] = N/n + ((N % n) > (i % n));
380:     }
381:   }
382:   if (lz) { /* user decided distribution */
383:     z  = lz[rank/(m*n)];
384:     if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
385:     zs = 0;
386:     for (i=0; i<(rank/(m*n)); i++) { zs += lz[i];}
387:   } else if (flg2) {
388:     SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
389:   } else { /* Normal PETSc distribution */
390:     z = P/p + ((P % p) > (rank / (m*n)));
391:     if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
392:     if ((P % p) > (rank / (m*n))) {zs = (rank/(m*n))*z;}
393:     else                          {zs = (P % p)*(z+1) + ((rank/(m*n))-(P % p))*z;}
394:     PetscMalloc(p*sizeof(int),&lz);
395:     flz = lz;
396:     for (i=0; i<p; i++) {
397:       lz[i] = P/p + ((P % p) > (i % p));
398:     }
399:   }
400:   ye = ys + y;
401:   xe = xs + x;
402:   ze = zs + z;

404:   /* determine ghost region */
405:   /* Assume No Periodicity */
406:   if (xs-s > 0) Xs = xs - s; else Xs = 0;
407:   if (ys-s > 0) Ys = ys - s; else Ys = 0;
408:   if (zs-s > 0) Zs = zs - s; else Zs = 0;
409:   if (xe+s <= M) Xe = xe + s; else Xe = M;
410:   if (ye+s <= N) Ye = ye + s; else Ye = N;
411:   if (ze+s <= P) Ze = ze + s; else Ze = P;

413:   /* X Periodic */
414:   if (DAXPeriodic(wrap)){
415:     Xs = xs - s;
416:     Xe = xe + s;
417:   }

419:   /* Y Periodic */
420:   if (DAYPeriodic(wrap)){
421:     Ys = ys - s;
422:     Ye = ye + s;
423:   }

425:   /* Z Periodic */
426:   if (DAZPeriodic(wrap)){
427:     Zs = zs - s;
428:     Ze = ze + s;
429:   }

431:   /* Resize all X parameters to reflect w */
432:   x   *= dof;
433:   xs  *= dof;
434:   xe  *= dof;
435:   Xs  *= dof;
436:   Xe  *= dof;
437:   s_x  = s*dof;
438:   s_y  = s;
439:   s_z  = s;

441:   /* determine starting point of each processor */
442:   nn = x*y*z;
443:   PetscMalloc((2*size+1)*sizeof(int),&bases);
444:   ldims = (int*)(bases+size+1);
445:   MPI_Allgather(&nn,1,MPI_INT,ldims,1,MPI_INT,comm);
446:   bases[0] = 0;
447:   for (i=1; i<=size; i++) {
448:     bases[i] = ldims[i-1];
449:   }
450:   for (i=1; i<=size; i++) {
451:     bases[i] += bases[i-1];
452:   }

454:   /* allocate the base parallel and sequential vectors */
455:   VecCreateMPI(comm,x*y*z,PETSC_DECIDE,&global);
456:   VecSetBlockSize(global,dof);
457:   VecCreateSeq(MPI_COMM_SELF,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),&local);
458:   VecSetBlockSize(local,dof);

460:   /* generate appropriate vector scatters */
461:   /* local to global inserts non-ghost point region into global */
462:   VecGetOwnershipRange(global,&start,&end);
463:   ISCreateStride(comm,x*y*z,start,1,&to);

465:   left   = xs - Xs;
466:   bottom = ys - Ys; top = bottom + y;
467:   down   = zs - Zs; up  = down + z;
468:   count  = x*(top-bottom)*(up-down);
469:   PetscMalloc(count*sizeof(int),&idx);
470:   count  = 0;
471:   for (i=down; i<up; i++) {
472:     for (j=bottom; j<top; j++) {
473:       for (k=0; k<x; k++) {
474:         idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
475:       }
476:     }
477:   }
478:   ISCreateGeneral(comm,count,idx,&from);
479:   PetscFree(idx);

481:   VecScatterCreate(local,from,global,to,&ltog);
482:   PetscLogObjectParent(da,to);
483:   PetscLogObjectParent(da,from);
484:   PetscLogObjectParent(da,ltog);
485:   ISDestroy(from);
486:   ISDestroy(to);

488:   /* global to local must include ghost points */
489:   if (stencil_type == DA_STENCIL_BOX) {
490:     ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
491:   } else {
492:     /* This is way ugly! We need to list the funny cross type region */
493:     /* the bottom chunck */
494:     left   = xs - Xs;
495:     bottom = ys - Ys; top = bottom + y;
496:     down   = zs - Zs;   up  = down + z;
497:     count  = down*(top-bottom)*x +
498:              (up-down)*(bottom*x  + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) +
499:              (Ze-Zs-up)*(top-bottom)*x;
500:     PetscMalloc(count*sizeof(int),&idx);
501:     count  = 0;
502:     for (i=0; i<down; i++) {
503:       for (j=bottom; j<top; j++) {
504:         for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
505:       }
506:     }
507:     /* the middle piece */
508:     for (i=down; i<up; i++) {
509:       /* front */
510:       for (j=0; j<bottom; j++) {
511:         for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
512:       }
513:       /* middle */
514:       for (j=bottom; j<top; j++) {
515:         for (k=0; k<Xe-Xs; k++) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
516:       }
517:       /* back */
518:       for (j=top; j<Ye-Ys; j++) {
519:         for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
520:       }
521:     }
522:     /* the top piece */
523:     for (i=up; i<Ze-Zs; i++) {
524:       for (j=bottom; j<top; j++) {
525:         for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
526:       }
527:     }
528:     ISCreateGeneral(comm,count,idx,&to);
529:     PetscFree(idx);
530:   }

532:   /* determine who lies on each side of use stored in    n24 n25 n26
533:                                                          n21 n22 n23
534:                                                          n18 n19 n20

536:                                                          n15 n16 n17
537:                                                          n12     n14
538:                                                          n9  n10 n11

540:                                                          n6  n7  n8
541:                                                          n3  n4  n5
542:                                                          n0  n1  n2
543:   */
544: 
545:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
546: 
547:   /* Assume Nodes are Internal to the Cube */
548: 
549:   n0  = rank - m*n - m - 1;
550:   n1  = rank - m*n - m;
551:   n2  = rank - m*n - m + 1;
552:   n3  = rank - m*n -1;
553:   n4  = rank - m*n;
554:   n5  = rank - m*n + 1;
555:   n6  = rank - m*n + m - 1;
556:   n7  = rank - m*n + m;
557:   n8  = rank - m*n + m + 1;

559:   n9  = rank - m - 1;
560:   n10 = rank - m;
561:   n11 = rank - m + 1;
562:   n12 = rank - 1;
563:   n14 = rank + 1;
564:   n15 = rank + m - 1;
565:   n16 = rank + m;
566:   n17 = rank + m + 1;

568:   n18 = rank + m*n - m - 1;
569:   n19 = rank + m*n - m;
570:   n20 = rank + m*n - m + 1;
571:   n21 = rank + m*n - 1;
572:   n22 = rank + m*n;
573:   n23 = rank + m*n + 1;
574:   n24 = rank + m*n + m - 1;
575:   n25 = rank + m*n + m;
576:   n26 = rank + m*n + m + 1;

578:   /* Assume Pieces are on Faces of Cube */

580:   if (xs == 0) { /* First assume not corner or edge */
581:     n0  = rank       -1 - (m*n);
582:     n3  = rank + m   -1 - (m*n);
583:     n6  = rank + 2*m -1 - (m*n);
584:     n9  = rank       -1;
585:     n12 = rank + m   -1;
586:     n15 = rank + 2*m -1;
587:     n18 = rank       -1 + (m*n);
588:     n21 = rank + m   -1 + (m*n);
589:     n24 = rank + 2*m -1 + (m*n);
590:    }

592:   if (xe == M*dof) { /* First assume not corner or edge */
593:     n2  = rank -2*m +1 - (m*n);
594:     n5  = rank - m  +1 - (m*n);
595:     n8  = rank      +1 - (m*n);
596:     n11 = rank -2*m +1;
597:     n14 = rank - m  +1;
598:     n17 = rank      +1;
599:     n20 = rank -2*m +1 + (m*n);
600:     n23 = rank - m  +1 + (m*n);
601:     n26 = rank      +1 + (m*n);
602:   }

604:   if (ys==0) { /* First assume not corner or edge */
605:     n0  = rank + m * (n-1) -1 - (m*n);
606:     n1  = rank + m * (n-1)    - (m*n);
607:     n2  = rank + m * (n-1) +1 - (m*n);
608:     n9  = rank + m * (n-1) -1;
609:     n10 = rank + m * (n-1);
610:     n11 = rank + m * (n-1) +1;
611:     n18 = rank + m * (n-1) -1 + (m*n);
612:     n19 = rank + m * (n-1)    + (m*n);
613:     n20 = rank + m * (n-1) +1 + (m*n);
614:   }

616:   if (ye == N) { /* First assume not corner or edge */
617:     n6  = rank - m * (n-1) -1 - (m*n);
618:     n7  = rank - m * (n-1)    - (m*n);
619:     n8  = rank - m * (n-1) +1 - (m*n);
620:     n15 = rank - m * (n-1) -1;
621:     n16 = rank - m * (n-1);
622:     n17 = rank - m * (n-1) +1;
623:     n24 = rank - m * (n-1) -1 + (m*n);
624:     n25 = rank - m * (n-1)    + (m*n);
625:     n26 = rank - m * (n-1) +1 + (m*n);
626:   }
627: 
628:   if (zs == 0) { /* First assume not corner or edge */
629:     n0 = size - (m*n) + rank - m - 1;
630:     n1 = size - (m*n) + rank - m;
631:     n2 = size - (m*n) + rank - m + 1;
632:     n3 = size - (m*n) + rank - 1;
633:     n4 = size - (m*n) + rank;
634:     n5 = size - (m*n) + rank + 1;
635:     n6 = size - (m*n) + rank + m - 1;
636:     n7 = size - (m*n) + rank + m ;
637:     n8 = size - (m*n) + rank + m + 1;
638:   }

640:   if (ze == P) { /* First assume not corner or edge */
641:     n18 = (m*n) - (size-rank) - m - 1;
642:     n19 = (m*n) - (size-rank) - m;
643:     n20 = (m*n) - (size-rank) - m + 1;
644:     n21 = (m*n) - (size-rank) - 1;
645:     n22 = (m*n) - (size-rank);
646:     n23 = (m*n) - (size-rank) + 1;
647:     n24 = (m*n) - (size-rank) + m - 1;
648:     n25 = (m*n) - (size-rank) + m;
649:     n26 = (m*n) - (size-rank) + m + 1;
650:   }

652:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
653:     n0 = size - m*n + rank + m-1 - m;
654:     n3 = size - m*n + rank + m-1;
655:     n6 = size - m*n + rank + m-1 + m;
656:   }
657: 
658:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
659:     n18 = m*n - (size - rank) + m-1 - m;
660:     n21 = m*n - (size - rank) + m-1;
661:     n24 = m*n - (size - rank) + m-1 + m;
662:   }

664:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
665:     n0  = rank + m*n -1 - m*n;
666:     n9  = rank + m*n -1;
667:     n18 = rank + m*n -1 + m*n;
668:   }

670:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
671:     n6  = rank - m*(n-1) + m-1 - m*n;
672:     n15 = rank - m*(n-1) + m-1;
673:     n24 = rank - m*(n-1) + m-1 + m*n;
674:   }

676:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
677:     n2 = size - (m*n-rank) - (m-1) - m;
678:     n5 = size - (m*n-rank) - (m-1);
679:     n8 = size - (m*n-rank) - (m-1) + m;
680:   }

682:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
683:     n20 = m*n - (size - rank) - (m-1) - m;
684:     n23 = m*n - (size - rank) - (m-1);
685:     n26 = m*n - (size - rank) - (m-1) + m;
686:   }

688:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
689:     n2  = rank + m*(n-1) - (m-1) - m*n;
690:     n11 = rank + m*(n-1) - (m-1);
691:     n20 = rank + m*(n-1) - (m-1) + m*n;
692:   }

694:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
695:     n8  = rank - m*n +1 - m*n;
696:     n17 = rank - m*n +1;
697:     n26 = rank - m*n +1 + m*n;
698:   }

700:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
701:     n0 = size - m + rank -1;
702:     n1 = size - m + rank;
703:     n2 = size - m + rank +1;
704:   }

706:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
707:     n18 = m*n - (size - rank) + m*(n-1) -1;
708:     n19 = m*n - (size - rank) + m*(n-1);
709:     n20 = m*n - (size - rank) + m*(n-1) +1;
710:   }

712:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
713:     n6 = size - (m*n-rank) - m * (n-1) -1;
714:     n7 = size - (m*n-rank) - m * (n-1);
715:     n8 = size - (m*n-rank) - m * (n-1) +1;
716:   }

718:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
719:     n24 = rank - (size-m) -1;
720:     n25 = rank - (size-m);
721:     n26 = rank - (size-m) +1;
722:   }

724:   /* Check for Corners */
725:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
726:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
727:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
728:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
729:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
730:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
731:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
732:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

734:   /* Check for when not X,Y, and Z Periodic */

736:   /* If not X periodic */
737:   if ((wrap != DA_XPERIODIC)  && (wrap != DA_XYPERIODIC) &&
738:      (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
739:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
740:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
741:   }

743:   /* If not Y periodic */
744:   if ((wrap != DA_YPERIODIC)  && (wrap != DA_XYPERIODIC) &&
745:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
746:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
747:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
748:   }

750:   /* If not Z periodic */
751:   if ((wrap != DA_ZPERIODIC)  && (wrap != DA_XZPERIODIC) &&
752:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
753:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
754:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
755:   }

757:   /* If star stencil then delete the corner neighbors */
758:   if (stencil_type == DA_STENCIL_STAR) {
759:      /* save information about corner neighbors */
760:      sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
761:      sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
762:      sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
763:      sn26 = n26;
764:      n0  = n1  = n2  = n3  = n5  = n6  = n7  = n8  = n9  = n11 =
765:      n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
766:   }


769:   PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int),&idx);
770:   PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int));

772:   nn = 0;

774:   /* Bottom Level */
775:   for (k=0; k<s_z; k++) {
776:     for (i=1; i<=s_y; i++) {
777:       if (n0 >= 0) { /* left below */
778:         x_t = lx[n0 % m]*dof;
779:         y_t = ly[(n0 % (m*n))/m];
780:         z_t = lz[n0 / (m*n)];
781:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
782:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
783:       }
784:       if (n1 >= 0) { /* directly below */
785:         x_t = x;
786:         y_t = ly[(n1 % (m*n))/m];
787:         z_t = lz[n1 / (m*n)];
788:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
789:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
790:       }
791:       if (n2 >= 0) { /* right below */
792:         x_t = lx[n2 % m]*dof;
793:         y_t = ly[(n2 % (m*n))/m];
794:         z_t = lz[n2 / (m*n)];
795:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
796:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
797:       }
798:     }

800:     for (i=0; i<y; i++) {
801:       if (n3 >= 0) { /* directly left */
802:         x_t = lx[n3 % m]*dof;
803:         y_t = y;
804:         z_t = lz[n3 / (m*n)];
805:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
806:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
807:       }

809:       if (n4 >= 0) { /* middle */
810:         x_t = x;
811:         y_t = y;
812:         z_t = lz[n4 / (m*n)];
813:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
814:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
815:       }

817:       if (n5 >= 0) { /* directly right */
818:         x_t = lx[n5 % m]*dof;
819:         y_t = y;
820:         z_t = lz[n5 / (m*n)];
821:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
822:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
823:       }
824:     }

826:     for (i=1; i<=s_y; i++) {
827:       if (n6 >= 0) { /* left above */
828:         x_t = lx[n6 % m]*dof;
829:         y_t = ly[(n6 % (m*n))/m];
830:         z_t = lz[n6 / (m*n)];
831:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
832:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
833:       }
834:       if (n7 >= 0) { /* directly above */
835:         x_t = x;
836:         y_t = ly[(n7 % (m*n))/m];
837:         z_t = lz[n7 / (m*n)];
838:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
839:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
840:       }
841:       if (n8 >= 0) { /* right above */
842:         x_t = lx[n8 % m]*dof;
843:         y_t = ly[(n8 % (m*n))/m];
844:         z_t = lz[n8 / (m*n)];
845:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
846:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
847:       }
848:     }
849:   }

851:   /* Middle Level */
852:   for (k=0; k<z; k++) {
853:     for (i=1; i<=s_y; i++) {
854:       if (n9 >= 0) { /* left below */
855:         x_t = lx[n9 % m]*dof;
856:         y_t = ly[(n9 % (m*n))/m];
857:         /* z_t = z; */
858:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
859:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
860:       }
861:       if (n10 >= 0) { /* directly below */
862:         x_t = x;
863:         y_t = ly[(n10 % (m*n))/m];
864:         /* z_t = z; */
865:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
866:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
867:       }
868:       if (n11 >= 0) { /* right below */
869:         x_t = lx[n11 % m]*dof;
870:         y_t = ly[(n11 % (m*n))/m];
871:         /* z_t = z; */
872:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
873:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
874:       }
875:     }

877:     for (i=0; i<y; i++) {
878:       if (n12 >= 0) { /* directly left */
879:         x_t = lx[n12 % m]*dof;
880:         y_t = y;
881:         /* z_t = z; */
882:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
883:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
884:       }

886:       /* Interior */
887:       s_t = bases[rank] + i*x + k*x*y;
888:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

890:       if (n14 >= 0) { /* directly right */
891:         x_t = lx[n14 % m]*dof;
892:         y_t = y;
893:         /* z_t = z; */
894:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
895:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
896:       }
897:     }

899:     for (i=1; i<=s_y; i++) {
900:       if (n15 >= 0) { /* left above */
901:         x_t = lx[n15 % m]*dof;
902:         y_t = ly[(n15 % (m*n))/m];
903:         /* z_t = z; */
904:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
905:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
906:       }
907:       if (n16 >= 0) { /* directly above */
908:         x_t = x;
909:         y_t = ly[(n16 % (m*n))/m];
910:         /* z_t = z; */
911:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
912:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
913:       }
914:       if (n17 >= 0) { /* right above */
915:         x_t = lx[n17 % m]*dof;
916:         y_t = ly[(n17 % (m*n))/m];
917:         /* z_t = z; */
918:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
919:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
920:       }
921:     }
922:   }
923: 
924:   /* Upper Level */
925:   for (k=0; k<s_z; k++) {
926:     for (i=1; i<=s_y; i++) {
927:       if (n18 >= 0) { /* left below */
928:         x_t = lx[n18 % m]*dof;
929:         y_t = ly[(n18 % (m*n))/m];
930:         /* z_t = lz[n18 / (m*n)]; */
931:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
932:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
933:       }
934:       if (n19 >= 0) { /* directly below */
935:         x_t = x;
936:         y_t = ly[(n19 % (m*n))/m];
937:         /* z_t = lz[n19 / (m*n)]; */
938:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
939:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
940:       }
941:       if (n20 >= 0) { /* right below */
942:         x_t = lx[n20 % m]*dof;
943:         y_t = ly[(n20 % (m*n))/m];
944:         /* z_t = lz[n20 / (m*n)]; */
945:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
946:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
947:       }
948:     }

950:     for (i=0; i<y; i++) {
951:       if (n21 >= 0) { /* directly left */
952:         x_t = lx[n21 % m]*dof;
953:         y_t = y;
954:         /* z_t = lz[n21 / (m*n)]; */
955:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
956:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
957:       }

959:       if (n22 >= 0) { /* middle */
960:         x_t = x;
961:         y_t = y;
962:         /* z_t = lz[n22 / (m*n)]; */
963:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
964:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
965:       }

967:       if (n23 >= 0) { /* directly right */
968:         x_t = lx[n23 % m]*dof;
969:         y_t = y;
970:         /* z_t = lz[n23 / (m*n)]; */
971:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
972:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
973:       }
974:     }

976:     for (i=1; i<=s_y; i++) {
977:       if (n24 >= 0) { /* left above */
978:         x_t = lx[n24 % m]*dof;
979:         y_t = ly[(n24 % (m*n))/m];
980:         /* z_t = lz[n24 / (m*n)]; */
981:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
982:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
983:       }
984:       if (n25 >= 0) { /* directly above */
985:         x_t = x;
986:         y_t = ly[(n25 % (m*n))/m];
987:         /* z_t = lz[n25 / (m*n)]; */
988:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
989:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
990:       }
991:       if (n26 >= 0) { /* right above */
992:         x_t = lx[n26 % m]*dof;
993:         y_t = ly[(n26 % (m*n))/m];
994:         /* z_t = lz[n26 / (m*n)]; */
995:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
996:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
997:       }
998:     }
999:   }
1000:   base = bases[rank];
1001:   ISCreateGeneral(comm,nn,idx,&from);
1002:   VecScatterCreate(global,from,local,to,&gtol);
1003:   PetscLogObjectParent(da,gtol);
1004:   PetscLogObjectParent(da,to);
1005:   PetscLogObjectParent(da,from);
1006:   ISDestroy(to);
1007:   ISDestroy(from);
1008:   da->stencil_type = stencil_type;
1009:   da->M  = M;  da->N  = N; da->P = P;
1010:   da->m  = m;  da->n  = n; da->p = p;
1011:   da->w  = dof;  da->s  = s;
1012:   da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1013:   da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;

1015:   PetscLogObjectParent(da,global);
1016:   PetscLogObjectParent(da,local);

1018:   if (stencil_type == DA_STENCIL_STAR) {
1019:     /*
1020:         Recompute the local to global mappings, this time keeping the 
1021:       information about the cross corner processor numbers.
1022:     */
1023:     n0  = sn0;  n1  = sn1;  n2  = sn2;  n3  = sn3;  n5  = sn5;  n6  = sn6; n7 = sn7;
1024:     n8  = sn8;  n9  = sn9;  n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1025:     n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1026:     n26 = sn26;

1028:     nn = 0;

1030:     /* Bottom Level */
1031:     for (k=0; k<s_z; k++) {
1032:       for (i=1; i<=s_y; i++) {
1033:         if (n0 >= 0) { /* left below */
1034:           x_t = lx[n0 % m]*dof;
1035:           y_t = ly[(n0 % (m*n))/m];
1036:           z_t = lz[n0 / (m*n)];
1037:           s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1038:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1039:         }
1040:         if (n1 >= 0) { /* directly below */
1041:           x_t = x;
1042:           y_t = ly[(n1 % (m*n))/m];
1043:           z_t = lz[n1 / (m*n)];
1044:           s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1045:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1046:         }
1047:         if (n2 >= 0) { /* right below */
1048:           x_t = lx[n2 % m]*dof;
1049:           y_t = ly[(n2 % (m*n))/m];
1050:           z_t = lz[n2 / (m*n)];
1051:           s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1052:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1053:         }
1054:       }

1056:       for (i=0; i<y; i++) {
1057:         if (n3 >= 0) { /* directly left */
1058:           x_t = lx[n3 % m]*dof;
1059:           y_t = y;
1060:           z_t = lz[n3 / (m*n)];
1061:           s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1062:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1063:         }

1065:         if (n4 >= 0) { /* middle */
1066:           x_t = x;
1067:           y_t = y;
1068:           z_t = lz[n4 / (m*n)];
1069:           s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1070:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1071:         }

1073:         if (n5 >= 0) { /* directly right */
1074:           x_t = lx[n5 % m]*dof;
1075:           y_t = y;
1076:           z_t = lz[n5 / (m*n)];
1077:           s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1078:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1079:         }
1080:       }

1082:       for (i=1; i<=s_y; i++) {
1083:         if (n6 >= 0) { /* left above */
1084:           x_t = lx[n6 % m]*dof;
1085:           y_t = ly[(n6 % (m*n))/m];
1086:           z_t = lz[n6 / (m*n)];
1087:           s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1088:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1089:         }
1090:         if (n7 >= 0) { /* directly above */
1091:           x_t = x;
1092:           y_t = ly[(n7 % (m*n))/m];
1093:           z_t = lz[n7 / (m*n)];
1094:           s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1095:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1096:         }
1097:         if (n8 >= 0) { /* right above */
1098:           x_t = lx[n8 % m]*dof;
1099:           y_t = ly[(n8 % (m*n))/m];
1100:           z_t = lz[n8 / (m*n)];
1101:           s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1102:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1103:         }
1104:       }
1105:     }

1107:     /* Middle Level */
1108:     for (k=0; k<z; k++) {
1109:       for (i=1; i<=s_y; i++) {
1110:         if (n9 >= 0) { /* left below */
1111:           x_t = lx[n9 % m]*dof;
1112:           y_t = ly[(n9 % (m*n))/m];
1113:           /* z_t = z; */
1114:           s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1115:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1116:         }
1117:         if (n10 >= 0) { /* directly below */
1118:           x_t = x;
1119:           y_t = ly[(n10 % (m*n))/m];
1120:           /* z_t = z; */
1121:           s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1122:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1123:         }
1124:         if (n11 >= 0) { /* right below */
1125:           x_t = lx[n11 % m]*dof;
1126:           y_t = ly[(n11 % (m*n))/m];
1127:           /* z_t = z; */
1128:           s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1129:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1130:         }
1131:       }

1133:       for (i=0; i<y; i++) {
1134:         if (n12 >= 0) { /* directly left */
1135:           x_t = lx[n12 % m]*dof;
1136:           y_t = y;
1137:           /* z_t = z; */
1138:           s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1139:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1140:         }

1142:         /* Interior */
1143:         s_t = bases[rank] + i*x + k*x*y;
1144:         for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1146:         if (n14 >= 0) { /* directly right */
1147:           x_t = lx[n14 % m]*dof;
1148:           y_t = y;
1149:           /* z_t = z; */
1150:           s_t = bases[n14] + i*x_t + k*x_t*y_t;
1151:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1152:         }
1153:       }

1155:       for (i=1; i<=s_y; i++) {
1156:         if (n15 >= 0) { /* left above */
1157:           x_t = lx[n15 % m]*dof;
1158:           y_t = ly[(n15 % (m*n))/m];
1159:           /* z_t = z; */
1160:           s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1161:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1162:         }
1163:         if (n16 >= 0) { /* directly above */
1164:           x_t = x;
1165:           y_t = ly[(n16 % (m*n))/m];
1166:           /* z_t = z; */
1167:           s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1168:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1169:         }
1170:         if (n17 >= 0) { /* right above */
1171:           x_t = lx[n17 % m]*dof;
1172:           y_t = ly[(n17 % (m*n))/m];
1173:           /* z_t = z; */
1174:           s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1175:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1176:         }
1177:       }
1178:     }
1179: 
1180:     /* Upper Level */
1181:     for (k=0; k<s_z; k++) {
1182:       for (i=1; i<=s_y; i++) {
1183:         if (n18 >= 0) { /* left below */
1184:           x_t = lx[n18 % m]*dof;
1185:           y_t = ly[(n18 % (m*n))/m];
1186:           /* z_t = lz[n18 / (m*n)]; */
1187:           s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1188:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1189:         }
1190:         if (n19 >= 0) { /* directly below */
1191:           x_t = x;
1192:           y_t = ly[(n19 % (m*n))/m];
1193:           /* z_t = lz[n19 / (m*n)]; */
1194:           s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1195:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1196:         }
1197:         if (n20 >= 0) { /* right below */
1198:           x_t = lx[n20 % m]*dof;
1199:           y_t = ly[(n20 % (m*n))/m];
1200:           /* z_t = lz[n20 / (m*n)]; */
1201:           s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1202:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1203:         }
1204:       }

1206:       for (i=0; i<y; i++) {
1207:         if (n21 >= 0) { /* directly left */
1208:           x_t = lx[n21 % m]*dof;
1209:           y_t = y;
1210:           /* z_t = lz[n21 / (m*n)]; */
1211:           s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1212:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1213:         }

1215:         if (n22 >= 0) { /* middle */
1216:           x_t = x;
1217:           y_t = y;
1218:           /* z_t = lz[n22 / (m*n)]; */
1219:           s_t = bases[n22] + i*x_t + k*x_t*y_t;
1220:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1221:         }

1223:         if (n23 >= 0) { /* directly right */
1224:           x_t = lx[n23 % m]*dof;
1225:           y_t = y;
1226:           /* z_t = lz[n23 / (m*n)]; */
1227:           s_t = bases[n23] + i*x_t + k*x_t*y_t;
1228:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1229:         }
1230:       }

1232:       for (i=1; i<=s_y; i++) {
1233:         if (n24 >= 0) { /* left above */
1234:           x_t = lx[n24 % m]*dof;
1235:           y_t = ly[(n24 % (m*n))/m];
1236:           /* z_t = lz[n24 / (m*n)]; */
1237:           s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1238:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1239:         }
1240:         if (n25 >= 0) { /* directly above */
1241:           x_t = x;
1242:           y_t = ly[(n25 % (m*n))/m];
1243:           /* z_t = lz[n25 / (m*n)]; */
1244:           s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1245:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1246:         }
1247:         if (n26 >= 0) { /* right above */
1248:           x_t = lx[n26 % m]*dof;
1249:           y_t = ly[(n26 % (m*n))/m];
1250:           /* z_t = lz[n26 / (m*n)]; */
1251:           s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1252:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1253:         }
1254:       }
1255:     }
1256:   }
1257:   da->global    = global;
1258:   da->local     = local;
1259:   da->gtol      = gtol;
1260:   da->ltog      = ltog;
1261:   da->idx       = idx;
1262:   da->Nl        = nn;
1263:   da->base      = base;
1264:   da->ops->view = DAView_3d;
1265:   da->wrap      = wrap;
1266:   *inra = da;

1268:   /* 
1269:      Set the local to global ordering in the global vector, this allows use
1270:      of VecSetValuesLocal().
1271:   */
1272:   ierr  = ISLocalToGlobalMappingCreate(comm,nn,idx,&da->ltogmap);
1273:   ierr  = VecSetLocalToGlobalMapping(da->global,da->ltogmap);
1274:   ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1275:   VecSetLocalToGlobalMappingBlock(da->global,da->ltogmapb);
1276:   PetscLogObjectParent(da,da->ltogmap);

1278:   /* redo idx to include "missing" ghost points */
1279:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1280: 
1281:   /* Assume Nodes are Internal to the Cube */
1282: 
1283:   n0  = rank - m*n - m - 1;
1284:   n1  = rank - m*n - m;
1285:   n2  = rank - m*n - m + 1;
1286:   n3  = rank - m*n -1;
1287:   n4  = rank - m*n;
1288:   n5  = rank - m*n + 1;
1289:   n6  = rank - m*n + m - 1;
1290:   n7  = rank - m*n + m;
1291:   n8  = rank - m*n + m + 1;

1293:   n9  = rank - m - 1;
1294:   n10 = rank - m;
1295:   n11 = rank - m + 1;
1296:   n12 = rank - 1;
1297:   n14 = rank + 1;
1298:   n15 = rank + m - 1;
1299:   n16 = rank + m;
1300:   n17 = rank + m + 1;

1302:   n18 = rank + m*n - m - 1;
1303:   n19 = rank + m*n - m;
1304:   n20 = rank + m*n - m + 1;
1305:   n21 = rank + m*n - 1;
1306:   n22 = rank + m*n;
1307:   n23 = rank + m*n + 1;
1308:   n24 = rank + m*n + m - 1;
1309:   n25 = rank + m*n + m;
1310:   n26 = rank + m*n + m + 1;

1312:   /* Assume Pieces are on Faces of Cube */

1314:   if (xs == 0) { /* First assume not corner or edge */
1315:     n0  = rank       -1 - (m*n);
1316:     n3  = rank + m   -1 - (m*n);
1317:     n6  = rank + 2*m -1 - (m*n);
1318:     n9  = rank       -1;
1319:     n12 = rank + m   -1;
1320:     n15 = rank + 2*m -1;
1321:     n18 = rank       -1 + (m*n);
1322:     n21 = rank + m   -1 + (m*n);
1323:     n24 = rank + 2*m -1 + (m*n);
1324:    }

1326:   if (xe == M*dof) { /* First assume not corner or edge */
1327:     n2  = rank -2*m +1 - (m*n);
1328:     n5  = rank - m  +1 - (m*n);
1329:     n8  = rank      +1 - (m*n);
1330:     n11 = rank -2*m +1;
1331:     n14 = rank - m  +1;
1332:     n17 = rank      +1;
1333:     n20 = rank -2*m +1 + (m*n);
1334:     n23 = rank - m  +1 + (m*n);
1335:     n26 = rank      +1 + (m*n);
1336:   }

1338:   if (ys==0) { /* First assume not corner or edge */
1339:     n0  = rank + m * (n-1) -1 - (m*n);
1340:     n1  = rank + m * (n-1)    - (m*n);
1341:     n2  = rank + m * (n-1) +1 - (m*n);
1342:     n9  = rank + m * (n-1) -1;
1343:     n10 = rank + m * (n-1);
1344:     n11 = rank + m * (n-1) +1;
1345:     n18 = rank + m * (n-1) -1 + (m*n);
1346:     n19 = rank + m * (n-1)    + (m*n);
1347:     n20 = rank + m * (n-1) +1 + (m*n);
1348:   }

1350:   if (ye == N) { /* First assume not corner or edge */
1351:     n6  = rank - m * (n-1) -1 - (m*n);
1352:     n7  = rank - m * (n-1)    - (m*n);
1353:     n8  = rank - m * (n-1) +1 - (m*n);
1354:     n15 = rank - m * (n-1) -1;
1355:     n16 = rank - m * (n-1);
1356:     n17 = rank - m * (n-1) +1;
1357:     n24 = rank - m * (n-1) -1 + (m*n);
1358:     n25 = rank - m * (n-1)    + (m*n);
1359:     n26 = rank - m * (n-1) +1 + (m*n);
1360:   }
1361: 
1362:   if (zs == 0) { /* First assume not corner or edge */
1363:     n0 = size - (m*n) + rank - m - 1;
1364:     n1 = size - (m*n) + rank - m;
1365:     n2 = size - (m*n) + rank - m + 1;
1366:     n3 = size - (m*n) + rank - 1;
1367:     n4 = size - (m*n) + rank;
1368:     n5 = size - (m*n) + rank + 1;
1369:     n6 = size - (m*n) + rank + m - 1;
1370:     n7 = size - (m*n) + rank + m ;
1371:     n8 = size - (m*n) + rank + m + 1;
1372:   }

1374:   if (ze == P) { /* First assume not corner or edge */
1375:     n18 = (m*n) - (size-rank) - m - 1;
1376:     n19 = (m*n) - (size-rank) - m;
1377:     n20 = (m*n) - (size-rank) - m + 1;
1378:     n21 = (m*n) - (size-rank) - 1;
1379:     n22 = (m*n) - (size-rank);
1380:     n23 = (m*n) - (size-rank) + 1;
1381:     n24 = (m*n) - (size-rank) + m - 1;
1382:     n25 = (m*n) - (size-rank) + m;
1383:     n26 = (m*n) - (size-rank) + m + 1;
1384:   }

1386:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1387:     n0 = size - m*n + rank + m-1 - m;
1388:     n3 = size - m*n + rank + m-1;
1389:     n6 = size - m*n + rank + m-1 + m;
1390:   }
1391: 
1392:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1393:     n18 = m*n - (size - rank) + m-1 - m;
1394:     n21 = m*n - (size - rank) + m-1;
1395:     n24 = m*n - (size - rank) + m-1 + m;
1396:   }

1398:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1399:     n0  = rank + m*n -1 - m*n;
1400:     n9  = rank + m*n -1;
1401:     n18 = rank + m*n -1 + m*n;
1402:   }

1404:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1405:     n6  = rank - m*(n-1) + m-1 - m*n;
1406:     n15 = rank - m*(n-1) + m-1;
1407:     n24 = rank - m*(n-1) + m-1 + m*n;
1408:   }

1410:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1411:     n2 = size - (m*n-rank) - (m-1) - m;
1412:     n5 = size - (m*n-rank) - (m-1);
1413:     n8 = size - (m*n-rank) - (m-1) + m;
1414:   }

1416:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1417:     n20 = m*n - (size - rank) - (m-1) - m;
1418:     n23 = m*n - (size - rank) - (m-1);
1419:     n26 = m*n - (size - rank) - (m-1) + m;
1420:   }

1422:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1423:     n2  = rank + m*(n-1) - (m-1) - m*n;
1424:     n11 = rank + m*(n-1) - (m-1);
1425:     n20 = rank + m*(n-1) - (m-1) + m*n;
1426:   }

1428:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1429:     n8  = rank - m*n +1 - m*n;
1430:     n17 = rank - m*n +1;
1431:     n26 = rank - m*n +1 + m*n;
1432:   }

1434:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1435:     n0 = size - m + rank -1;
1436:     n1 = size - m + rank;
1437:     n2 = size - m + rank +1;
1438:   }

1440:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1441:     n18 = m*n - (size - rank) + m*(n-1) -1;
1442:     n19 = m*n - (size - rank) + m*(n-1);
1443:     n20 = m*n - (size - rank) + m*(n-1) +1;
1444:   }

1446:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1447:     n6 = size - (m*n-rank) - m * (n-1) -1;
1448:     n7 = size - (m*n-rank) - m * (n-1);
1449:     n8 = size - (m*n-rank) - m * (n-1) +1;
1450:   }

1452:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1453:     n24 = rank - (size-m) -1;
1454:     n25 = rank - (size-m);
1455:     n26 = rank - (size-m) +1;
1456:   }

1458:   /* Check for Corners */
1459:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
1460:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
1461:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
1462:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
1463:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
1464:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1465:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
1466:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

1468:   /* Check for when not X,Y, and Z Periodic */

1470:   /* If not X periodic */
1471:   if (!DAXPeriodic(wrap)){
1472:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
1473:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1474:   }

1476:   /* If not Y periodic */
1477:   if (!DAYPeriodic(wrap)){
1478:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
1479:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1480:   }

1482:   /* If not Z periodic */
1483:   if (!DAZPeriodic(wrap)){
1484:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
1485:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1486:   }

1488:   nn = 0;

1490:   /* Bottom Level */
1491:   for (k=0; k<s_z; k++) {
1492:     for (i=1; i<=s_y; i++) {
1493:       if (n0 >= 0) { /* left below */
1494:         x_t = lx[n0 % m]*dof;
1495:         y_t = ly[(n0 % (m*n))/m];
1496:         z_t = lz[n0 / (m*n)];
1497:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1498:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1499:       }
1500:       if (n1 >= 0) { /* directly below */
1501:         x_t = x;
1502:         y_t = ly[(n1 % (m*n))/m];
1503:         z_t = lz[n1 / (m*n)];
1504:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1505:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1506:       }
1507:       if (n2 >= 0) { /* right below */
1508:         x_t = lx[n2 % m]*dof;
1509:         y_t = ly[(n2 % (m*n))/m];
1510:         z_t = lz[n2 / (m*n)];
1511:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1512:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1513:       }
1514:     }

1516:     for (i=0; i<y; i++) {
1517:       if (n3 >= 0) { /* directly left */
1518:         x_t = lx[n3 % m]*dof;
1519:         y_t = y;
1520:         z_t = lz[n3 / (m*n)];
1521:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1522:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1523:       }

1525:       if (n4 >= 0) { /* middle */
1526:         x_t = x;
1527:         y_t = y;
1528:         z_t = lz[n4 / (m*n)];
1529:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1530:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1531:       }

1533:       if (n5 >= 0) { /* directly right */
1534:         x_t = lx[n5 % m]*dof;
1535:         y_t = y;
1536:         z_t = lz[n5 / (m*n)];
1537:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1538:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1539:       }
1540:     }

1542:     for (i=1; i<=s_y; i++) {
1543:       if (n6 >= 0) { /* left above */
1544:         x_t = lx[n6 % m]*dof;
1545:         y_t = ly[(n6 % (m*n))/m];
1546:         z_t = lz[n6 / (m*n)];
1547:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1548:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1549:       }
1550:       if (n7 >= 0) { /* directly above */
1551:         x_t = x;
1552:         y_t = ly[(n7 % (m*n))/m];
1553:         z_t = lz[n7 / (m*n)];
1554:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1555:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1556:       }
1557:       if (n8 >= 0) { /* right above */
1558:         x_t = lx[n8 % m]*dof;
1559:         y_t = ly[(n8 % (m*n))/m];
1560:         z_t = lz[n8 / (m*n)];
1561:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1562:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1563:       }
1564:     }
1565:   }

1567:   /* Middle Level */
1568:   for (k=0; k<z; k++) {
1569:     for (i=1; i<=s_y; i++) {
1570:       if (n9 >= 0) { /* left below */
1571:         x_t = lx[n9 % m]*dof;
1572:         y_t = ly[(n9 % (m*n))/m];
1573:         /* z_t = z; */
1574:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1575:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1576:       }
1577:       if (n10 >= 0) { /* directly below */
1578:         x_t = x;
1579:         y_t = ly[(n10 % (m*n))/m];
1580:         /* z_t = z; */
1581:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1582:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1583:       }
1584:       if (n11 >= 0) { /* right below */
1585:         x_t = lx[n11 % m]*dof;
1586:         y_t = ly[(n11 % (m*n))/m];
1587:         /* z_t = z; */
1588:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1589:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1590:       }
1591:     }

1593:     for (i=0; i<y; i++) {
1594:       if (n12 >= 0) { /* directly left */
1595:         x_t = lx[n12 % m]*dof;
1596:         y_t = y;
1597:         /* z_t = z; */
1598:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1599:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1600:       }

1602:       /* Interior */
1603:       s_t = bases[rank] + i*x + k*x*y;
1604:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1606:       if (n14 >= 0) { /* directly right */
1607:         x_t = lx[n14 % m]*dof;
1608:         y_t = y;
1609:         /* z_t = z; */
1610:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
1611:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1612:       }
1613:     }

1615:     for (i=1; i<=s_y; i++) {
1616:       if (n15 >= 0) { /* left above */
1617:         x_t = lx[n15 % m]*dof;
1618:         y_t = ly[(n15 % (m*n))/m];
1619:         /* z_t = z; */
1620:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1621:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1622:       }
1623:       if (n16 >= 0) { /* directly above */
1624:         x_t = x;
1625:         y_t = ly[(n16 % (m*n))/m];
1626:         /* z_t = z; */
1627:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1628:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1629:       }
1630:       if (n17 >= 0) { /* right above */
1631:         x_t = lx[n17 % m]*dof;
1632:         y_t = ly[(n17 % (m*n))/m];
1633:         /* z_t = z; */
1634:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1635:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1636:       }
1637:     }
1638:   }
1639: 
1640:   /* Upper Level */
1641:   for (k=0; k<s_z; k++) {
1642:     for (i=1; i<=s_y; i++) {
1643:       if (n18 >= 0) { /* left below */
1644:         x_t = lx[n18 % m]*dof;
1645:         y_t = ly[(n18 % (m*n))/m];
1646:         /* z_t = lz[n18 / (m*n)]; */
1647:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1648:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1649:       }
1650:       if (n19 >= 0) { /* directly below */
1651:         x_t = x;
1652:         y_t = ly[(n19 % (m*n))/m];
1653:         /* z_t = lz[n19 / (m*n)]; */
1654:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1655:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1656:       }
1657:       if (n20 >= 0) { /* right belodof */
1658:         x_t = lx[n20 % m]*dof;
1659:         y_t = ly[(n20 % (m*n))/m];
1660:         /* z_t = lz[n20 / (m*n)]; */
1661:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1662:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1663:       }
1664:     }

1666:     for (i=0; i<y; i++) {
1667:       if (n21 >= 0) { /* directly left */
1668:         x_t = lx[n21 % m]*dof;
1669:         y_t = y;
1670:         /* z_t = lz[n21 / (m*n)]; */
1671:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1672:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1673:       }

1675:       if (n22 >= 0) { /* middle */
1676:         x_t = x;
1677:         y_t = y;
1678:         /* z_t = lz[n22 / (m*n)]; */
1679:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
1680:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1681:       }

1683:       if (n23 >= 0) { /* directly right */
1684:         x_t = lx[n23 % m]*dof;
1685:         y_t = y;
1686:         /* z_t = lz[n23 / (m*n)]; */
1687:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
1688:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1689:       }
1690:     }

1692:     for (i=1; i<=s_y; i++) {
1693:       if (n24 >= 0) { /* left above */
1694:         x_t = lx[n24 % m]*dof;
1695:         y_t = ly[(n24 % (m*n))/m];
1696:         /* z_t = lz[n24 / (m*n)]; */
1697:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1698:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1699:       }
1700:       if (n25 >= 0) { /* directly above */
1701:         x_t = x;
1702:         y_t = ly[(n25 % (m*n))/m];
1703:         /* z_t = lz[n25 / (m*n)]; */
1704:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1705:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1706:       }
1707:       if (n26 >= 0) { /* right above */
1708:         x_t = lx[n26 % m]*dof;
1709:         y_t = ly[(n26 % (m*n))/m];
1710:         /* z_t = lz[n26 / (m*n)]; */
1711:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1712:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1713:       }
1714:     }
1715:   }

1717:   /* construct the local to local scatter context */
1718:   /* 
1719:       We simply remap the values in the from part of 
1720:     global to local to read from an array with the ghost values 
1721:     rather then from the plan array.
1722:   */
1723:   VecScatterCopy(gtol,&da->ltol);
1724:   PetscLogObjectParent(da,da->ltol);
1725:   left   = xs - Xs;
1726:   bottom = ys - Ys; top = bottom + y;
1727:   down   = zs - Zs; up  = down + z;
1728:   count  = x*(top-bottom)*(up-down);
1729:   PetscMalloc(count*sizeof(int),&idx);
1730:   count  = 0;
1731:   for (i=down; i<up; i++) {
1732:     for (j=bottom; j<top; j++) {
1733:       for (k=0; k<x; k++) {
1734:         idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
1735:       }
1736:     }
1737:   }
1738:   VecScatterRemap(da->ltol,idx,PETSC_NULL);
1739:   PetscFree(idx);


1742:   /* 
1743:      Build the natural ordering to PETSc ordering mappings.
1744:   */
1745:   PetscOptionsHasName(PETSC_NULL,"-da_noao",&flg1);
1746:   if (!flg1) {
1747:     IS  ispetsc,isnatural;
1748:     int *lidx,lict = 0;
1749:     int Nlocal = (da->xe-da->xs)*(da->ye-da->ys)*(da->ze-da->zs);

1751:     ISCreateStride(comm,Nlocal,da->base,1,&ispetsc);

1753:     PetscMalloc(Nlocal*sizeof(int),&lidx);
1754:     for (k=zs; k<ze; k++) {
1755:       for (j=ys; j<ye; j++) {
1756:         for (i=xs; i<xe; i++) {
1757:           lidx[lict++] = i + j*M*dof + k*M*N*dof;
1758:         }
1759:       }
1760:     }
1761:     ISCreateGeneral(comm,Nlocal,lidx,&isnatural);
1762:     PetscFree(lidx);

1764:     AOCreateBasicIS(isnatural,ispetsc,&da->ao);
1765:     PetscLogObjectParent(da,da->ao);
1766:     ISDestroy(ispetsc);
1767:     ISDestroy(isnatural);
1768:   } else {
1769:     da->ao = PETSC_NULL;
1770:   }

1772:   if (!flx) {
1773:     PetscMalloc(m*sizeof(int),&flx);
1774:     PetscMemcpy(flx,lx,m*sizeof(int));
1775:   }
1776:   if (!fly) {
1777:     PetscMalloc(n*sizeof(int),&fly);
1778:     PetscMemcpy(fly,ly,n*sizeof(int));
1779:   }
1780:   if (!flz) {
1781:     PetscMalloc(p*sizeof(int),&flz);
1782:     PetscMemcpy(flz,lz,p*sizeof(int));
1783:   }
1784:   da->lx = flx;
1785:   da->ly = fly;
1786:   da->lz = flz;

1788:   /*
1789:      Note the following will be removed soon. Since the functionality 
1790:     is replaced by the above. */

1792:   /* Construct the mapping from current global ordering to global
1793:      ordering that would be used if only 1 processor were employed.
1794:      This mapping is intended only for internal use by discrete
1795:      function and matrix viewers.

1797:      Note: At this point, x has already been adjusted for multiple
1798:      degrees of freedom per node.
1799:    */
1800:   ldim = x*y*z;
1801:   VecGetSize(global,&gdim);
1802:   PetscMalloc(gdim*sizeof(int),&da->gtog1);
1803:   PetscLogObjectMemory(da,gdim*sizeof(int));
1804:   PetscMalloc((2*(gdim+ldim))*sizeof(int),&gA);
1805:   gB        = (int *)(gA + ldim);
1806:   gAall     = (int *)(gB + ldim);
1807:   gBall     = (int *)(gAall + gdim);
1808:   /* Compute local parts of global orderings */
1809:   ict = 0;
1810:   for (k=zs; k<ze; k++) {
1811:     for (j=ys; j<ye; j++) {
1812:       for (i=xs; i<xe; i++) {
1813:         /* gA = global number for 1 proc; gB = current global number */
1814:         gA[ict] = i + j*M*dof + k*M*N*dof;
1815:         gB[ict] = start + ict;
1816:         ict++;
1817:       }
1818:     }
1819:   }
1820:   /* Broadcast the orderings */
1821:   MPI_Allgatherv(gA,ldim,MPI_INT,gAall,ldims,bases,MPI_INT,comm);
1822:   MPI_Allgatherv(gB,ldim,MPI_INT,gBall,ldims,bases,MPI_INT,comm);
1823:   for (i=0; i<gdim; i++) da->gtog1[gBall[i]] = gAall[i];
1824:   PetscFree(gA);
1825:   PetscFree(bases);

1827:   PetscOptionsHasName(PETSC_NULL,"-da_view",&flg1);
1828:   if (flg1) {DAView(da,PETSC_VIEWER_STDOUT_(da->comm));}
1829:   PetscOptionsHasName(PETSC_NULL,"-da_view_draw",&flg1);
1830:   if (flg1) {DAView(da,PETSC_VIEWER_DRAW_(da->comm));}
1831:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
1832:   if (flg1) {DAPrintHelp(da);}
1833:   PetscPublishAll(da);

1835: #if defined(PETSC_HAVE_AMS)
1836:   PetscObjectComposeFunctionDynamic((PetscObject)global,"AMSSetFieldBlock_C",
1837:          "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1838:   PetscObjectComposeFunctionDynamic((PetscObject)local,"AMSSetFieldBlock_C",
1839:          "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1840:   if (((PetscObject)global)->amem > -1) {
1841:     AMSSetFieldBlock_DA(((PetscObject)global)->amem,"values",global);
1842:   }
1843: #endif
1844:   VecSetOperation(global,VECOP_VIEW,(void(*)())VecView_MPI_DA);
1845:   VecSetOperation(global,VECOP_LOADINTOVECTOR,(void(*)())VecLoadIntoVector_Binary_DA);
1846:   return(0);
1847: }