Actual source code: da3.c
1: /*$Id: da3.c,v 1.136 2001/09/07 20:12:17 bsmith Exp $*/
3: /*
4: Code for manipulating distributed regular 3d arrays in parallel.
5: File created by Peter Mell 7/14/95
6: */
8: #include src/dm/da/daimpl.h
10: #if defined (PETSC_HAVE_AMS)
11: EXTERN_C_BEGIN
12: EXTERN int AMSSetFieldBlock_DA(AMS_Memory,char *,Vec);
13: EXTERN_C_END
14: #endif
16: int DAView_3d(DA da,PetscViewer viewer)
17: {
18: int rank,ierr;
19: PetscTruth isascii,isdraw,isbinary;
22: MPI_Comm_rank(da->comm,&rank);
24: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
25: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
26: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_BINARY,&isbinary);
27: if (isascii) {
28: PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %d N %d P %d m %d n %d p %d w %d s %dn",
29: rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
30: PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %d %d, Y range of indices: %d %d, Z range of indices: %d %dn",
31: da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
32: #if !defined(PETSC_USE_COMPLEX)
33: if (da->coordinates) {
34: int last;
35: PetscReal *coors;
36: VecGetArray(da->coordinates,&coors);
37: VecGetLocalSize(da->coordinates,&last);
38: last = last - 3;
39: PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %g %g %g : Upper right %g %g %gn",
40: coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
41: VecRestoreArray(da->coordinates,&coors);
42: }
43: #endif
44: PetscViewerFlush(viewer);
45: } else if (isdraw) {
46: PetscDraw draw;
47: PetscReal ymin = -1.0,ymax = (PetscReal)da->N;
48: PetscReal xmin = -1.0,xmax = (PetscReal)((da->M+2)*da->P),x,y,ycoord,xcoord;
49: int k,plane,base,*idx;
50: char node[10];
51: PetscTruth isnull;
53: PetscViewerDrawGetDraw(viewer,0,&draw);
54: PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
55: PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
56: PetscDrawSynchronizedClear(draw);
58: /* first processor draw all node lines */
59: if (!rank) {
60: for (k=0; k<da->P; k++) {
61: ymin = 0.0; ymax = (PetscReal)(da->N - 1);
62: for (xmin=(PetscReal)(k*(da->M+1)); xmin<(PetscReal)(da->M+(k*(da->M+1))); xmin++) {
63: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
64: }
65:
66: xmin = (PetscReal)(k*(da->M+1)); xmax = xmin + (PetscReal)(da->M - 1);
67: for (ymin=0; ymin<(PetscReal)da->N; ymin++) {
68: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
69: }
70: }
71: }
72: PetscDrawSynchronizedFlush(draw);
73: PetscDrawPause(draw);
75: for (k=0; k<da->P; k++) { /*Go through and draw for each plane*/
76: if ((k >= da->zs) && (k < da->ze)) {
77: /* draw my box */
78: ymin = da->ys;
79: ymax = da->ye - 1;
80: xmin = da->xs/da->w + (da->M+1)*k;
81: xmax =(da->xe-1)/da->w + (da->M+1)*k;
83: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
84: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
85: PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
86: PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);
88: xmin = da->xs/da->w;
89: xmax =(da->xe-1)/da->w;
91: /* put in numbers*/
92: base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;
94: /* Identify which processor owns the box */
95: sprintf(node,"%d",rank);
96: PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);
98: for (y=ymin; y<=ymax; y++) {
99: for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
100: sprintf(node,"%d",base++);
101: PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
102: }
103: }
104:
105: }
106: }
107: PetscDrawSynchronizedFlush(draw);
108: PetscDrawPause(draw);
110: for (k=0-da->s; k<da->P+da->s; k++) {
111: /* Go through and draw for each plane */
112: if ((k >= da->Zs) && (k < da->Ze)) {
113:
114: /* overlay ghost numbers, useful for error checking */
115: base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
116: plane=k;
117: /* Keep z wrap around points on the dradrawg */
118: if (k<0) { plane=da->P+k; }
119: if (k>=da->P) { plane=k-da->P; }
120: ymin = da->Ys; ymax = da->Ye;
121: xmin = (da->M+1)*plane*da->w;
122: xmax = (da->M+1)*plane*da->w+da->M*da->w;
123: for (y=ymin; y<ymax; y++) {
124: for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
125: sprintf(node,"%d",idx[base]/da->w);
126: ycoord = y;
127: /*Keep y wrap around points on drawing */
128: if (y<0) { ycoord = da->N+y; }
130: if (y>=da->N) { ycoord = y-da->N; }
131: xcoord = x; /* Keep x wrap points on drawing */
133: if (x<xmin) { xcoord = xmax - (xmin-x); }
134: if (x>=xmax) { xcoord = xmin + (x-xmax); }
135: PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
136: base+=da->w;
137: }
138: }
139: }
140: }
141: PetscDrawSynchronizedFlush(draw);
142: PetscDrawPause(draw);
143: } else if (isbinary) {
144: DAView_Binary(da,viewer);
145: } else {
146: SETERRQ1(1,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
147: }
148: return(0);
149: }
151: EXTERN int DAPublish_Petsc(PetscObject);
153: /*@C
154: DACreate3d - Creates an object that will manage the communication of three-dimensional
155: regular array data that is distributed across some processors.
157: Collective on MPI_Comm
159: Input Parameters:
160: + comm - MPI communicator
161: . wrap - type of periodicity the array should have, if any. Use one
162: of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
163: . stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
164: . M,N,P - global dimension in each direction of the array
165: . m,n,p - corresponding number of processors in each dimension
166: (or PETSC_DECIDE to have calculated)
167: . dof - number of degrees of freedom per node
168: . lx, ly, lz - arrays containing the number of nodes in each cell along
169: the x, y, and z coordinates, or PETSC_NULL. If non-null, these
170: must be of length as m,n,p and the corresponding
171: m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
172: the ly[] must n, sum of the lz[] must be P
173: - s - stencil width
175: Output Parameter:
176: . inra - the resulting distributed array object
178: Options Database Key:
179: + -da_view - Calls DAView() at the conclusion of DACreate3d()
180: . -da_grid_x <nx> - number of grid points in x direction, if M < 0
181: . -da_grid_y <ny> - number of grid points in y direction, if N < 0
182: . -da_grid_z <nz> - number of grid points in z direction, if P < 0
183: - -da_noao - do not compute natural to PETSc ordering object
185: Level: beginner
187: Notes:
188: If you are having problems with running out of memory than run with the option -da_noao
190: The stencil type DA_STENCIL_STAR with width 1 corresponds to the
191: standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
192: the standard 27-pt stencil.
194: The array data itself is NOT stored in the DA, it is stored in Vec objects;
195: The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
196: and DACreateLocalVector() and calls to VecDuplicate() if more are needed.
198: .keywords: distributed array, create, three-dimensional
200: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(),
201: DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(),
202: DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView()
204: @*/
205: int DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,int M,
206: int N,int P,int m,int n,int p,int dof,int s,int *lx,int *ly,int *lz,DA *inra)
207: {
208: int rank,size,ierr,start,end,pm;
209: int xs,xe,ys,ye,zs,ze,x,y,z,Xs,Xe,Ys,Ye,Zs,Ze;
210: int left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
211: int n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
212: int n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
213: int *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
214: int tM = M,tN = N,tP = P;
215: int sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
216: int sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
217: int sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
218: PetscTruth flg1,flg2;
219: DA da;
220: Vec local,global;
221: VecScatter ltog,gtol;
222: IS to,from;
226: *inra = 0;
227: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
228: DMInitializePackage(PETSC_NULL);
229: #endif
231: if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %d",dof);
232: if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %d",s);
234: PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
235: if (M < 0){
236: tM = -M;
237: PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",tM,&tM,PETSC_NULL);
238: }
239: if (N < 0){
240: tN = -N;
241: PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",tN,&tN,PETSC_NULL);
242: }
243: if (P < 0){
244: tP = -P;
245: PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",tP,&tP,PETSC_NULL);
246: }
247: PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
248: PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
249: PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
250: PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DACreate3d",refine_x,&refine_x,PETSC_NULL);
251: PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DACreate3d",refine_y,&refine_y,PETSC_NULL);
252: PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DACreate3d",refine_z,&refine_z,PETSC_NULL);
253: PetscOptionsEnd();
254: M = tM; N = tN; P = tP;
256: PetscHeaderCreate(da,_p_DA,struct _DAOps,DA_COOKIE,0,"DA",comm,DADestroy,DAView);
257: da->bops->publish = DAPublish_Petsc;
258: da->ops->createglobalvector = DACreateGlobalVector;
259: da->ops->getinterpolation = DAGetInterpolation;
260: da->ops->getcoloring = DAGetColoring;
261: da->ops->getmatrix = DAGetMatrix;
262: da->ops->refine = DARefine;
264: PetscLogObjectCreate(da);
265: PetscLogObjectMemory(da,sizeof(struct _p_DA));
266: da->dim = 3;
267: da->interptype = DA_Q1;
268: da->refine_x = refine_x;
269: da->refine_y = refine_y;
270: da->refine_z = refine_z;
271: PetscMalloc(dof*sizeof(char*),&da->fieldname);
272: PetscMemzero(da->fieldname,dof*sizeof(char*));
274: MPI_Comm_size(comm,&size);
275: MPI_Comm_rank(comm,&rank);
277: if (m != PETSC_DECIDE) {
278: if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %d",m);}
279: else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %d %d",m,size);}
280: }
281: if (n != PETSC_DECIDE) {
282: if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %d",n);}
283: else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %d %d",n,size);}
284: }
285: if (p != PETSC_DECIDE) {
286: if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %d",p);}
287: else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %d %d",p,size);}
288: }
290: /* Partition the array among the processors */
291: if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
292: m = size/(n*p);
293: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
294: n = size/(m*p);
295: } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
296: p = size/(m*n);
297: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
298: /* try for squarish distribution */
299: m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)N*p)));
300: if (!m) m = 1;
301: while (m > 0) {
302: n = size/(m*p);
303: if (m*n*p == size) break;
304: m--;
305: }
306: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %d",p);
307: if (M > N && m < n) {int _m = m; m = n; n = _m;}
308: } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
309: /* try for squarish distribution */
310: m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
311: if (!m) m = 1;
312: while (m > 0) {
313: p = size/(m*n);
314: if (m*n*p == size) break;
315: m--;
316: }
317: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %d",n);
318: if (M > P && m < p) {int _m = m; m = p; p = _m;}
319: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
320: /* try for squarish distribution */
321: n = (int)(0.5 + sqrt(((PetscReal)N)*((PetscReal)size)/((PetscReal)P*m)));
322: if (!n) n = 1;
323: while (n > 0) {
324: p = size/(m*n);
325: if (m*n*p == size) break;
326: n--;
327: }
328: if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %d",n);
329: if (N > P && n < p) {int _n = n; n = p; p = _n;}
330: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
331: /* try for squarish distribution */
332: n = (int)(0.5 + pow(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),1./3.));
333: if (!n) n = 1;
334: while (n > 0) {
335: pm = size/n;
336: if (n*pm == size) break;
337: n--;
338: }
339: if (!n) n = 1;
340: m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
341: if (!m) m = 1;
342: while (m > 0) {
343: p = size/(m*n);
344: if (m*n*p == size) break;
345: m--;
346: }
347: if (M > P && m < p) {int _m = m; m = p; p = _m;}
348: } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");
350: if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
351: if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %d %d",M,m);
352: if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %d %d",N,n);
353: if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %d %d",P,p);
355: PetscOptionsHasName(PETSC_NULL,"-da_partition_nodes_at_end",&flg2);
356: /*
357: Determine locally owned region
358: [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes
359: */
360: if (lx) { /* user decided distribution */
361: x = lx[rank % m];
362: xs = 0;
363: for (i=0; i<(rank%m); i++) { xs += lx[i];}
364: if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
365: } else if (flg2) {
366: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
367: } else { /* Normal PETSc distribution */
368: x = M/m + ((M % m) > (rank % m));
369: if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
370: if ((M % m) > (rank % m)) { xs = (rank % m)*x; }
371: else { xs = (M % m)*(x+1) + ((rank % m)-(M % m))*x; }
372: PetscMalloc(m*sizeof(int),&lx);
373: flx = lx;
374: for (i=0; i<m; i++) {
375: lx[i] = M/m + ((M % m) > (i % m));
376: }
377: }
378: if (ly) { /* user decided distribution */
379: y = ly[(rank % (m*n))/m];
380: if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
381: ys = 0;
382: for (i=0; i<(rank % (m*n))/m; i++) { ys += ly[i];}
383: } else if (flg2) {
384: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
385: } else { /* Normal PETSc distribution */
386: y = N/n + ((N % n) > ((rank % (m*n)) /m));
387: if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
388: if ((N % n) > ((rank % (m*n)) /m)) {ys = ((rank % (m*n))/m)*y;}
389: else {ys = (N % n)*(y+1) + (((rank % (m*n))/m)-(N % n))*y;}
390: PetscMalloc(n*sizeof(int),&ly);
391: fly = ly;
392: for (i=0; i<n; i++) {
393: ly[i] = N/n + ((N % n) > (i % n));
394: }
395: }
396: if (lz) { /* user decided distribution */
397: z = lz[rank/(m*n)];
398: if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
399: zs = 0;
400: for (i=0; i<(rank/(m*n)); i++) { zs += lz[i];}
401: } else if (flg2) {
402: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
403: } else { /* Normal PETSc distribution */
404: z = P/p + ((P % p) > (rank / (m*n)));
405: if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
406: if ((P % p) > (rank / (m*n))) {zs = (rank/(m*n))*z;}
407: else {zs = (P % p)*(z+1) + ((rank/(m*n))-(P % p))*z;}
408: PetscMalloc(p*sizeof(int),&lz);
409: flz = lz;
410: for (i=0; i<p; i++) {
411: lz[i] = P/p + ((P % p) > (i % p));
412: }
413: }
414: ye = ys + y;
415: xe = xs + x;
416: ze = zs + z;
418: /* determine ghost region */
419: /* Assume No Periodicity */
420: if (xs-s > 0) Xs = xs - s; else Xs = 0;
421: if (ys-s > 0) Ys = ys - s; else Ys = 0;
422: if (zs-s > 0) Zs = zs - s; else Zs = 0;
423: if (xe+s <= M) Xe = xe + s; else Xe = M;
424: if (ye+s <= N) Ye = ye + s; else Ye = N;
425: if (ze+s <= P) Ze = ze + s; else Ze = P;
427: /* X Periodic */
428: if (DAXPeriodic(wrap)){
429: Xs = xs - s;
430: Xe = xe + s;
431: }
433: /* Y Periodic */
434: if (DAYPeriodic(wrap)){
435: Ys = ys - s;
436: Ye = ye + s;
437: }
439: /* Z Periodic */
440: if (DAZPeriodic(wrap)){
441: Zs = zs - s;
442: Ze = ze + s;
443: }
445: /* Resize all X parameters to reflect w */
446: x *= dof;
447: xs *= dof;
448: xe *= dof;
449: Xs *= dof;
450: Xe *= dof;
451: s_x = s*dof;
452: s_y = s;
453: s_z = s;
455: /* determine starting point of each processor */
456: nn = x*y*z;
457: ierr = PetscMalloc((2*size+1)*sizeof(int),&bases);
458: ldims = (int*)(bases+size+1);
459: ierr = MPI_Allgather(&nn,1,MPI_INT,ldims,1,MPI_INT,comm);
460: bases[0] = 0;
461: for (i=1; i<=size; i++) {
462: bases[i] = ldims[i-1];
463: }
464: for (i=1; i<=size; i++) {
465: bases[i] += bases[i-1];
466: }
468: /* allocate the base parallel and sequential vectors */
469: VecCreateMPI(comm,x*y*z,PETSC_DECIDE,&global);
470: VecSetBlockSize(global,dof);
471: VecCreateSeq(MPI_COMM_SELF,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),&local);
472: VecSetBlockSize(local,dof);
474: /* generate appropriate vector scatters */
475: /* local to global inserts non-ghost point region into global */
476: VecGetOwnershipRange(global,&start,&end);
477: ISCreateStride(comm,x*y*z,start,1,&to);
479: left = xs - Xs;
480: bottom = ys - Ys; top = bottom + y;
481: down = zs - Zs; up = down + z;
482: count = x*(top-bottom)*(up-down);
483: PetscMalloc(count*sizeof(int),&idx);
484: count = 0;
485: for (i=down; i<up; i++) {
486: for (j=bottom; j<top; j++) {
487: for (k=0; k<x; k++) {
488: idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
489: }
490: }
491: }
492: ISCreateGeneral(comm,count,idx,&from);
493: PetscFree(idx);
495: VecScatterCreate(local,from,global,to,<og);
496: PetscLogObjectParent(da,to);
497: PetscLogObjectParent(da,from);
498: PetscLogObjectParent(da,ltog);
499: ISDestroy(from);
500: ISDestroy(to);
502: /* global to local must include ghost points */
503: if (stencil_type == DA_STENCIL_BOX) {
504: ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
505: } else {
506: /* This is way ugly! We need to list the funny cross type region */
507: /* the bottom chunck */
508: left = xs - Xs;
509: bottom = ys - Ys; top = bottom + y;
510: down = zs - Zs; up = down + z;
511: count = down*(top-bottom)*x +
512: (up-down)*(bottom*x + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) +
513: (Ze-Zs-up)*(top-bottom)*x;
514: PetscMalloc(count*sizeof(int),&idx);
515: count = 0;
516: for (i=0; i<down; i++) {
517: for (j=bottom; j<top; j++) {
518: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
519: }
520: }
521: /* the middle piece */
522: for (i=down; i<up; i++) {
523: /* front */
524: for (j=0; j<bottom; j++) {
525: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
526: }
527: /* middle */
528: for (j=bottom; j<top; j++) {
529: for (k=0; k<Xe-Xs; k++) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
530: }
531: /* back */
532: for (j=top; j<Ye-Ys; j++) {
533: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
534: }
535: }
536: /* the top piece */
537: for (i=up; i<Ze-Zs; i++) {
538: for (j=bottom; j<top; j++) {
539: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
540: }
541: }
542: ISCreateGeneral(comm,count,idx,&to);
543: PetscFree(idx);
544: }
546: /* determine who lies on each side of use stored in n24 n25 n26
547: n21 n22 n23
548: n18 n19 n20
550: n15 n16 n17
551: n12 n14
552: n9 n10 n11
554: n6 n7 n8
555: n3 n4 n5
556: n0 n1 n2
557: */
558:
559: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
560:
561: /* Assume Nodes are Internal to the Cube */
562:
563: n0 = rank - m*n - m - 1;
564: n1 = rank - m*n - m;
565: n2 = rank - m*n - m + 1;
566: n3 = rank - m*n -1;
567: n4 = rank - m*n;
568: n5 = rank - m*n + 1;
569: n6 = rank - m*n + m - 1;
570: n7 = rank - m*n + m;
571: n8 = rank - m*n + m + 1;
573: n9 = rank - m - 1;
574: n10 = rank - m;
575: n11 = rank - m + 1;
576: n12 = rank - 1;
577: n14 = rank + 1;
578: n15 = rank + m - 1;
579: n16 = rank + m;
580: n17 = rank + m + 1;
582: n18 = rank + m*n - m - 1;
583: n19 = rank + m*n - m;
584: n20 = rank + m*n - m + 1;
585: n21 = rank + m*n - 1;
586: n22 = rank + m*n;
587: n23 = rank + m*n + 1;
588: n24 = rank + m*n + m - 1;
589: n25 = rank + m*n + m;
590: n26 = rank + m*n + m + 1;
592: /* Assume Pieces are on Faces of Cube */
594: if (xs == 0) { /* First assume not corner or edge */
595: n0 = rank -1 - (m*n);
596: n3 = rank + m -1 - (m*n);
597: n6 = rank + 2*m -1 - (m*n);
598: n9 = rank -1;
599: n12 = rank + m -1;
600: n15 = rank + 2*m -1;
601: n18 = rank -1 + (m*n);
602: n21 = rank + m -1 + (m*n);
603: n24 = rank + 2*m -1 + (m*n);
604: }
606: if (xe == M*dof) { /* First assume not corner or edge */
607: n2 = rank -2*m +1 - (m*n);
608: n5 = rank - m +1 - (m*n);
609: n8 = rank +1 - (m*n);
610: n11 = rank -2*m +1;
611: n14 = rank - m +1;
612: n17 = rank +1;
613: n20 = rank -2*m +1 + (m*n);
614: n23 = rank - m +1 + (m*n);
615: n26 = rank +1 + (m*n);
616: }
618: if (ys==0) { /* First assume not corner or edge */
619: n0 = rank + m * (n-1) -1 - (m*n);
620: n1 = rank + m * (n-1) - (m*n);
621: n2 = rank + m * (n-1) +1 - (m*n);
622: n9 = rank + m * (n-1) -1;
623: n10 = rank + m * (n-1);
624: n11 = rank + m * (n-1) +1;
625: n18 = rank + m * (n-1) -1 + (m*n);
626: n19 = rank + m * (n-1) + (m*n);
627: n20 = rank + m * (n-1) +1 + (m*n);
628: }
630: if (ye == N) { /* First assume not corner or edge */
631: n6 = rank - m * (n-1) -1 - (m*n);
632: n7 = rank - m * (n-1) - (m*n);
633: n8 = rank - m * (n-1) +1 - (m*n);
634: n15 = rank - m * (n-1) -1;
635: n16 = rank - m * (n-1);
636: n17 = rank - m * (n-1) +1;
637: n24 = rank - m * (n-1) -1 + (m*n);
638: n25 = rank - m * (n-1) + (m*n);
639: n26 = rank - m * (n-1) +1 + (m*n);
640: }
641:
642: if (zs == 0) { /* First assume not corner or edge */
643: n0 = size - (m*n) + rank - m - 1;
644: n1 = size - (m*n) + rank - m;
645: n2 = size - (m*n) + rank - m + 1;
646: n3 = size - (m*n) + rank - 1;
647: n4 = size - (m*n) + rank;
648: n5 = size - (m*n) + rank + 1;
649: n6 = size - (m*n) + rank + m - 1;
650: n7 = size - (m*n) + rank + m ;
651: n8 = size - (m*n) + rank + m + 1;
652: }
654: if (ze == P) { /* First assume not corner or edge */
655: n18 = (m*n) - (size-rank) - m - 1;
656: n19 = (m*n) - (size-rank) - m;
657: n20 = (m*n) - (size-rank) - m + 1;
658: n21 = (m*n) - (size-rank) - 1;
659: n22 = (m*n) - (size-rank);
660: n23 = (m*n) - (size-rank) + 1;
661: n24 = (m*n) - (size-rank) + m - 1;
662: n25 = (m*n) - (size-rank) + m;
663: n26 = (m*n) - (size-rank) + m + 1;
664: }
666: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
667: n0 = size - m*n + rank + m-1 - m;
668: n3 = size - m*n + rank + m-1;
669: n6 = size - m*n + rank + m-1 + m;
670: }
671:
672: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
673: n18 = m*n - (size - rank) + m-1 - m;
674: n21 = m*n - (size - rank) + m-1;
675: n24 = m*n - (size - rank) + m-1 + m;
676: }
678: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
679: n0 = rank + m*n -1 - m*n;
680: n9 = rank + m*n -1;
681: n18 = rank + m*n -1 + m*n;
682: }
684: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
685: n6 = rank - m*(n-1) + m-1 - m*n;
686: n15 = rank - m*(n-1) + m-1;
687: n24 = rank - m*(n-1) + m-1 + m*n;
688: }
690: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
691: n2 = size - (m*n-rank) - (m-1) - m;
692: n5 = size - (m*n-rank) - (m-1);
693: n8 = size - (m*n-rank) - (m-1) + m;
694: }
696: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
697: n20 = m*n - (size - rank) - (m-1) - m;
698: n23 = m*n - (size - rank) - (m-1);
699: n26 = m*n - (size - rank) - (m-1) + m;
700: }
702: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
703: n2 = rank + m*(n-1) - (m-1) - m*n;
704: n11 = rank + m*(n-1) - (m-1);
705: n20 = rank + m*(n-1) - (m-1) + m*n;
706: }
708: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
709: n8 = rank - m*n +1 - m*n;
710: n17 = rank - m*n +1;
711: n26 = rank - m*n +1 + m*n;
712: }
714: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
715: n0 = size - m + rank -1;
716: n1 = size - m + rank;
717: n2 = size - m + rank +1;
718: }
720: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
721: n18 = m*n - (size - rank) + m*(n-1) -1;
722: n19 = m*n - (size - rank) + m*(n-1);
723: n20 = m*n - (size - rank) + m*(n-1) +1;
724: }
726: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
727: n6 = size - (m*n-rank) - m * (n-1) -1;
728: n7 = size - (m*n-rank) - m * (n-1);
729: n8 = size - (m*n-rank) - m * (n-1) +1;
730: }
732: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
733: n24 = rank - (size-m) -1;
734: n25 = rank - (size-m);
735: n26 = rank - (size-m) +1;
736: }
738: /* Check for Corners */
739: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
740: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
741: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
742: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
743: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
744: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
745: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
746: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
748: /* Check for when not X,Y, and Z Periodic */
750: /* If not X periodic */
751: if ((wrap != DA_XPERIODIC) && (wrap != DA_XYPERIODIC) &&
752: (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
753: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
754: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
755: }
757: /* If not Y periodic */
758: if ((wrap != DA_YPERIODIC) && (wrap != DA_XYPERIODIC) &&
759: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
760: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
761: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
762: }
764: /* If not Z periodic */
765: if ((wrap != DA_ZPERIODIC) && (wrap != DA_XZPERIODIC) &&
766: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
767: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
768: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
769: }
771: /* If star stencil then delete the corner neighbors */
772: if (stencil_type == DA_STENCIL_STAR) {
773: /* save information about corner neighbors */
774: sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
775: sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
776: sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
777: sn26 = n26;
778: n0 = n1 = n2 = n3 = n5 = n6 = n7 = n8 = n9 = n11 =
779: n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
780: }
783: PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int),&idx);
784: PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int));
786: nn = 0;
788: /* Bottom Level */
789: for (k=0; k<s_z; k++) {
790: for (i=1; i<=s_y; i++) {
791: if (n0 >= 0) { /* left below */
792: x_t = lx[n0 % m]*dof;
793: y_t = ly[(n0 % (m*n))/m];
794: z_t = lz[n0 / (m*n)];
795: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
796: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
797: }
798: if (n1 >= 0) { /* directly below */
799: x_t = x;
800: y_t = ly[(n1 % (m*n))/m];
801: z_t = lz[n1 / (m*n)];
802: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
803: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
804: }
805: if (n2 >= 0) { /* right below */
806: x_t = lx[n2 % m]*dof;
807: y_t = ly[(n2 % (m*n))/m];
808: z_t = lz[n2 / (m*n)];
809: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
810: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
811: }
812: }
814: for (i=0; i<y; i++) {
815: if (n3 >= 0) { /* directly left */
816: x_t = lx[n3 % m]*dof;
817: y_t = y;
818: z_t = lz[n3 / (m*n)];
819: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
820: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
821: }
823: if (n4 >= 0) { /* middle */
824: x_t = x;
825: y_t = y;
826: z_t = lz[n4 / (m*n)];
827: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
828: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
829: }
831: if (n5 >= 0) { /* directly right */
832: x_t = lx[n5 % m]*dof;
833: y_t = y;
834: z_t = lz[n5 / (m*n)];
835: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
836: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
837: }
838: }
840: for (i=1; i<=s_y; i++) {
841: if (n6 >= 0) { /* left above */
842: x_t = lx[n6 % m]*dof;
843: y_t = ly[(n6 % (m*n))/m];
844: z_t = lz[n6 / (m*n)];
845: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
846: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
847: }
848: if (n7 >= 0) { /* directly above */
849: x_t = x;
850: y_t = ly[(n7 % (m*n))/m];
851: z_t = lz[n7 / (m*n)];
852: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
853: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
854: }
855: if (n8 >= 0) { /* right above */
856: x_t = lx[n8 % m]*dof;
857: y_t = ly[(n8 % (m*n))/m];
858: z_t = lz[n8 / (m*n)];
859: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
860: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
861: }
862: }
863: }
865: /* Middle Level */
866: for (k=0; k<z; k++) {
867: for (i=1; i<=s_y; i++) {
868: if (n9 >= 0) { /* left below */
869: x_t = lx[n9 % m]*dof;
870: y_t = ly[(n9 % (m*n))/m];
871: /* z_t = z; */
872: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
873: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
874: }
875: if (n10 >= 0) { /* directly below */
876: x_t = x;
877: y_t = ly[(n10 % (m*n))/m];
878: /* z_t = z; */
879: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
880: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
881: }
882: if (n11 >= 0) { /* right below */
883: x_t = lx[n11 % m]*dof;
884: y_t = ly[(n11 % (m*n))/m];
885: /* z_t = z; */
886: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
887: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
888: }
889: }
891: for (i=0; i<y; i++) {
892: if (n12 >= 0) { /* directly left */
893: x_t = lx[n12 % m]*dof;
894: y_t = y;
895: /* z_t = z; */
896: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
897: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
898: }
900: /* Interior */
901: s_t = bases[rank] + i*x + k*x*y;
902: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
904: if (n14 >= 0) { /* directly right */
905: x_t = lx[n14 % m]*dof;
906: y_t = y;
907: /* z_t = z; */
908: s_t = bases[n14] + i*x_t + k*x_t*y_t;
909: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
910: }
911: }
913: for (i=1; i<=s_y; i++) {
914: if (n15 >= 0) { /* left above */
915: x_t = lx[n15 % m]*dof;
916: y_t = ly[(n15 % (m*n))/m];
917: /* z_t = z; */
918: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
919: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
920: }
921: if (n16 >= 0) { /* directly above */
922: x_t = x;
923: y_t = ly[(n16 % (m*n))/m];
924: /* z_t = z; */
925: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
926: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
927: }
928: if (n17 >= 0) { /* right above */
929: x_t = lx[n17 % m]*dof;
930: y_t = ly[(n17 % (m*n))/m];
931: /* z_t = z; */
932: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
933: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
934: }
935: }
936: }
937:
938: /* Upper Level */
939: for (k=0; k<s_z; k++) {
940: for (i=1; i<=s_y; i++) {
941: if (n18 >= 0) { /* left below */
942: x_t = lx[n18 % m]*dof;
943: y_t = ly[(n18 % (m*n))/m];
944: /* z_t = lz[n18 / (m*n)]; */
945: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
946: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
947: }
948: if (n19 >= 0) { /* directly below */
949: x_t = x;
950: y_t = ly[(n19 % (m*n))/m];
951: /* z_t = lz[n19 / (m*n)]; */
952: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
953: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
954: }
955: if (n20 >= 0) { /* right below */
956: x_t = lx[n20 % m]*dof;
957: y_t = ly[(n20 % (m*n))/m];
958: /* z_t = lz[n20 / (m*n)]; */
959: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
960: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
961: }
962: }
964: for (i=0; i<y; i++) {
965: if (n21 >= 0) { /* directly left */
966: x_t = lx[n21 % m]*dof;
967: y_t = y;
968: /* z_t = lz[n21 / (m*n)]; */
969: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
970: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
971: }
973: if (n22 >= 0) { /* middle */
974: x_t = x;
975: y_t = y;
976: /* z_t = lz[n22 / (m*n)]; */
977: s_t = bases[n22] + i*x_t + k*x_t*y_t;
978: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
979: }
981: if (n23 >= 0) { /* directly right */
982: x_t = lx[n23 % m]*dof;
983: y_t = y;
984: /* z_t = lz[n23 / (m*n)]; */
985: s_t = bases[n23] + i*x_t + k*x_t*y_t;
986: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
987: }
988: }
990: for (i=1; i<=s_y; i++) {
991: if (n24 >= 0) { /* left above */
992: x_t = lx[n24 % m]*dof;
993: y_t = ly[(n24 % (m*n))/m];
994: /* z_t = lz[n24 / (m*n)]; */
995: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
996: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
997: }
998: if (n25 >= 0) { /* directly above */
999: x_t = x;
1000: y_t = ly[(n25 % (m*n))/m];
1001: /* z_t = lz[n25 / (m*n)]; */
1002: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1003: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1004: }
1005: if (n26 >= 0) { /* right above */
1006: x_t = lx[n26 % m]*dof;
1007: y_t = ly[(n26 % (m*n))/m];
1008: /* z_t = lz[n26 / (m*n)]; */
1009: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1010: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1011: }
1012: }
1013: }
1014: base = bases[rank];
1015: ISCreateGeneral(comm,nn,idx,&from);
1016: VecScatterCreate(global,from,local,to,>ol);
1017: PetscLogObjectParent(da,gtol);
1018: PetscLogObjectParent(da,to);
1019: PetscLogObjectParent(da,from);
1020: ISDestroy(to);
1021: ISDestroy(from);
1022: da->stencil_type = stencil_type;
1023: da->M = M; da->N = N; da->P = P;
1024: da->m = m; da->n = n; da->p = p;
1025: da->w = dof; da->s = s;
1026: da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1027: da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;
1029: PetscLogObjectParent(da,global);
1030: PetscLogObjectParent(da,local);
1032: if (stencil_type == DA_STENCIL_STAR) {
1033: /*
1034: Recompute the local to global mappings, this time keeping the
1035: information about the cross corner processor numbers.
1036: */
1037: n0 = sn0; n1 = sn1; n2 = sn2; n3 = sn3; n5 = sn5; n6 = sn6; n7 = sn7;
1038: n8 = sn8; n9 = sn9; n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1039: n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1040: n26 = sn26;
1042: nn = 0;
1044: /* Bottom Level */
1045: for (k=0; k<s_z; k++) {
1046: for (i=1; i<=s_y; i++) {
1047: if (n0 >= 0) { /* left below */
1048: x_t = lx[n0 % m]*dof;
1049: y_t = ly[(n0 % (m*n))/m];
1050: z_t = lz[n0 / (m*n)];
1051: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1052: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1053: }
1054: if (n1 >= 0) { /* directly below */
1055: x_t = x;
1056: y_t = ly[(n1 % (m*n))/m];
1057: z_t = lz[n1 / (m*n)];
1058: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1059: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1060: }
1061: if (n2 >= 0) { /* right below */
1062: x_t = lx[n2 % m]*dof;
1063: y_t = ly[(n2 % (m*n))/m];
1064: z_t = lz[n2 / (m*n)];
1065: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1066: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1067: }
1068: }
1070: for (i=0; i<y; i++) {
1071: if (n3 >= 0) { /* directly left */
1072: x_t = lx[n3 % m]*dof;
1073: y_t = y;
1074: z_t = lz[n3 / (m*n)];
1075: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1076: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1077: }
1079: if (n4 >= 0) { /* middle */
1080: x_t = x;
1081: y_t = y;
1082: z_t = lz[n4 / (m*n)];
1083: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1084: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1085: }
1087: if (n5 >= 0) { /* directly right */
1088: x_t = lx[n5 % m]*dof;
1089: y_t = y;
1090: z_t = lz[n5 / (m*n)];
1091: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1092: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1093: }
1094: }
1096: for (i=1; i<=s_y; i++) {
1097: if (n6 >= 0) { /* left above */
1098: x_t = lx[n6 % m]*dof;
1099: y_t = ly[(n6 % (m*n))/m];
1100: z_t = lz[n6 / (m*n)];
1101: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1102: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1103: }
1104: if (n7 >= 0) { /* directly above */
1105: x_t = x;
1106: y_t = ly[(n7 % (m*n))/m];
1107: z_t = lz[n7 / (m*n)];
1108: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1109: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1110: }
1111: if (n8 >= 0) { /* right above */
1112: x_t = lx[n8 % m]*dof;
1113: y_t = ly[(n8 % (m*n))/m];
1114: z_t = lz[n8 / (m*n)];
1115: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1116: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1117: }
1118: }
1119: }
1121: /* Middle Level */
1122: for (k=0; k<z; k++) {
1123: for (i=1; i<=s_y; i++) {
1124: if (n9 >= 0) { /* left below */
1125: x_t = lx[n9 % m]*dof;
1126: y_t = ly[(n9 % (m*n))/m];
1127: /* z_t = z; */
1128: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1129: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1130: }
1131: if (n10 >= 0) { /* directly below */
1132: x_t = x;
1133: y_t = ly[(n10 % (m*n))/m];
1134: /* z_t = z; */
1135: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1136: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1137: }
1138: if (n11 >= 0) { /* right below */
1139: x_t = lx[n11 % m]*dof;
1140: y_t = ly[(n11 % (m*n))/m];
1141: /* z_t = z; */
1142: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1143: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1144: }
1145: }
1147: for (i=0; i<y; i++) {
1148: if (n12 >= 0) { /* directly left */
1149: x_t = lx[n12 % m]*dof;
1150: y_t = y;
1151: /* z_t = z; */
1152: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1153: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1154: }
1156: /* Interior */
1157: s_t = bases[rank] + i*x + k*x*y;
1158: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1160: if (n14 >= 0) { /* directly right */
1161: x_t = lx[n14 % m]*dof;
1162: y_t = y;
1163: /* z_t = z; */
1164: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1165: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1166: }
1167: }
1169: for (i=1; i<=s_y; i++) {
1170: if (n15 >= 0) { /* left above */
1171: x_t = lx[n15 % m]*dof;
1172: y_t = ly[(n15 % (m*n))/m];
1173: /* z_t = z; */
1174: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1175: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1176: }
1177: if (n16 >= 0) { /* directly above */
1178: x_t = x;
1179: y_t = ly[(n16 % (m*n))/m];
1180: /* z_t = z; */
1181: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1182: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1183: }
1184: if (n17 >= 0) { /* right above */
1185: x_t = lx[n17 % m]*dof;
1186: y_t = ly[(n17 % (m*n))/m];
1187: /* z_t = z; */
1188: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1189: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1190: }
1191: }
1192: }
1193:
1194: /* Upper Level */
1195: for (k=0; k<s_z; k++) {
1196: for (i=1; i<=s_y; i++) {
1197: if (n18 >= 0) { /* left below */
1198: x_t = lx[n18 % m]*dof;
1199: y_t = ly[(n18 % (m*n))/m];
1200: /* z_t = lz[n18 / (m*n)]; */
1201: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1202: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1203: }
1204: if (n19 >= 0) { /* directly below */
1205: x_t = x;
1206: y_t = ly[(n19 % (m*n))/m];
1207: /* z_t = lz[n19 / (m*n)]; */
1208: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1209: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1210: }
1211: if (n20 >= 0) { /* right below */
1212: x_t = lx[n20 % m]*dof;
1213: y_t = ly[(n20 % (m*n))/m];
1214: /* z_t = lz[n20 / (m*n)]; */
1215: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1216: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1217: }
1218: }
1220: for (i=0; i<y; i++) {
1221: if (n21 >= 0) { /* directly left */
1222: x_t = lx[n21 % m]*dof;
1223: y_t = y;
1224: /* z_t = lz[n21 / (m*n)]; */
1225: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1226: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1227: }
1229: if (n22 >= 0) { /* middle */
1230: x_t = x;
1231: y_t = y;
1232: /* z_t = lz[n22 / (m*n)]; */
1233: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1234: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1235: }
1237: if (n23 >= 0) { /* directly right */
1238: x_t = lx[n23 % m]*dof;
1239: y_t = y;
1240: /* z_t = lz[n23 / (m*n)]; */
1241: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1242: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1243: }
1244: }
1246: for (i=1; i<=s_y; i++) {
1247: if (n24 >= 0) { /* left above */
1248: x_t = lx[n24 % m]*dof;
1249: y_t = ly[(n24 % (m*n))/m];
1250: /* z_t = lz[n24 / (m*n)]; */
1251: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1252: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1253: }
1254: if (n25 >= 0) { /* directly above */
1255: x_t = x;
1256: y_t = ly[(n25 % (m*n))/m];
1257: /* z_t = lz[n25 / (m*n)]; */
1258: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1259: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1260: }
1261: if (n26 >= 0) { /* right above */
1262: x_t = lx[n26 % m]*dof;
1263: y_t = ly[(n26 % (m*n))/m];
1264: /* z_t = lz[n26 / (m*n)]; */
1265: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1266: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1267: }
1268: }
1269: }
1270: }
1271: da->global = global;
1272: da->local = local;
1273: da->gtol = gtol;
1274: da->ltog = ltog;
1275: da->idx = idx;
1276: da->Nl = nn;
1277: da->base = base;
1278: da->ops->view = DAView_3d;
1279: da->wrap = wrap;
1280: *inra = da;
1282: /*
1283: Set the local to global ordering in the global vector, this allows use
1284: of VecSetValuesLocal().
1285: */
1286: ierr = ISLocalToGlobalMappingCreate(comm,nn,idx,&da->ltogmap);
1287: ierr = VecSetLocalToGlobalMapping(da->global,da->ltogmap);
1288: ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1289: VecSetLocalToGlobalMappingBlock(da->global,da->ltogmapb);
1290: PetscLogObjectParent(da,da->ltogmap);
1292: /* redo idx to include "missing" ghost points */
1293: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1294:
1295: /* Assume Nodes are Internal to the Cube */
1296:
1297: n0 = rank - m*n - m - 1;
1298: n1 = rank - m*n - m;
1299: n2 = rank - m*n - m + 1;
1300: n3 = rank - m*n -1;
1301: n4 = rank - m*n;
1302: n5 = rank - m*n + 1;
1303: n6 = rank - m*n + m - 1;
1304: n7 = rank - m*n + m;
1305: n8 = rank - m*n + m + 1;
1307: n9 = rank - m - 1;
1308: n10 = rank - m;
1309: n11 = rank - m + 1;
1310: n12 = rank - 1;
1311: n14 = rank + 1;
1312: n15 = rank + m - 1;
1313: n16 = rank + m;
1314: n17 = rank + m + 1;
1316: n18 = rank + m*n - m - 1;
1317: n19 = rank + m*n - m;
1318: n20 = rank + m*n - m + 1;
1319: n21 = rank + m*n - 1;
1320: n22 = rank + m*n;
1321: n23 = rank + m*n + 1;
1322: n24 = rank + m*n + m - 1;
1323: n25 = rank + m*n + m;
1324: n26 = rank + m*n + m + 1;
1326: /* Assume Pieces are on Faces of Cube */
1328: if (xs == 0) { /* First assume not corner or edge */
1329: n0 = rank -1 - (m*n);
1330: n3 = rank + m -1 - (m*n);
1331: n6 = rank + 2*m -1 - (m*n);
1332: n9 = rank -1;
1333: n12 = rank + m -1;
1334: n15 = rank + 2*m -1;
1335: n18 = rank -1 + (m*n);
1336: n21 = rank + m -1 + (m*n);
1337: n24 = rank + 2*m -1 + (m*n);
1338: }
1340: if (xe == M*dof) { /* First assume not corner or edge */
1341: n2 = rank -2*m +1 - (m*n);
1342: n5 = rank - m +1 - (m*n);
1343: n8 = rank +1 - (m*n);
1344: n11 = rank -2*m +1;
1345: n14 = rank - m +1;
1346: n17 = rank +1;
1347: n20 = rank -2*m +1 + (m*n);
1348: n23 = rank - m +1 + (m*n);
1349: n26 = rank +1 + (m*n);
1350: }
1352: if (ys==0) { /* First assume not corner or edge */
1353: n0 = rank + m * (n-1) -1 - (m*n);
1354: n1 = rank + m * (n-1) - (m*n);
1355: n2 = rank + m * (n-1) +1 - (m*n);
1356: n9 = rank + m * (n-1) -1;
1357: n10 = rank + m * (n-1);
1358: n11 = rank + m * (n-1) +1;
1359: n18 = rank + m * (n-1) -1 + (m*n);
1360: n19 = rank + m * (n-1) + (m*n);
1361: n20 = rank + m * (n-1) +1 + (m*n);
1362: }
1364: if (ye == N) { /* First assume not corner or edge */
1365: n6 = rank - m * (n-1) -1 - (m*n);
1366: n7 = rank - m * (n-1) - (m*n);
1367: n8 = rank - m * (n-1) +1 - (m*n);
1368: n15 = rank - m * (n-1) -1;
1369: n16 = rank - m * (n-1);
1370: n17 = rank - m * (n-1) +1;
1371: n24 = rank - m * (n-1) -1 + (m*n);
1372: n25 = rank - m * (n-1) + (m*n);
1373: n26 = rank - m * (n-1) +1 + (m*n);
1374: }
1375:
1376: if (zs == 0) { /* First assume not corner or edge */
1377: n0 = size - (m*n) + rank - m - 1;
1378: n1 = size - (m*n) + rank - m;
1379: n2 = size - (m*n) + rank - m + 1;
1380: n3 = size - (m*n) + rank - 1;
1381: n4 = size - (m*n) + rank;
1382: n5 = size - (m*n) + rank + 1;
1383: n6 = size - (m*n) + rank + m - 1;
1384: n7 = size - (m*n) + rank + m ;
1385: n8 = size - (m*n) + rank + m + 1;
1386: }
1388: if (ze == P) { /* First assume not corner or edge */
1389: n18 = (m*n) - (size-rank) - m - 1;
1390: n19 = (m*n) - (size-rank) - m;
1391: n20 = (m*n) - (size-rank) - m + 1;
1392: n21 = (m*n) - (size-rank) - 1;
1393: n22 = (m*n) - (size-rank);
1394: n23 = (m*n) - (size-rank) + 1;
1395: n24 = (m*n) - (size-rank) + m - 1;
1396: n25 = (m*n) - (size-rank) + m;
1397: n26 = (m*n) - (size-rank) + m + 1;
1398: }
1400: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1401: n0 = size - m*n + rank + m-1 - m;
1402: n3 = size - m*n + rank + m-1;
1403: n6 = size - m*n + rank + m-1 + m;
1404: }
1405:
1406: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1407: n18 = m*n - (size - rank) + m-1 - m;
1408: n21 = m*n - (size - rank) + m-1;
1409: n24 = m*n - (size - rank) + m-1 + m;
1410: }
1412: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1413: n0 = rank + m*n -1 - m*n;
1414: n9 = rank + m*n -1;
1415: n18 = rank + m*n -1 + m*n;
1416: }
1418: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1419: n6 = rank - m*(n-1) + m-1 - m*n;
1420: n15 = rank - m*(n-1) + m-1;
1421: n24 = rank - m*(n-1) + m-1 + m*n;
1422: }
1424: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1425: n2 = size - (m*n-rank) - (m-1) - m;
1426: n5 = size - (m*n-rank) - (m-1);
1427: n8 = size - (m*n-rank) - (m-1) + m;
1428: }
1430: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1431: n20 = m*n - (size - rank) - (m-1) - m;
1432: n23 = m*n - (size - rank) - (m-1);
1433: n26 = m*n - (size - rank) - (m-1) + m;
1434: }
1436: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1437: n2 = rank + m*(n-1) - (m-1) - m*n;
1438: n11 = rank + m*(n-1) - (m-1);
1439: n20 = rank + m*(n-1) - (m-1) + m*n;
1440: }
1442: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1443: n8 = rank - m*n +1 - m*n;
1444: n17 = rank - m*n +1;
1445: n26 = rank - m*n +1 + m*n;
1446: }
1448: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1449: n0 = size - m + rank -1;
1450: n1 = size - m + rank;
1451: n2 = size - m + rank +1;
1452: }
1454: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1455: n18 = m*n - (size - rank) + m*(n-1) -1;
1456: n19 = m*n - (size - rank) + m*(n-1);
1457: n20 = m*n - (size - rank) + m*(n-1) +1;
1458: }
1460: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1461: n6 = size - (m*n-rank) - m * (n-1) -1;
1462: n7 = size - (m*n-rank) - m * (n-1);
1463: n8 = size - (m*n-rank) - m * (n-1) +1;
1464: }
1466: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1467: n24 = rank - (size-m) -1;
1468: n25 = rank - (size-m);
1469: n26 = rank - (size-m) +1;
1470: }
1472: /* Check for Corners */
1473: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
1474: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
1475: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
1476: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
1477: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
1478: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1479: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
1480: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
1482: /* Check for when not X,Y, and Z Periodic */
1484: /* If not X periodic */
1485: if (!DAXPeriodic(wrap)){
1486: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
1487: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1488: }
1490: /* If not Y periodic */
1491: if (!DAYPeriodic(wrap)){
1492: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
1493: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1494: }
1496: /* If not Z periodic */
1497: if (!DAZPeriodic(wrap)){
1498: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
1499: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1500: }
1502: nn = 0;
1504: /* Bottom Level */
1505: for (k=0; k<s_z; k++) {
1506: for (i=1; i<=s_y; i++) {
1507: if (n0 >= 0) { /* left below */
1508: x_t = lx[n0 % m]*dof;
1509: y_t = ly[(n0 % (m*n))/m];
1510: z_t = lz[n0 / (m*n)];
1511: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1512: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1513: }
1514: if (n1 >= 0) { /* directly below */
1515: x_t = x;
1516: y_t = ly[(n1 % (m*n))/m];
1517: z_t = lz[n1 / (m*n)];
1518: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1519: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1520: }
1521: if (n2 >= 0) { /* right below */
1522: x_t = lx[n2 % m]*dof;
1523: y_t = ly[(n2 % (m*n))/m];
1524: z_t = lz[n2 / (m*n)];
1525: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1526: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1527: }
1528: }
1530: for (i=0; i<y; i++) {
1531: if (n3 >= 0) { /* directly left */
1532: x_t = lx[n3 % m]*dof;
1533: y_t = y;
1534: z_t = lz[n3 / (m*n)];
1535: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1536: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1537: }
1539: if (n4 >= 0) { /* middle */
1540: x_t = x;
1541: y_t = y;
1542: z_t = lz[n4 / (m*n)];
1543: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1544: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1545: }
1547: if (n5 >= 0) { /* directly right */
1548: x_t = lx[n5 % m]*dof;
1549: y_t = y;
1550: z_t = lz[n5 / (m*n)];
1551: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1552: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1553: }
1554: }
1556: for (i=1; i<=s_y; i++) {
1557: if (n6 >= 0) { /* left above */
1558: x_t = lx[n6 % m]*dof;
1559: y_t = ly[(n6 % (m*n))/m];
1560: z_t = lz[n6 / (m*n)];
1561: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1562: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1563: }
1564: if (n7 >= 0) { /* directly above */
1565: x_t = x;
1566: y_t = ly[(n7 % (m*n))/m];
1567: z_t = lz[n7 / (m*n)];
1568: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1569: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1570: }
1571: if (n8 >= 0) { /* right above */
1572: x_t = lx[n8 % m]*dof;
1573: y_t = ly[(n8 % (m*n))/m];
1574: z_t = lz[n8 / (m*n)];
1575: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1576: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1577: }
1578: }
1579: }
1581: /* Middle Level */
1582: for (k=0; k<z; k++) {
1583: for (i=1; i<=s_y; i++) {
1584: if (n9 >= 0) { /* left below */
1585: x_t = lx[n9 % m]*dof;
1586: y_t = ly[(n9 % (m*n))/m];
1587: /* z_t = z; */
1588: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1589: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1590: }
1591: if (n10 >= 0) { /* directly below */
1592: x_t = x;
1593: y_t = ly[(n10 % (m*n))/m];
1594: /* z_t = z; */
1595: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1596: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1597: }
1598: if (n11 >= 0) { /* right below */
1599: x_t = lx[n11 % m]*dof;
1600: y_t = ly[(n11 % (m*n))/m];
1601: /* z_t = z; */
1602: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1603: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1604: }
1605: }
1607: for (i=0; i<y; i++) {
1608: if (n12 >= 0) { /* directly left */
1609: x_t = lx[n12 % m]*dof;
1610: y_t = y;
1611: /* z_t = z; */
1612: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1613: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1614: }
1616: /* Interior */
1617: s_t = bases[rank] + i*x + k*x*y;
1618: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1620: if (n14 >= 0) { /* directly right */
1621: x_t = lx[n14 % m]*dof;
1622: y_t = y;
1623: /* z_t = z; */
1624: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1625: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1626: }
1627: }
1629: for (i=1; i<=s_y; i++) {
1630: if (n15 >= 0) { /* left above */
1631: x_t = lx[n15 % m]*dof;
1632: y_t = ly[(n15 % (m*n))/m];
1633: /* z_t = z; */
1634: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1635: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1636: }
1637: if (n16 >= 0) { /* directly above */
1638: x_t = x;
1639: y_t = ly[(n16 % (m*n))/m];
1640: /* z_t = z; */
1641: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1642: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1643: }
1644: if (n17 >= 0) { /* right above */
1645: x_t = lx[n17 % m]*dof;
1646: y_t = ly[(n17 % (m*n))/m];
1647: /* z_t = z; */
1648: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1649: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1650: }
1651: }
1652: }
1653:
1654: /* Upper Level */
1655: for (k=0; k<s_z; k++) {
1656: for (i=1; i<=s_y; i++) {
1657: if (n18 >= 0) { /* left below */
1658: x_t = lx[n18 % m]*dof;
1659: y_t = ly[(n18 % (m*n))/m];
1660: /* z_t = lz[n18 / (m*n)]; */
1661: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1662: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1663: }
1664: if (n19 >= 0) { /* directly below */
1665: x_t = x;
1666: y_t = ly[(n19 % (m*n))/m];
1667: /* z_t = lz[n19 / (m*n)]; */
1668: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1669: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1670: }
1671: if (n20 >= 0) { /* right belodof */
1672: x_t = lx[n20 % m]*dof;
1673: y_t = ly[(n20 % (m*n))/m];
1674: /* z_t = lz[n20 / (m*n)]; */
1675: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1676: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1677: }
1678: }
1680: for (i=0; i<y; i++) {
1681: if (n21 >= 0) { /* directly left */
1682: x_t = lx[n21 % m]*dof;
1683: y_t = y;
1684: /* z_t = lz[n21 / (m*n)]; */
1685: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1686: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1687: }
1689: if (n22 >= 0) { /* middle */
1690: x_t = x;
1691: y_t = y;
1692: /* z_t = lz[n22 / (m*n)]; */
1693: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1694: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1695: }
1697: if (n23 >= 0) { /* directly right */
1698: x_t = lx[n23 % m]*dof;
1699: y_t = y;
1700: /* z_t = lz[n23 / (m*n)]; */
1701: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1702: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1703: }
1704: }
1706: for (i=1; i<=s_y; i++) {
1707: if (n24 >= 0) { /* left above */
1708: x_t = lx[n24 % m]*dof;
1709: y_t = ly[(n24 % (m*n))/m];
1710: /* z_t = lz[n24 / (m*n)]; */
1711: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1712: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1713: }
1714: if (n25 >= 0) { /* directly above */
1715: x_t = x;
1716: y_t = ly[(n25 % (m*n))/m];
1717: /* z_t = lz[n25 / (m*n)]; */
1718: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1719: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1720: }
1721: if (n26 >= 0) { /* right above */
1722: x_t = lx[n26 % m]*dof;
1723: y_t = ly[(n26 % (m*n))/m];
1724: /* z_t = lz[n26 / (m*n)]; */
1725: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1726: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1727: }
1728: }
1729: }
1730: PetscFree(bases);
1732: /* construct the local to local scatter context */
1733: /*
1734: We simply remap the values in the from part of
1735: global to local to read from an array with the ghost values
1736: rather then from the plan array.
1737: */
1738: VecScatterCopy(gtol,&da->ltol);
1739: PetscLogObjectParent(da,da->ltol);
1740: left = xs - Xs;
1741: bottom = ys - Ys; top = bottom + y;
1742: down = zs - Zs; up = down + z;
1743: count = x*(top-bottom)*(up-down);
1744: PetscMalloc(count*sizeof(int),&idx);
1745: count = 0;
1746: for (i=down; i<up; i++) {
1747: for (j=bottom; j<top; j++) {
1748: for (k=0; k<x; k++) {
1749: idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
1750: }
1751: }
1752: }
1753: VecScatterRemap(da->ltol,idx,PETSC_NULL);
1754: PetscFree(idx);
1757: /*
1758: Build the natural ordering to PETSc ordering mappings.
1759: */
1760: PetscOptionsHasName(PETSC_NULL,"-da_noao",&flg1);
1761: if (!flg1) {
1762: IS ispetsc,isnatural;
1763: int *lidx,lict = 0;
1764: int Nlocal = (da->xe-da->xs)*(da->ye-da->ys)*(da->ze-da->zs);
1766: ISCreateStride(comm,Nlocal,da->base,1,&ispetsc);
1768: PetscMalloc(Nlocal*sizeof(int),&lidx);
1769: for (k=zs; k<ze; k++) {
1770: for (j=ys; j<ye; j++) {
1771: for (i=xs; i<xe; i++) {
1772: lidx[lict++] = i + j*M*dof + k*M*N*dof;
1773: }
1774: }
1775: }
1776: ISCreateGeneral(comm,Nlocal,lidx,&isnatural);
1777: PetscFree(lidx);
1779: AOCreateBasicIS(isnatural,ispetsc,&da->ao);
1780: PetscLogObjectParent(da,da->ao);
1781: ISDestroy(ispetsc);
1782: ISDestroy(isnatural);
1783: } else {
1784: da->ao = PETSC_NULL;
1785: }
1787: if (!flx) {
1788: PetscMalloc(m*sizeof(int),&flx);
1789: PetscMemcpy(flx,lx,m*sizeof(int));
1790: }
1791: if (!fly) {
1792: PetscMalloc(n*sizeof(int),&fly);
1793: PetscMemcpy(fly,ly,n*sizeof(int));
1794: }
1795: if (!flz) {
1796: PetscMalloc(p*sizeof(int),&flz);
1797: PetscMemcpy(flz,lz,p*sizeof(int));
1798: }
1799: da->lx = flx;
1800: da->ly = fly;
1801: da->lz = flz;
1804: PetscOptionsHasName(PETSC_NULL,"-da_view",&flg1);
1805: if (flg1) {DAView(da,PETSC_VIEWER_STDOUT_(da->comm));}
1806: PetscOptionsHasName(PETSC_NULL,"-da_view_draw",&flg1);
1807: if (flg1) {DAView(da,PETSC_VIEWER_DRAW_(da->comm));}
1808: PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
1809: if (flg1) {DAPrintHelp(da);}
1810: PetscPublishAll(da);
1812: #if defined(PETSC_HAVE_AMS)
1813: PetscObjectComposeFunctionDynamic((PetscObject)global,"AMSSetFieldBlock_C",
1814: "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1815: PetscObjectComposeFunctionDynamic((PetscObject)local,"AMSSetFieldBlock_C",
1816: "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1817: if (((PetscObject)global)->amem > -1) {
1818: AMSSetFieldBlock_DA(((PetscObject)global)->amem,"values",global);
1819: }
1820: #endif
1821: VecSetOperation(global,VECOP_VIEW,(void(*)(void))VecView_MPI_DA);
1822: VecSetOperation(global,VECOP_LOADINTOVECTOR,(void(*)(void))VecLoadIntoVector_Binary_DA);
1823: return(0);
1824: }