Actual source code: da3.c
1: /*$Id: da3.c,v 1.132 2001/03/28 19:42:42 balay Exp bsmith $*/
3: /*
4: Code for manipulating distributed regular 3d arrays in parallel.
5: File created by Peter Mell 7/14/95
6: */
8: #include src/dm/da/daimpl.h
10: #if defined (PETSC_HAVE_AMS)
11: EXTERN_C_BEGIN
12: EXTERN int AMSSetFieldBlock_DA(AMS_Memory,char *,Vec);
13: EXTERN_C_END
14: #endif
16: int DAView_3d(DA da,PetscViewer viewer)
17: {
18: int rank,ierr;
19: PetscTruth isascii,isdraw,isbinary;
22: MPI_Comm_rank(da->comm,&rank);
24: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
25: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
26: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_BINARY,&isbinary);
27: if (isascii) {
28: PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %d N %d P %d m %d n %d p %d w %d s %dn",
29: rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
30: PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %d %d, Y range of indices: %d %d, Z range of indices: %d %dn",
31: da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
32: #if !defined(PETSC_USE_COMPLEX)
33: if (da->coordinates) {
34: int last;
35: double *coors;
36: VecGetArray(da->coordinates,&coors);
37: VecGetLocalSize(da->coordinates,&last);
38: last = last - 3;
39: PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %g %g %g : Upper right %g %g %gn",
40: coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
41: VecRestoreArray(da->coordinates,&coors);
42: }
43: #endif
44: PetscViewerFlush(viewer);
45: } else if (isdraw) {
46: PetscDraw draw;
47: double ymin = -1.0,ymax = (double)da->N;
48: double xmin = -1.0,xmax = (double)((da->M+2)*da->P),x,y,ycoord,xcoord;
49: int k,plane,base,*idx;
50: char node[10];
51: PetscTruth isnull;
53: PetscViewerDrawGetDraw(viewer,0,&draw);
54: PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
55: PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
56: PetscDrawSynchronizedClear(draw);
58: /* first processor draw all node lines */
59: if (!rank) {
60: for (k=0; k<da->P; k++) {
61: ymin = 0.0; ymax = (double)(da->N - 1);
62: for (xmin=(double)(k*(da->M+1)); xmin<(double)(da->M+(k*(da->M+1))); xmin++) {
63: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
64: }
65:
66: xmin = (double)(k*(da->M+1)); xmax = xmin + (double)(da->M - 1);
67: for (ymin=0; ymin<(double)da->N; ymin++) {
68: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
69: }
70: }
71: }
72: PetscDrawSynchronizedFlush(draw);
73: PetscDrawPause(draw);
75: for (k=0; k<da->P; k++) { /*Go through and draw for each plane*/
76: if ((k >= da->zs) && (k < da->ze)) {
77: /* draw my box */
78: ymin = da->ys;
79: ymax = da->ye - 1;
80: xmin = da->xs/da->w + (da->M+1)*k;
81: xmax =(da->xe-1)/da->w + (da->M+1)*k;
83: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
84: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
85: PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
86: PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);
88: xmin = da->xs/da->w;
89: xmax =(da->xe-1)/da->w;
91: /* put in numbers*/
92: base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;
94: /* Identify which processor owns the box */
95: sprintf(node,"%d",rank);
96: PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);
98: for (y=ymin; y<=ymax; y++) {
99: for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
100: sprintf(node,"%d",base++);
101: PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
102: }
103: }
104:
105: }
106: }
107: PetscDrawSynchronizedFlush(draw);
108: PetscDrawPause(draw);
110: for (k=0-da->s; k<da->P+da->s; k++) {
111: /* Go through and draw for each plane */
112: if ((k >= da->Zs) && (k < da->Ze)) {
113:
114: /* overlay ghost numbers, useful for error checking */
115: base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
116: plane=k;
117: /* Keep z wrap around points on the dradrawg */
118: if (k<0) { plane=da->P+k; }
119: if (k>=da->P) { plane=k-da->P; }
120: ymin = da->Ys; ymax = da->Ye;
121: xmin = (da->M+1)*plane*da->w;
122: xmax = (da->M+1)*plane*da->w+da->M*da->w;
123: for (y=ymin; y<ymax; y++) {
124: for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
125: sprintf(node,"%d",idx[base]/da->w);
126: ycoord = y;
127: /*Keep y wrap around points on drawing */
128: if (y<0) { ycoord = da->N+y; }
130: if (y>=da->N) { ycoord = y-da->N; }
131: xcoord = x; /* Keep x wrap points on drawing */
133: if (x<xmin) { xcoord = xmax - (xmin-x); }
134: if (x>=xmax) { xcoord = xmin + (x-xmax); }
135: PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
136: base+=da->w;
137: }
138: }
139: }
140: }
141: PetscDrawSynchronizedFlush(draw);
142: PetscDrawPause(draw);
143: } else if (isbinary) {
144: DAView_Binary(da,viewer);
145: } else {
146: SETERRQ1(1,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
147: }
148: return(0);
149: }
151: EXTERN int DAPublish_Petsc(PetscObject);
153: /*@C
154: DACreate3d - Creates an object that will manage the communication of three-dimensional
155: regular array data that is distributed across some processors.
157: Collective on MPI_Comm
159: Input Parameters:
160: + comm - MPI communicator
161: . wrap - type of periodicity the array should have, if any. Use one
162: of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
163: . stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
164: . M,N,P - global dimension in each direction of the array
165: . m,n,p - corresponding number of processors in each dimension
166: (or PETSC_DECIDE to have calculated)
167: . dof - number of degrees of freedom per node
168: . lx, ly, lz - arrays containing the number of nodes in each cell along
169: the x, y, and z coordinates, or PETSC_NULL. If non-null, these
170: must be of length as m,n,p and the corresponding
171: m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
172: the ly[] must n, sum of the lz[] must be P
173: - s - stencil width
175: Output Parameter:
176: . inra - the resulting distributed array object
178: Options Database Key:
179: + -da_view - Calls DAView() at the conclusion of DACreate3d()
180: . -da_grid_x <nx> - number of grid points in x direction
181: . -da_grid_y <ny> - number of grid points in y direction
182: . -da_grid_z <nz> - number of grid points in z direction
183: - -da_noao - do not compute natural to PETSc ordering object
185: Level: beginner
187: Notes:
188: The stencil type DA_STENCIL_STAR with width 1 corresponds to the
189: standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
190: the standard 27-pt stencil.
192: The array data itself is NOT stored in the DA, it is stored in Vec objects;
193: The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
194: and DACreateLocalVector() and calls to VecDuplicate() if more are needed.
196: .keywords: distributed array, create, three-dimensional
198: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(),
199: DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(),
200: DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView()
202: @*/
203: int DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,int M,
204: int N,int P,int m,int n,int p,int dof,int s,int *lx,int *ly,int *lz,DA *inra)
205: {
206: int rank,size,ierr,start,end,pm;
207: int xs,xe,ys,ye,zs,ze,x,y,z,Xs,Xe,Ys,Ye,Zs,Ze;
208: int left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
209: int n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
210: int n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
211: int *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
212: int *gA,*gB,*gAall,*gBall,ict,ldim,gdim;
213: int sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
214: int sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
215: int sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
216: PetscTruth flg1,flg2;
217: DA da;
218: Vec local,global;
219: VecScatter ltog,gtol;
220: IS to,from;
223: *inra = 0;
225: if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %d",dof);
226: if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %d",s);
227: if (M < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have M positive");
228: if (N < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have N positive");
229: if (P < 1) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have P positive");
231: PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
232: PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",M,&M,PETSC_NULL);
233: PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",N,&N,PETSC_NULL);
234: PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",P,&P,PETSC_NULL);
235: PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
236: PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
237: PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
238: PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DACreate3d",refine_x,&refine_x,PETSC_NULL);
239: PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DACreate3d",refine_y,&refine_y,PETSC_NULL);
240: PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DACreate3d",refine_z,&refine_z,PETSC_NULL);
241: PetscOptionsEnd();
243: PetscHeaderCreate(da,_p_DA,struct _DAOps,DA_COOKIE,0,"DA",comm,DADestroy,DAView);
244: da->bops->publish = DAPublish_Petsc;
245: da->ops->createglobalvector = DACreateGlobalVector;
246: da->ops->getinterpolation = DAGetInterpolation;
247: da->ops->getcoloring = DAGetColoring;
248: da->ops->refine = DARefine;
250: PetscLogObjectCreate(da);
251: PetscLogObjectMemory(da,sizeof(struct _p_DA));
252: da->dim = 3;
253: da->gtog1 = 0;
254: da->refine_x = refine_x;
255: da->refine_y = refine_y;
256: da->refine_z = refine_z;
257: PetscMalloc(dof*sizeof(char*),&da->fieldname);
258: PetscMemzero(da->fieldname,dof*sizeof(char*));
260: MPI_Comm_size(comm,&size);
261: MPI_Comm_rank(comm,&rank);
263: if (m != PETSC_DECIDE) {
264: if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %d",m);}
265: else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %d %d",m,size);}
266: }
267: if (n != PETSC_DECIDE) {
268: if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %d",n);}
269: else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %d %d",n,size);}
270: }
271: if (p != PETSC_DECIDE) {
272: if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %d",p);}
273: else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %d %d",p,size);}
274: }
276: /* Partition the array among the processors */
277: if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
278: m = size/(n*p);
279: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
280: n = size/(m*p);
281: } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
282: p = size/(m*n);
283: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
284: /* try for squarish distribution */
285: m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)N*p)));
286: if (!m) m = 1;
287: while (m > 0) {
288: n = size/(m*p);
289: if (m*n*p == size) break;
290: m--;
291: }
292: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %d",p);
293: if (M > N && m < n) {int _m = m; m = n; n = _m;}
294: } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
295: /* try for squarish distribution */
296: m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)P*n)));
297: if (!m) m = 1;
298: while (m > 0) {
299: p = size/(m*n);
300: if (m*n*p == size) break;
301: m--;
302: }
303: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %d",n);
304: if (M > P && m < p) {int _m = m; m = p; p = _m;}
305: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
306: /* try for squarish distribution */
307: n = (int)(0.5 + sqrt(((double)N)*((double)size)/((double)P*m)));
308: if (!n) n = 1;
309: while (n > 0) {
310: p = size/(m*n);
311: if (m*n*p == size) break;
312: n--;
313: }
314: if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %d",n);
315: if (N > P && n < p) {int _n = n; n = p; p = _n;}
316: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
317: /* try for squarish distribution */
318: n = (int)(0.5 + pow(((double)N*N)*((double)size)/((double)P*M),1./3.));
319: if (!n) n = 1;
320: while (n > 0) {
321: pm = size/n;
322: if (n*pm == size) break;
323: n--;
324: }
325: if (!n) n = 1;
326: m = (int)(0.5 + sqrt(((double)M)*((double)size)/((double)P*n)));
327: if (!m) m = 1;
328: while (m > 0) {
329: p = size/(m*n);
330: if (m*n*p == size) break;
331: m--;
332: }
333: if (M > P && m < p) {int _m = m; m = p; p = _m;}
334: } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");
336: if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
337: if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %d %d",M,m);
338: if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %d %d",N,n);
339: if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %d %d",P,p);
341: PetscOptionsHasName(PETSC_NULL,"-da_partition_nodes_at_end",&flg2);
342: /*
343: Determine locally owned region
344: [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes
345: */
346: if (lx) { /* user decided distribution */
347: x = lx[rank % m];
348: xs = 0;
349: for (i=0; i<(rank%m); i++) { xs += lx[i];}
350: if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
351: } else if (flg2) {
352: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
353: } else { /* Normal PETSc distribution */
354: x = M/m + ((M % m) > (rank % m));
355: if (x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %d %d",x,s);
356: if ((M % m) > (rank % m)) { xs = (rank % m)*x; }
357: else { xs = (M % m)*(x+1) + ((rank % m)-(M % m))*x; }
358: PetscMalloc(m*sizeof(int),&lx);
359: flx = lx;
360: for (i=0; i<m; i++) {
361: lx[i] = M/m + ((M % m) > (i % m));
362: }
363: }
364: if (ly) { /* user decided distribution */
365: y = ly[(rank % (m*n))/m];
366: if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
367: ys = 0;
368: for (i=0; i<(rank % (m*n))/m; i++) { ys += ly[i];}
369: } else if (flg2) {
370: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
371: } else { /* Normal PETSc distribution */
372: y = N/n + ((N % n) > ((rank % (m*n)) /m));
373: if (y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %d %d",y,s);
374: if ((N % n) > ((rank % (m*n)) /m)) {ys = ((rank % (m*n))/m)*y;}
375: else {ys = (N % n)*(y+1) + (((rank % (m*n))/m)-(N % n))*y;}
376: PetscMalloc(n*sizeof(int),&ly);
377: fly = ly;
378: for (i=0; i<n; i++) {
379: ly[i] = N/n + ((N % n) > (i % n));
380: }
381: }
382: if (lz) { /* user decided distribution */
383: z = lz[rank/(m*n)];
384: if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
385: zs = 0;
386: for (i=0; i<(rank/(m*n)); i++) { zs += lz[i];}
387: } else if (flg2) {
388: SETERRQ(PETSC_ERR_SUP,"-da_partition_nodes_at_end not supported");
389: } else { /* Normal PETSc distribution */
390: z = P/p + ((P % p) > (rank / (m*n)));
391: if (z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %d %d",z,s);
392: if ((P % p) > (rank / (m*n))) {zs = (rank/(m*n))*z;}
393: else {zs = (P % p)*(z+1) + ((rank/(m*n))-(P % p))*z;}
394: PetscMalloc(p*sizeof(int),&lz);
395: flz = lz;
396: for (i=0; i<p; i++) {
397: lz[i] = P/p + ((P % p) > (i % p));
398: }
399: }
400: ye = ys + y;
401: xe = xs + x;
402: ze = zs + z;
404: /* determine ghost region */
405: /* Assume No Periodicity */
406: if (xs-s > 0) Xs = xs - s; else Xs = 0;
407: if (ys-s > 0) Ys = ys - s; else Ys = 0;
408: if (zs-s > 0) Zs = zs - s; else Zs = 0;
409: if (xe+s <= M) Xe = xe + s; else Xe = M;
410: if (ye+s <= N) Ye = ye + s; else Ye = N;
411: if (ze+s <= P) Ze = ze + s; else Ze = P;
413: /* X Periodic */
414: if (DAXPeriodic(wrap)){
415: Xs = xs - s;
416: Xe = xe + s;
417: }
419: /* Y Periodic */
420: if (DAYPeriodic(wrap)){
421: Ys = ys - s;
422: Ye = ye + s;
423: }
425: /* Z Periodic */
426: if (DAZPeriodic(wrap)){
427: Zs = zs - s;
428: Ze = ze + s;
429: }
431: /* Resize all X parameters to reflect w */
432: x *= dof;
433: xs *= dof;
434: xe *= dof;
435: Xs *= dof;
436: Xe *= dof;
437: s_x = s*dof;
438: s_y = s;
439: s_z = s;
441: /* determine starting point of each processor */
442: nn = x*y*z;
443: PetscMalloc((2*size+1)*sizeof(int),&bases);
444: ldims = (int*)(bases+size+1);
445: MPI_Allgather(&nn,1,MPI_INT,ldims,1,MPI_INT,comm);
446: bases[0] = 0;
447: for (i=1; i<=size; i++) {
448: bases[i] = ldims[i-1];
449: }
450: for (i=1; i<=size; i++) {
451: bases[i] += bases[i-1];
452: }
454: /* allocate the base parallel and sequential vectors */
455: VecCreateMPI(comm,x*y*z,PETSC_DECIDE,&global);
456: VecSetBlockSize(global,dof);
457: VecCreateSeq(MPI_COMM_SELF,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),&local);
458: VecSetBlockSize(local,dof);
460: /* generate appropriate vector scatters */
461: /* local to global inserts non-ghost point region into global */
462: VecGetOwnershipRange(global,&start,&end);
463: ISCreateStride(comm,x*y*z,start,1,&to);
465: left = xs - Xs;
466: bottom = ys - Ys; top = bottom + y;
467: down = zs - Zs; up = down + z;
468: count = x*(top-bottom)*(up-down);
469: PetscMalloc(count*sizeof(int),&idx);
470: count = 0;
471: for (i=down; i<up; i++) {
472: for (j=bottom; j<top; j++) {
473: for (k=0; k<x; k++) {
474: idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
475: }
476: }
477: }
478: ISCreateGeneral(comm,count,idx,&from);
479: PetscFree(idx);
481: VecScatterCreate(local,from,global,to,<og);
482: PetscLogObjectParent(da,to);
483: PetscLogObjectParent(da,from);
484: PetscLogObjectParent(da,ltog);
485: ISDestroy(from);
486: ISDestroy(to);
488: /* global to local must include ghost points */
489: if (stencil_type == DA_STENCIL_BOX) {
490: ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
491: } else {
492: /* This is way ugly! We need to list the funny cross type region */
493: /* the bottom chunck */
494: left = xs - Xs;
495: bottom = ys - Ys; top = bottom + y;
496: down = zs - Zs; up = down + z;
497: count = down*(top-bottom)*x +
498: (up-down)*(bottom*x + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) +
499: (Ze-Zs-up)*(top-bottom)*x;
500: PetscMalloc(count*sizeof(int),&idx);
501: count = 0;
502: for (i=0; i<down; i++) {
503: for (j=bottom; j<top; j++) {
504: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
505: }
506: }
507: /* the middle piece */
508: for (i=down; i<up; i++) {
509: /* front */
510: for (j=0; j<bottom; j++) {
511: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
512: }
513: /* middle */
514: for (j=bottom; j<top; j++) {
515: for (k=0; k<Xe-Xs; k++) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
516: }
517: /* back */
518: for (j=top; j<Ye-Ys; j++) {
519: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
520: }
521: }
522: /* the top piece */
523: for (i=up; i<Ze-Zs; i++) {
524: for (j=bottom; j<top; j++) {
525: for (k=0; k<x; k++) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
526: }
527: }
528: ISCreateGeneral(comm,count,idx,&to);
529: PetscFree(idx);
530: }
532: /* determine who lies on each side of use stored in n24 n25 n26
533: n21 n22 n23
534: n18 n19 n20
536: n15 n16 n17
537: n12 n14
538: n9 n10 n11
540: n6 n7 n8
541: n3 n4 n5
542: n0 n1 n2
543: */
544:
545: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
546:
547: /* Assume Nodes are Internal to the Cube */
548:
549: n0 = rank - m*n - m - 1;
550: n1 = rank - m*n - m;
551: n2 = rank - m*n - m + 1;
552: n3 = rank - m*n -1;
553: n4 = rank - m*n;
554: n5 = rank - m*n + 1;
555: n6 = rank - m*n + m - 1;
556: n7 = rank - m*n + m;
557: n8 = rank - m*n + m + 1;
559: n9 = rank - m - 1;
560: n10 = rank - m;
561: n11 = rank - m + 1;
562: n12 = rank - 1;
563: n14 = rank + 1;
564: n15 = rank + m - 1;
565: n16 = rank + m;
566: n17 = rank + m + 1;
568: n18 = rank + m*n - m - 1;
569: n19 = rank + m*n - m;
570: n20 = rank + m*n - m + 1;
571: n21 = rank + m*n - 1;
572: n22 = rank + m*n;
573: n23 = rank + m*n + 1;
574: n24 = rank + m*n + m - 1;
575: n25 = rank + m*n + m;
576: n26 = rank + m*n + m + 1;
578: /* Assume Pieces are on Faces of Cube */
580: if (xs == 0) { /* First assume not corner or edge */
581: n0 = rank -1 - (m*n);
582: n3 = rank + m -1 - (m*n);
583: n6 = rank + 2*m -1 - (m*n);
584: n9 = rank -1;
585: n12 = rank + m -1;
586: n15 = rank + 2*m -1;
587: n18 = rank -1 + (m*n);
588: n21 = rank + m -1 + (m*n);
589: n24 = rank + 2*m -1 + (m*n);
590: }
592: if (xe == M*dof) { /* First assume not corner or edge */
593: n2 = rank -2*m +1 - (m*n);
594: n5 = rank - m +1 - (m*n);
595: n8 = rank +1 - (m*n);
596: n11 = rank -2*m +1;
597: n14 = rank - m +1;
598: n17 = rank +1;
599: n20 = rank -2*m +1 + (m*n);
600: n23 = rank - m +1 + (m*n);
601: n26 = rank +1 + (m*n);
602: }
604: if (ys==0) { /* First assume not corner or edge */
605: n0 = rank + m * (n-1) -1 - (m*n);
606: n1 = rank + m * (n-1) - (m*n);
607: n2 = rank + m * (n-1) +1 - (m*n);
608: n9 = rank + m * (n-1) -1;
609: n10 = rank + m * (n-1);
610: n11 = rank + m * (n-1) +1;
611: n18 = rank + m * (n-1) -1 + (m*n);
612: n19 = rank + m * (n-1) + (m*n);
613: n20 = rank + m * (n-1) +1 + (m*n);
614: }
616: if (ye == N) { /* First assume not corner or edge */
617: n6 = rank - m * (n-1) -1 - (m*n);
618: n7 = rank - m * (n-1) - (m*n);
619: n8 = rank - m * (n-1) +1 - (m*n);
620: n15 = rank - m * (n-1) -1;
621: n16 = rank - m * (n-1);
622: n17 = rank - m * (n-1) +1;
623: n24 = rank - m * (n-1) -1 + (m*n);
624: n25 = rank - m * (n-1) + (m*n);
625: n26 = rank - m * (n-1) +1 + (m*n);
626: }
627:
628: if (zs == 0) { /* First assume not corner or edge */
629: n0 = size - (m*n) + rank - m - 1;
630: n1 = size - (m*n) + rank - m;
631: n2 = size - (m*n) + rank - m + 1;
632: n3 = size - (m*n) + rank - 1;
633: n4 = size - (m*n) + rank;
634: n5 = size - (m*n) + rank + 1;
635: n6 = size - (m*n) + rank + m - 1;
636: n7 = size - (m*n) + rank + m ;
637: n8 = size - (m*n) + rank + m + 1;
638: }
640: if (ze == P) { /* First assume not corner or edge */
641: n18 = (m*n) - (size-rank) - m - 1;
642: n19 = (m*n) - (size-rank) - m;
643: n20 = (m*n) - (size-rank) - m + 1;
644: n21 = (m*n) - (size-rank) - 1;
645: n22 = (m*n) - (size-rank);
646: n23 = (m*n) - (size-rank) + 1;
647: n24 = (m*n) - (size-rank) + m - 1;
648: n25 = (m*n) - (size-rank) + m;
649: n26 = (m*n) - (size-rank) + m + 1;
650: }
652: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
653: n0 = size - m*n + rank + m-1 - m;
654: n3 = size - m*n + rank + m-1;
655: n6 = size - m*n + rank + m-1 + m;
656: }
657:
658: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
659: n18 = m*n - (size - rank) + m-1 - m;
660: n21 = m*n - (size - rank) + m-1;
661: n24 = m*n - (size - rank) + m-1 + m;
662: }
664: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
665: n0 = rank + m*n -1 - m*n;
666: n9 = rank + m*n -1;
667: n18 = rank + m*n -1 + m*n;
668: }
670: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
671: n6 = rank - m*(n-1) + m-1 - m*n;
672: n15 = rank - m*(n-1) + m-1;
673: n24 = rank - m*(n-1) + m-1 + m*n;
674: }
676: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
677: n2 = size - (m*n-rank) - (m-1) - m;
678: n5 = size - (m*n-rank) - (m-1);
679: n8 = size - (m*n-rank) - (m-1) + m;
680: }
682: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
683: n20 = m*n - (size - rank) - (m-1) - m;
684: n23 = m*n - (size - rank) - (m-1);
685: n26 = m*n - (size - rank) - (m-1) + m;
686: }
688: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
689: n2 = rank + m*(n-1) - (m-1) - m*n;
690: n11 = rank + m*(n-1) - (m-1);
691: n20 = rank + m*(n-1) - (m-1) + m*n;
692: }
694: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
695: n8 = rank - m*n +1 - m*n;
696: n17 = rank - m*n +1;
697: n26 = rank - m*n +1 + m*n;
698: }
700: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
701: n0 = size - m + rank -1;
702: n1 = size - m + rank;
703: n2 = size - m + rank +1;
704: }
706: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
707: n18 = m*n - (size - rank) + m*(n-1) -1;
708: n19 = m*n - (size - rank) + m*(n-1);
709: n20 = m*n - (size - rank) + m*(n-1) +1;
710: }
712: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
713: n6 = size - (m*n-rank) - m * (n-1) -1;
714: n7 = size - (m*n-rank) - m * (n-1);
715: n8 = size - (m*n-rank) - m * (n-1) +1;
716: }
718: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
719: n24 = rank - (size-m) -1;
720: n25 = rank - (size-m);
721: n26 = rank - (size-m) +1;
722: }
724: /* Check for Corners */
725: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
726: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
727: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
728: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
729: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
730: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
731: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
732: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
734: /* Check for when not X,Y, and Z Periodic */
736: /* If not X periodic */
737: if ((wrap != DA_XPERIODIC) && (wrap != DA_XYPERIODIC) &&
738: (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
739: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
740: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
741: }
743: /* If not Y periodic */
744: if ((wrap != DA_YPERIODIC) && (wrap != DA_XYPERIODIC) &&
745: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
746: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
747: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
748: }
750: /* If not Z periodic */
751: if ((wrap != DA_ZPERIODIC) && (wrap != DA_XZPERIODIC) &&
752: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
753: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
754: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
755: }
757: /* If star stencil then delete the corner neighbors */
758: if (stencil_type == DA_STENCIL_STAR) {
759: /* save information about corner neighbors */
760: sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
761: sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
762: sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
763: sn26 = n26;
764: n0 = n1 = n2 = n3 = n5 = n6 = n7 = n8 = n9 = n11 =
765: n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
766: }
769: PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int),&idx);
770: PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(int));
772: nn = 0;
774: /* Bottom Level */
775: for (k=0; k<s_z; k++) {
776: for (i=1; i<=s_y; i++) {
777: if (n0 >= 0) { /* left below */
778: x_t = lx[n0 % m]*dof;
779: y_t = ly[(n0 % (m*n))/m];
780: z_t = lz[n0 / (m*n)];
781: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
782: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
783: }
784: if (n1 >= 0) { /* directly below */
785: x_t = x;
786: y_t = ly[(n1 % (m*n))/m];
787: z_t = lz[n1 / (m*n)];
788: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
789: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
790: }
791: if (n2 >= 0) { /* right below */
792: x_t = lx[n2 % m]*dof;
793: y_t = ly[(n2 % (m*n))/m];
794: z_t = lz[n2 / (m*n)];
795: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
796: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
797: }
798: }
800: for (i=0; i<y; i++) {
801: if (n3 >= 0) { /* directly left */
802: x_t = lx[n3 % m]*dof;
803: y_t = y;
804: z_t = lz[n3 / (m*n)];
805: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
806: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
807: }
809: if (n4 >= 0) { /* middle */
810: x_t = x;
811: y_t = y;
812: z_t = lz[n4 / (m*n)];
813: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
814: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
815: }
817: if (n5 >= 0) { /* directly right */
818: x_t = lx[n5 % m]*dof;
819: y_t = y;
820: z_t = lz[n5 / (m*n)];
821: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
822: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
823: }
824: }
826: for (i=1; i<=s_y; i++) {
827: if (n6 >= 0) { /* left above */
828: x_t = lx[n6 % m]*dof;
829: y_t = ly[(n6 % (m*n))/m];
830: z_t = lz[n6 / (m*n)];
831: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
832: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
833: }
834: if (n7 >= 0) { /* directly above */
835: x_t = x;
836: y_t = ly[(n7 % (m*n))/m];
837: z_t = lz[n7 / (m*n)];
838: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
839: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
840: }
841: if (n8 >= 0) { /* right above */
842: x_t = lx[n8 % m]*dof;
843: y_t = ly[(n8 % (m*n))/m];
844: z_t = lz[n8 / (m*n)];
845: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
846: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
847: }
848: }
849: }
851: /* Middle Level */
852: for (k=0; k<z; k++) {
853: for (i=1; i<=s_y; i++) {
854: if (n9 >= 0) { /* left below */
855: x_t = lx[n9 % m]*dof;
856: y_t = ly[(n9 % (m*n))/m];
857: /* z_t = z; */
858: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
859: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
860: }
861: if (n10 >= 0) { /* directly below */
862: x_t = x;
863: y_t = ly[(n10 % (m*n))/m];
864: /* z_t = z; */
865: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
866: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
867: }
868: if (n11 >= 0) { /* right below */
869: x_t = lx[n11 % m]*dof;
870: y_t = ly[(n11 % (m*n))/m];
871: /* z_t = z; */
872: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
873: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
874: }
875: }
877: for (i=0; i<y; i++) {
878: if (n12 >= 0) { /* directly left */
879: x_t = lx[n12 % m]*dof;
880: y_t = y;
881: /* z_t = z; */
882: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
883: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
884: }
886: /* Interior */
887: s_t = bases[rank] + i*x + k*x*y;
888: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
890: if (n14 >= 0) { /* directly right */
891: x_t = lx[n14 % m]*dof;
892: y_t = y;
893: /* z_t = z; */
894: s_t = bases[n14] + i*x_t + k*x_t*y_t;
895: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
896: }
897: }
899: for (i=1; i<=s_y; i++) {
900: if (n15 >= 0) { /* left above */
901: x_t = lx[n15 % m]*dof;
902: y_t = ly[(n15 % (m*n))/m];
903: /* z_t = z; */
904: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
905: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
906: }
907: if (n16 >= 0) { /* directly above */
908: x_t = x;
909: y_t = ly[(n16 % (m*n))/m];
910: /* z_t = z; */
911: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
912: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
913: }
914: if (n17 >= 0) { /* right above */
915: x_t = lx[n17 % m]*dof;
916: y_t = ly[(n17 % (m*n))/m];
917: /* z_t = z; */
918: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
919: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
920: }
921: }
922: }
923:
924: /* Upper Level */
925: for (k=0; k<s_z; k++) {
926: for (i=1; i<=s_y; i++) {
927: if (n18 >= 0) { /* left below */
928: x_t = lx[n18 % m]*dof;
929: y_t = ly[(n18 % (m*n))/m];
930: /* z_t = lz[n18 / (m*n)]; */
931: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
932: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
933: }
934: if (n19 >= 0) { /* directly below */
935: x_t = x;
936: y_t = ly[(n19 % (m*n))/m];
937: /* z_t = lz[n19 / (m*n)]; */
938: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
939: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
940: }
941: if (n20 >= 0) { /* right below */
942: x_t = lx[n20 % m]*dof;
943: y_t = ly[(n20 % (m*n))/m];
944: /* z_t = lz[n20 / (m*n)]; */
945: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
946: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
947: }
948: }
950: for (i=0; i<y; i++) {
951: if (n21 >= 0) { /* directly left */
952: x_t = lx[n21 % m]*dof;
953: y_t = y;
954: /* z_t = lz[n21 / (m*n)]; */
955: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
956: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
957: }
959: if (n22 >= 0) { /* middle */
960: x_t = x;
961: y_t = y;
962: /* z_t = lz[n22 / (m*n)]; */
963: s_t = bases[n22] + i*x_t + k*x_t*y_t;
964: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
965: }
967: if (n23 >= 0) { /* directly right */
968: x_t = lx[n23 % m]*dof;
969: y_t = y;
970: /* z_t = lz[n23 / (m*n)]; */
971: s_t = bases[n23] + i*x_t + k*x_t*y_t;
972: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
973: }
974: }
976: for (i=1; i<=s_y; i++) {
977: if (n24 >= 0) { /* left above */
978: x_t = lx[n24 % m]*dof;
979: y_t = ly[(n24 % (m*n))/m];
980: /* z_t = lz[n24 / (m*n)]; */
981: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
982: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
983: }
984: if (n25 >= 0) { /* directly above */
985: x_t = x;
986: y_t = ly[(n25 % (m*n))/m];
987: /* z_t = lz[n25 / (m*n)]; */
988: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
989: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
990: }
991: if (n26 >= 0) { /* right above */
992: x_t = lx[n26 % m]*dof;
993: y_t = ly[(n26 % (m*n))/m];
994: /* z_t = lz[n26 / (m*n)]; */
995: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
996: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
997: }
998: }
999: }
1000: base = bases[rank];
1001: ISCreateGeneral(comm,nn,idx,&from);
1002: VecScatterCreate(global,from,local,to,>ol);
1003: PetscLogObjectParent(da,gtol);
1004: PetscLogObjectParent(da,to);
1005: PetscLogObjectParent(da,from);
1006: ISDestroy(to);
1007: ISDestroy(from);
1008: da->stencil_type = stencil_type;
1009: da->M = M; da->N = N; da->P = P;
1010: da->m = m; da->n = n; da->p = p;
1011: da->w = dof; da->s = s;
1012: da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1013: da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;
1015: PetscLogObjectParent(da,global);
1016: PetscLogObjectParent(da,local);
1018: if (stencil_type == DA_STENCIL_STAR) {
1019: /*
1020: Recompute the local to global mappings, this time keeping the
1021: information about the cross corner processor numbers.
1022: */
1023: n0 = sn0; n1 = sn1; n2 = sn2; n3 = sn3; n5 = sn5; n6 = sn6; n7 = sn7;
1024: n8 = sn8; n9 = sn9; n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1025: n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1026: n26 = sn26;
1028: nn = 0;
1030: /* Bottom Level */
1031: for (k=0; k<s_z; k++) {
1032: for (i=1; i<=s_y; i++) {
1033: if (n0 >= 0) { /* left below */
1034: x_t = lx[n0 % m]*dof;
1035: y_t = ly[(n0 % (m*n))/m];
1036: z_t = lz[n0 / (m*n)];
1037: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1038: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1039: }
1040: if (n1 >= 0) { /* directly below */
1041: x_t = x;
1042: y_t = ly[(n1 % (m*n))/m];
1043: z_t = lz[n1 / (m*n)];
1044: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1045: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1046: }
1047: if (n2 >= 0) { /* right below */
1048: x_t = lx[n2 % m]*dof;
1049: y_t = ly[(n2 % (m*n))/m];
1050: z_t = lz[n2 / (m*n)];
1051: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1052: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1053: }
1054: }
1056: for (i=0; i<y; i++) {
1057: if (n3 >= 0) { /* directly left */
1058: x_t = lx[n3 % m]*dof;
1059: y_t = y;
1060: z_t = lz[n3 / (m*n)];
1061: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1062: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1063: }
1065: if (n4 >= 0) { /* middle */
1066: x_t = x;
1067: y_t = y;
1068: z_t = lz[n4 / (m*n)];
1069: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1070: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1071: }
1073: if (n5 >= 0) { /* directly right */
1074: x_t = lx[n5 % m]*dof;
1075: y_t = y;
1076: z_t = lz[n5 / (m*n)];
1077: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1078: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1079: }
1080: }
1082: for (i=1; i<=s_y; i++) {
1083: if (n6 >= 0) { /* left above */
1084: x_t = lx[n6 % m]*dof;
1085: y_t = ly[(n6 % (m*n))/m];
1086: z_t = lz[n6 / (m*n)];
1087: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1088: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1089: }
1090: if (n7 >= 0) { /* directly above */
1091: x_t = x;
1092: y_t = ly[(n7 % (m*n))/m];
1093: z_t = lz[n7 / (m*n)];
1094: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1095: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1096: }
1097: if (n8 >= 0) { /* right above */
1098: x_t = lx[n8 % m]*dof;
1099: y_t = ly[(n8 % (m*n))/m];
1100: z_t = lz[n8 / (m*n)];
1101: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1102: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1103: }
1104: }
1105: }
1107: /* Middle Level */
1108: for (k=0; k<z; k++) {
1109: for (i=1; i<=s_y; i++) {
1110: if (n9 >= 0) { /* left below */
1111: x_t = lx[n9 % m]*dof;
1112: y_t = ly[(n9 % (m*n))/m];
1113: /* z_t = z; */
1114: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1115: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1116: }
1117: if (n10 >= 0) { /* directly below */
1118: x_t = x;
1119: y_t = ly[(n10 % (m*n))/m];
1120: /* z_t = z; */
1121: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1122: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1123: }
1124: if (n11 >= 0) { /* right below */
1125: x_t = lx[n11 % m]*dof;
1126: y_t = ly[(n11 % (m*n))/m];
1127: /* z_t = z; */
1128: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1129: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1130: }
1131: }
1133: for (i=0; i<y; i++) {
1134: if (n12 >= 0) { /* directly left */
1135: x_t = lx[n12 % m]*dof;
1136: y_t = y;
1137: /* z_t = z; */
1138: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1139: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1140: }
1142: /* Interior */
1143: s_t = bases[rank] + i*x + k*x*y;
1144: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1146: if (n14 >= 0) { /* directly right */
1147: x_t = lx[n14 % m]*dof;
1148: y_t = y;
1149: /* z_t = z; */
1150: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1151: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1152: }
1153: }
1155: for (i=1; i<=s_y; i++) {
1156: if (n15 >= 0) { /* left above */
1157: x_t = lx[n15 % m]*dof;
1158: y_t = ly[(n15 % (m*n))/m];
1159: /* z_t = z; */
1160: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1161: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1162: }
1163: if (n16 >= 0) { /* directly above */
1164: x_t = x;
1165: y_t = ly[(n16 % (m*n))/m];
1166: /* z_t = z; */
1167: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1168: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1169: }
1170: if (n17 >= 0) { /* right above */
1171: x_t = lx[n17 % m]*dof;
1172: y_t = ly[(n17 % (m*n))/m];
1173: /* z_t = z; */
1174: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1175: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1176: }
1177: }
1178: }
1179:
1180: /* Upper Level */
1181: for (k=0; k<s_z; k++) {
1182: for (i=1; i<=s_y; i++) {
1183: if (n18 >= 0) { /* left below */
1184: x_t = lx[n18 % m]*dof;
1185: y_t = ly[(n18 % (m*n))/m];
1186: /* z_t = lz[n18 / (m*n)]; */
1187: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1188: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1189: }
1190: if (n19 >= 0) { /* directly below */
1191: x_t = x;
1192: y_t = ly[(n19 % (m*n))/m];
1193: /* z_t = lz[n19 / (m*n)]; */
1194: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1195: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1196: }
1197: if (n20 >= 0) { /* right below */
1198: x_t = lx[n20 % m]*dof;
1199: y_t = ly[(n20 % (m*n))/m];
1200: /* z_t = lz[n20 / (m*n)]; */
1201: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1202: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1203: }
1204: }
1206: for (i=0; i<y; i++) {
1207: if (n21 >= 0) { /* directly left */
1208: x_t = lx[n21 % m]*dof;
1209: y_t = y;
1210: /* z_t = lz[n21 / (m*n)]; */
1211: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1212: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1213: }
1215: if (n22 >= 0) { /* middle */
1216: x_t = x;
1217: y_t = y;
1218: /* z_t = lz[n22 / (m*n)]; */
1219: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1220: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1221: }
1223: if (n23 >= 0) { /* directly right */
1224: x_t = lx[n23 % m]*dof;
1225: y_t = y;
1226: /* z_t = lz[n23 / (m*n)]; */
1227: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1228: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1229: }
1230: }
1232: for (i=1; i<=s_y; i++) {
1233: if (n24 >= 0) { /* left above */
1234: x_t = lx[n24 % m]*dof;
1235: y_t = ly[(n24 % (m*n))/m];
1236: /* z_t = lz[n24 / (m*n)]; */
1237: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1238: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1239: }
1240: if (n25 >= 0) { /* directly above */
1241: x_t = x;
1242: y_t = ly[(n25 % (m*n))/m];
1243: /* z_t = lz[n25 / (m*n)]; */
1244: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1245: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1246: }
1247: if (n26 >= 0) { /* right above */
1248: x_t = lx[n26 % m]*dof;
1249: y_t = ly[(n26 % (m*n))/m];
1250: /* z_t = lz[n26 / (m*n)]; */
1251: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1252: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1253: }
1254: }
1255: }
1256: }
1257: da->global = global;
1258: da->local = local;
1259: da->gtol = gtol;
1260: da->ltog = ltog;
1261: da->idx = idx;
1262: da->Nl = nn;
1263: da->base = base;
1264: da->ops->view = DAView_3d;
1265: da->wrap = wrap;
1266: *inra = da;
1268: /*
1269: Set the local to global ordering in the global vector, this allows use
1270: of VecSetValuesLocal().
1271: */
1272: ierr = ISLocalToGlobalMappingCreate(comm,nn,idx,&da->ltogmap);
1273: ierr = VecSetLocalToGlobalMapping(da->global,da->ltogmap);
1274: ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1275: VecSetLocalToGlobalMappingBlock(da->global,da->ltogmapb);
1276: PetscLogObjectParent(da,da->ltogmap);
1278: /* redo idx to include "missing" ghost points */
1279: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1280:
1281: /* Assume Nodes are Internal to the Cube */
1282:
1283: n0 = rank - m*n - m - 1;
1284: n1 = rank - m*n - m;
1285: n2 = rank - m*n - m + 1;
1286: n3 = rank - m*n -1;
1287: n4 = rank - m*n;
1288: n5 = rank - m*n + 1;
1289: n6 = rank - m*n + m - 1;
1290: n7 = rank - m*n + m;
1291: n8 = rank - m*n + m + 1;
1293: n9 = rank - m - 1;
1294: n10 = rank - m;
1295: n11 = rank - m + 1;
1296: n12 = rank - 1;
1297: n14 = rank + 1;
1298: n15 = rank + m - 1;
1299: n16 = rank + m;
1300: n17 = rank + m + 1;
1302: n18 = rank + m*n - m - 1;
1303: n19 = rank + m*n - m;
1304: n20 = rank + m*n - m + 1;
1305: n21 = rank + m*n - 1;
1306: n22 = rank + m*n;
1307: n23 = rank + m*n + 1;
1308: n24 = rank + m*n + m - 1;
1309: n25 = rank + m*n + m;
1310: n26 = rank + m*n + m + 1;
1312: /* Assume Pieces are on Faces of Cube */
1314: if (xs == 0) { /* First assume not corner or edge */
1315: n0 = rank -1 - (m*n);
1316: n3 = rank + m -1 - (m*n);
1317: n6 = rank + 2*m -1 - (m*n);
1318: n9 = rank -1;
1319: n12 = rank + m -1;
1320: n15 = rank + 2*m -1;
1321: n18 = rank -1 + (m*n);
1322: n21 = rank + m -1 + (m*n);
1323: n24 = rank + 2*m -1 + (m*n);
1324: }
1326: if (xe == M*dof) { /* First assume not corner or edge */
1327: n2 = rank -2*m +1 - (m*n);
1328: n5 = rank - m +1 - (m*n);
1329: n8 = rank +1 - (m*n);
1330: n11 = rank -2*m +1;
1331: n14 = rank - m +1;
1332: n17 = rank +1;
1333: n20 = rank -2*m +1 + (m*n);
1334: n23 = rank - m +1 + (m*n);
1335: n26 = rank +1 + (m*n);
1336: }
1338: if (ys==0) { /* First assume not corner or edge */
1339: n0 = rank + m * (n-1) -1 - (m*n);
1340: n1 = rank + m * (n-1) - (m*n);
1341: n2 = rank + m * (n-1) +1 - (m*n);
1342: n9 = rank + m * (n-1) -1;
1343: n10 = rank + m * (n-1);
1344: n11 = rank + m * (n-1) +1;
1345: n18 = rank + m * (n-1) -1 + (m*n);
1346: n19 = rank + m * (n-1) + (m*n);
1347: n20 = rank + m * (n-1) +1 + (m*n);
1348: }
1350: if (ye == N) { /* First assume not corner or edge */
1351: n6 = rank - m * (n-1) -1 - (m*n);
1352: n7 = rank - m * (n-1) - (m*n);
1353: n8 = rank - m * (n-1) +1 - (m*n);
1354: n15 = rank - m * (n-1) -1;
1355: n16 = rank - m * (n-1);
1356: n17 = rank - m * (n-1) +1;
1357: n24 = rank - m * (n-1) -1 + (m*n);
1358: n25 = rank - m * (n-1) + (m*n);
1359: n26 = rank - m * (n-1) +1 + (m*n);
1360: }
1361:
1362: if (zs == 0) { /* First assume not corner or edge */
1363: n0 = size - (m*n) + rank - m - 1;
1364: n1 = size - (m*n) + rank - m;
1365: n2 = size - (m*n) + rank - m + 1;
1366: n3 = size - (m*n) + rank - 1;
1367: n4 = size - (m*n) + rank;
1368: n5 = size - (m*n) + rank + 1;
1369: n6 = size - (m*n) + rank + m - 1;
1370: n7 = size - (m*n) + rank + m ;
1371: n8 = size - (m*n) + rank + m + 1;
1372: }
1374: if (ze == P) { /* First assume not corner or edge */
1375: n18 = (m*n) - (size-rank) - m - 1;
1376: n19 = (m*n) - (size-rank) - m;
1377: n20 = (m*n) - (size-rank) - m + 1;
1378: n21 = (m*n) - (size-rank) - 1;
1379: n22 = (m*n) - (size-rank);
1380: n23 = (m*n) - (size-rank) + 1;
1381: n24 = (m*n) - (size-rank) + m - 1;
1382: n25 = (m*n) - (size-rank) + m;
1383: n26 = (m*n) - (size-rank) + m + 1;
1384: }
1386: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1387: n0 = size - m*n + rank + m-1 - m;
1388: n3 = size - m*n + rank + m-1;
1389: n6 = size - m*n + rank + m-1 + m;
1390: }
1391:
1392: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1393: n18 = m*n - (size - rank) + m-1 - m;
1394: n21 = m*n - (size - rank) + m-1;
1395: n24 = m*n - (size - rank) + m-1 + m;
1396: }
1398: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1399: n0 = rank + m*n -1 - m*n;
1400: n9 = rank + m*n -1;
1401: n18 = rank + m*n -1 + m*n;
1402: }
1404: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1405: n6 = rank - m*(n-1) + m-1 - m*n;
1406: n15 = rank - m*(n-1) + m-1;
1407: n24 = rank - m*(n-1) + m-1 + m*n;
1408: }
1410: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1411: n2 = size - (m*n-rank) - (m-1) - m;
1412: n5 = size - (m*n-rank) - (m-1);
1413: n8 = size - (m*n-rank) - (m-1) + m;
1414: }
1416: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1417: n20 = m*n - (size - rank) - (m-1) - m;
1418: n23 = m*n - (size - rank) - (m-1);
1419: n26 = m*n - (size - rank) - (m-1) + m;
1420: }
1422: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1423: n2 = rank + m*(n-1) - (m-1) - m*n;
1424: n11 = rank + m*(n-1) - (m-1);
1425: n20 = rank + m*(n-1) - (m-1) + m*n;
1426: }
1428: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1429: n8 = rank - m*n +1 - m*n;
1430: n17 = rank - m*n +1;
1431: n26 = rank - m*n +1 + m*n;
1432: }
1434: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1435: n0 = size - m + rank -1;
1436: n1 = size - m + rank;
1437: n2 = size - m + rank +1;
1438: }
1440: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1441: n18 = m*n - (size - rank) + m*(n-1) -1;
1442: n19 = m*n - (size - rank) + m*(n-1);
1443: n20 = m*n - (size - rank) + m*(n-1) +1;
1444: }
1446: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1447: n6 = size - (m*n-rank) - m * (n-1) -1;
1448: n7 = size - (m*n-rank) - m * (n-1);
1449: n8 = size - (m*n-rank) - m * (n-1) +1;
1450: }
1452: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1453: n24 = rank - (size-m) -1;
1454: n25 = rank - (size-m);
1455: n26 = rank - (size-m) +1;
1456: }
1458: /* Check for Corners */
1459: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
1460: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
1461: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
1462: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
1463: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
1464: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1465: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
1466: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
1468: /* Check for when not X,Y, and Z Periodic */
1470: /* If not X periodic */
1471: if (!DAXPeriodic(wrap)){
1472: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
1473: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1474: }
1476: /* If not Y periodic */
1477: if (!DAYPeriodic(wrap)){
1478: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
1479: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1480: }
1482: /* If not Z periodic */
1483: if (!DAZPeriodic(wrap)){
1484: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
1485: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1486: }
1488: nn = 0;
1490: /* Bottom Level */
1491: for (k=0; k<s_z; k++) {
1492: for (i=1; i<=s_y; i++) {
1493: if (n0 >= 0) { /* left below */
1494: x_t = lx[n0 % m]*dof;
1495: y_t = ly[(n0 % (m*n))/m];
1496: z_t = lz[n0 / (m*n)];
1497: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1498: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1499: }
1500: if (n1 >= 0) { /* directly below */
1501: x_t = x;
1502: y_t = ly[(n1 % (m*n))/m];
1503: z_t = lz[n1 / (m*n)];
1504: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1505: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1506: }
1507: if (n2 >= 0) { /* right below */
1508: x_t = lx[n2 % m]*dof;
1509: y_t = ly[(n2 % (m*n))/m];
1510: z_t = lz[n2 / (m*n)];
1511: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1512: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1513: }
1514: }
1516: for (i=0; i<y; i++) {
1517: if (n3 >= 0) { /* directly left */
1518: x_t = lx[n3 % m]*dof;
1519: y_t = y;
1520: z_t = lz[n3 / (m*n)];
1521: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1522: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1523: }
1525: if (n4 >= 0) { /* middle */
1526: x_t = x;
1527: y_t = y;
1528: z_t = lz[n4 / (m*n)];
1529: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1530: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1531: }
1533: if (n5 >= 0) { /* directly right */
1534: x_t = lx[n5 % m]*dof;
1535: y_t = y;
1536: z_t = lz[n5 / (m*n)];
1537: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1538: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1539: }
1540: }
1542: for (i=1; i<=s_y; i++) {
1543: if (n6 >= 0) { /* left above */
1544: x_t = lx[n6 % m]*dof;
1545: y_t = ly[(n6 % (m*n))/m];
1546: z_t = lz[n6 / (m*n)];
1547: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1548: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1549: }
1550: if (n7 >= 0) { /* directly above */
1551: x_t = x;
1552: y_t = ly[(n7 % (m*n))/m];
1553: z_t = lz[n7 / (m*n)];
1554: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1555: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1556: }
1557: if (n8 >= 0) { /* right above */
1558: x_t = lx[n8 % m]*dof;
1559: y_t = ly[(n8 % (m*n))/m];
1560: z_t = lz[n8 / (m*n)];
1561: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1562: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1563: }
1564: }
1565: }
1567: /* Middle Level */
1568: for (k=0; k<z; k++) {
1569: for (i=1; i<=s_y; i++) {
1570: if (n9 >= 0) { /* left below */
1571: x_t = lx[n9 % m]*dof;
1572: y_t = ly[(n9 % (m*n))/m];
1573: /* z_t = z; */
1574: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1575: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1576: }
1577: if (n10 >= 0) { /* directly below */
1578: x_t = x;
1579: y_t = ly[(n10 % (m*n))/m];
1580: /* z_t = z; */
1581: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1582: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1583: }
1584: if (n11 >= 0) { /* right below */
1585: x_t = lx[n11 % m]*dof;
1586: y_t = ly[(n11 % (m*n))/m];
1587: /* z_t = z; */
1588: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1589: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1590: }
1591: }
1593: for (i=0; i<y; i++) {
1594: if (n12 >= 0) { /* directly left */
1595: x_t = lx[n12 % m]*dof;
1596: y_t = y;
1597: /* z_t = z; */
1598: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1599: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1600: }
1602: /* Interior */
1603: s_t = bases[rank] + i*x + k*x*y;
1604: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1606: if (n14 >= 0) { /* directly right */
1607: x_t = lx[n14 % m]*dof;
1608: y_t = y;
1609: /* z_t = z; */
1610: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1611: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1612: }
1613: }
1615: for (i=1; i<=s_y; i++) {
1616: if (n15 >= 0) { /* left above */
1617: x_t = lx[n15 % m]*dof;
1618: y_t = ly[(n15 % (m*n))/m];
1619: /* z_t = z; */
1620: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1621: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1622: }
1623: if (n16 >= 0) { /* directly above */
1624: x_t = x;
1625: y_t = ly[(n16 % (m*n))/m];
1626: /* z_t = z; */
1627: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1628: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1629: }
1630: if (n17 >= 0) { /* right above */
1631: x_t = lx[n17 % m]*dof;
1632: y_t = ly[(n17 % (m*n))/m];
1633: /* z_t = z; */
1634: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1635: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1636: }
1637: }
1638: }
1639:
1640: /* Upper Level */
1641: for (k=0; k<s_z; k++) {
1642: for (i=1; i<=s_y; i++) {
1643: if (n18 >= 0) { /* left below */
1644: x_t = lx[n18 % m]*dof;
1645: y_t = ly[(n18 % (m*n))/m];
1646: /* z_t = lz[n18 / (m*n)]; */
1647: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1648: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1649: }
1650: if (n19 >= 0) { /* directly below */
1651: x_t = x;
1652: y_t = ly[(n19 % (m*n))/m];
1653: /* z_t = lz[n19 / (m*n)]; */
1654: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1655: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1656: }
1657: if (n20 >= 0) { /* right belodof */
1658: x_t = lx[n20 % m]*dof;
1659: y_t = ly[(n20 % (m*n))/m];
1660: /* z_t = lz[n20 / (m*n)]; */
1661: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1662: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1663: }
1664: }
1666: for (i=0; i<y; i++) {
1667: if (n21 >= 0) { /* directly left */
1668: x_t = lx[n21 % m]*dof;
1669: y_t = y;
1670: /* z_t = lz[n21 / (m*n)]; */
1671: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1672: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1673: }
1675: if (n22 >= 0) { /* middle */
1676: x_t = x;
1677: y_t = y;
1678: /* z_t = lz[n22 / (m*n)]; */
1679: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1680: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1681: }
1683: if (n23 >= 0) { /* directly right */
1684: x_t = lx[n23 % m]*dof;
1685: y_t = y;
1686: /* z_t = lz[n23 / (m*n)]; */
1687: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1688: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1689: }
1690: }
1692: for (i=1; i<=s_y; i++) {
1693: if (n24 >= 0) { /* left above */
1694: x_t = lx[n24 % m]*dof;
1695: y_t = ly[(n24 % (m*n))/m];
1696: /* z_t = lz[n24 / (m*n)]; */
1697: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1698: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1699: }
1700: if (n25 >= 0) { /* directly above */
1701: x_t = x;
1702: y_t = ly[(n25 % (m*n))/m];
1703: /* z_t = lz[n25 / (m*n)]; */
1704: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1705: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1706: }
1707: if (n26 >= 0) { /* right above */
1708: x_t = lx[n26 % m]*dof;
1709: y_t = ly[(n26 % (m*n))/m];
1710: /* z_t = lz[n26 / (m*n)]; */
1711: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1712: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1713: }
1714: }
1715: }
1717: /* construct the local to local scatter context */
1718: /*
1719: We simply remap the values in the from part of
1720: global to local to read from an array with the ghost values
1721: rather then from the plan array.
1722: */
1723: VecScatterCopy(gtol,&da->ltol);
1724: PetscLogObjectParent(da,da->ltol);
1725: left = xs - Xs;
1726: bottom = ys - Ys; top = bottom + y;
1727: down = zs - Zs; up = down + z;
1728: count = x*(top-bottom)*(up-down);
1729: PetscMalloc(count*sizeof(int),&idx);
1730: count = 0;
1731: for (i=down; i<up; i++) {
1732: for (j=bottom; j<top; j++) {
1733: for (k=0; k<x; k++) {
1734: idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
1735: }
1736: }
1737: }
1738: VecScatterRemap(da->ltol,idx,PETSC_NULL);
1739: PetscFree(idx);
1742: /*
1743: Build the natural ordering to PETSc ordering mappings.
1744: */
1745: PetscOptionsHasName(PETSC_NULL,"-da_noao",&flg1);
1746: if (!flg1) {
1747: IS ispetsc,isnatural;
1748: int *lidx,lict = 0;
1749: int Nlocal = (da->xe-da->xs)*(da->ye-da->ys)*(da->ze-da->zs);
1751: ISCreateStride(comm,Nlocal,da->base,1,&ispetsc);
1753: PetscMalloc(Nlocal*sizeof(int),&lidx);
1754: for (k=zs; k<ze; k++) {
1755: for (j=ys; j<ye; j++) {
1756: for (i=xs; i<xe; i++) {
1757: lidx[lict++] = i + j*M*dof + k*M*N*dof;
1758: }
1759: }
1760: }
1761: ISCreateGeneral(comm,Nlocal,lidx,&isnatural);
1762: PetscFree(lidx);
1764: AOCreateBasicIS(isnatural,ispetsc,&da->ao);
1765: PetscLogObjectParent(da,da->ao);
1766: ISDestroy(ispetsc);
1767: ISDestroy(isnatural);
1768: } else {
1769: da->ao = PETSC_NULL;
1770: }
1772: if (!flx) {
1773: PetscMalloc(m*sizeof(int),&flx);
1774: PetscMemcpy(flx,lx,m*sizeof(int));
1775: }
1776: if (!fly) {
1777: PetscMalloc(n*sizeof(int),&fly);
1778: PetscMemcpy(fly,ly,n*sizeof(int));
1779: }
1780: if (!flz) {
1781: PetscMalloc(p*sizeof(int),&flz);
1782: PetscMemcpy(flz,lz,p*sizeof(int));
1783: }
1784: da->lx = flx;
1785: da->ly = fly;
1786: da->lz = flz;
1788: /*
1789: Note the following will be removed soon. Since the functionality
1790: is replaced by the above. */
1792: /* Construct the mapping from current global ordering to global
1793: ordering that would be used if only 1 processor were employed.
1794: This mapping is intended only for internal use by discrete
1795: function and matrix viewers.
1797: Note: At this point, x has already been adjusted for multiple
1798: degrees of freedom per node.
1799: */
1800: ldim = x*y*z;
1801: VecGetSize(global,&gdim);
1802: PetscMalloc(gdim*sizeof(int),&da->gtog1);
1803: PetscLogObjectMemory(da,gdim*sizeof(int));
1804: PetscMalloc((2*(gdim+ldim))*sizeof(int),&gA);
1805: gB = (int *)(gA + ldim);
1806: gAall = (int *)(gB + ldim);
1807: gBall = (int *)(gAall + gdim);
1808: /* Compute local parts of global orderings */
1809: ict = 0;
1810: for (k=zs; k<ze; k++) {
1811: for (j=ys; j<ye; j++) {
1812: for (i=xs; i<xe; i++) {
1813: /* gA = global number for 1 proc; gB = current global number */
1814: gA[ict] = i + j*M*dof + k*M*N*dof;
1815: gB[ict] = start + ict;
1816: ict++;
1817: }
1818: }
1819: }
1820: /* Broadcast the orderings */
1821: MPI_Allgatherv(gA,ldim,MPI_INT,gAall,ldims,bases,MPI_INT,comm);
1822: MPI_Allgatherv(gB,ldim,MPI_INT,gBall,ldims,bases,MPI_INT,comm);
1823: for (i=0; i<gdim; i++) da->gtog1[gBall[i]] = gAall[i];
1824: PetscFree(gA);
1825: PetscFree(bases);
1827: PetscOptionsHasName(PETSC_NULL,"-da_view",&flg1);
1828: if (flg1) {DAView(da,PETSC_VIEWER_STDOUT_(da->comm));}
1829: PetscOptionsHasName(PETSC_NULL,"-da_view_draw",&flg1);
1830: if (flg1) {DAView(da,PETSC_VIEWER_DRAW_(da->comm));}
1831: PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
1832: if (flg1) {DAPrintHelp(da);}
1833: PetscPublishAll(da);
1835: #if defined(PETSC_HAVE_AMS)
1836: PetscObjectComposeFunctionDynamic((PetscObject)global,"AMSSetFieldBlock_C",
1837: "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1838: PetscObjectComposeFunctionDynamic((PetscObject)local,"AMSSetFieldBlock_C",
1839: "AMSSetFieldBlock_DA",AMSSetFieldBlock_DA);
1840: if (((PetscObject)global)->amem > -1) {
1841: AMSSetFieldBlock_DA(((PetscObject)global)->amem,"values",global);
1842: }
1843: #endif
1844: VecSetOperation(global,VECOP_VIEW,(void(*)())VecView_MPI_DA);
1845: VecSetOperation(global,VECOP_LOADINTOVECTOR,(void(*)())VecLoadIntoVector_Binary_DA);
1846: return(0);
1847: }